diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 274fbd3..788fa95 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,9 +13,9 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: "3.12" + python-version: "3.14" cache: 'pip' - cache-dependency-path: 'requirements-dev.txt' + cache-dependency-path: 'requirements-all.txt' - name: Install Ruff run: | python -m pip install --upgrade pip @@ -26,7 +26,7 @@ jobs: build: strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] platform: [octave] os: [ubuntu-latest] @@ -41,7 +41,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: 'requirements-dev.txt' + cache-dependency-path: 'requirements-all.txt' - name: Install Octave (Linux) if: matrix.platform == 'octave' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 27c23df..339c39a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,7 +7,7 @@ jobs: build: strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] platform: [octave] os: [ubuntu-latest] @@ -22,7 +22,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: 'requirements-dev.txt' + cache-dependency-path: 'requirements-all.txt' - name: Install Octave (Linux) if: matrix.platform == 'octave' @@ -45,12 +45,12 @@ jobs: - name: Clone this repository uses: actions/checkout@v3 - - name: Set up Python 3.12 + - name: Set up Python 3.14 uses: actions/setup-python@v4 with: - python-version: 3.12 + python-version: 3.14 cache: 'pip' - cache-dependency-path: 'requirements-dev.txt' + cache-dependency-path: 'requirements-all.txt' - name: Install build dependencies run: | diff --git a/.gitignore b/.gitignore index baeba57..60f82c6 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,6 @@ dmypy.json # Mac .DS_Store + +# test +tests/data/*.xlsx diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c33521c..412bb7e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,9 @@ -default_stages: [commit] +default_stages: [pre-commit] repos: # check yaml and end of file fixer - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v6.0.0 hooks: - id: check-yaml - id: end-of-file-fixer @@ -12,7 +12,7 @@ repos: # autofix using ruff - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.5.0 + rev: v0.14.10 hooks: # Run the linter. - id: ruff @@ -21,4 +21,3 @@ repos: # Run the formatter. - id: ruff-format types_or: [ python, pyi, jupyter ] - # args: [ --verbose ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bb88acd..4c20be2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,19 @@ -# Contirbuting +# Contributing + +## env + +```sh +uv venv env --python 3.14 +source env/bin/activate +uv pip install pip +``` + +## packages + +```sh +pip install pru +pru -r requirements-all.txt +``` ## Install in development mode diff --git a/matpowercaseframes/core.py b/matpowercaseframes/core.py index a2e6c3a..c4f7ddf 100644 --- a/matpowercaseframes/core.py +++ b/matpowercaseframes/core.py @@ -78,37 +78,46 @@ def _read_data( # TODO: support Path # TYPE: str of path path = self._get_path(data) - path_no_ext, ext = os.path.splitext(path) - if ext == ".m": - # read `.m` file - if load_case_engine is None: - # read with matpower parser - self._read_matpower( - filepath=path, - allow_any_keys=allow_any_keys, - ) - else: - # read using loadcase - mpc = load_case_engine.loadcase(path) - self._read_oct2py_struct( - struct=mpc, - allow_any_keys=allow_any_keys, - ) - elif ext == ".xlsx": - # read `.xlsx` file - self._read_excel( - filepath=path, + # check if path is a directory (for CSV files) + if os.path.isdir(path): + self._read_csv_dir( + dirpath=path, prefix=prefix, suffix=suffix, allow_any_keys=allow_any_keys, ) - self.name = os.path.basename(path_no_ext) + self.name = os.path.basename(path) else: - # TODO: support read directory of csv for schema and .csv data - message = f"Can't find data at {data}" - raise FileNotFoundError(message) - + path_no_ext, ext = os.path.splitext(path) + + if ext == ".m": + # read `.m` file + if load_case_engine is None: + # read with matpower parser + self._read_matpower( + filepath=path, + allow_any_keys=allow_any_keys, + ) + else: + # read using loadcase + mpc = load_case_engine.loadcase(path) + self._read_oct2py_struct( + struct=mpc, + allow_any_keys=allow_any_keys, + ) + elif ext == ".xlsx": + # read `.xlsx` file + self._read_excel( + filepath=path, + prefix=prefix, + suffix=suffix, + allow_any_keys=allow_any_keys, + ) + self.name = os.path.basename(path_no_ext) + else: + message = f"Can't find data at {os.path.abspath(data)}" + raise FileNotFoundError(message) elif isinstance(data, dict): # TYPE: dict | oct2py.io.Struct self._read_oct2py_struct( @@ -162,15 +171,17 @@ def _get_path(path): Determine the correct file path for the given input. Args: - path (str): File path or MATPOWER case name. + path (str): File path, directory path, or MATPOWER case name. Returns: - str: Resolved file path. + str: Resolved file path or directory path. Raises: FileNotFoundError: If the file or MATPOWER case cannot be found. """ - # TODO: support read directory of csv for schema and .csv data + # directory exist on path (for CSV directory) + if os.path.isdir(path): + return path # file exist on path if os.path.isfile(path): @@ -198,7 +209,9 @@ def _get_path(path): if os.path.isfile(path_added_matpower_m): return path_added_matpower_m - raise FileNotFoundError + # Create detailed error message + error_msg = f"Could not find file or directory '{path}'." + raise FileNotFoundError(error_msg) def _read_matpower(self, filepath, allow_any_keys=False): """ @@ -333,6 +346,64 @@ def _read_excel(self, filepath, prefix="", suffix="", allow_any_keys=False): self.setattr(attribute, value) + def _read_csv_dir(self, dirpath, prefix="", suffix="", allow_any_keys=False): + """ + Read data from a directory of CSV files. + + Args: + dirpath (str): Directory path containing the CSV files. + prefix (str): File prefix for each attribute CSV file. + suffix (str): File suffix for each attribute CSV file. + allow_any_keys (bool): Whether to allow any keys beyond ATTRIBUTES. + """ + # create a dictionary mapping attribute names to file paths + csv_data = {} + for csv_file in os.listdir(dirpath): + if csv_file.endswith(".csv"): + # remove prefix and suffix to get the attribute name + attribute = csv_file[:-4] # remove '.csv' extension + + if prefix and attribute.startswith(prefix): + attribute = attribute[len(prefix) :] + if suffix and attribute.endswith(suffix): + attribute = attribute[: -len(suffix)] + + csv_data[attribute] = os.path.join(dirpath, csv_file) + + self._attributes = [] + + # info CSV to extract general metadata + info_name = "info" + if info_name in csv_data: + info_data = pd.read_csv(csv_data[info_name], index_col=0) + + value = info_data.loc["version", "INFO"].item() + self.setattr("version", str(value)) + + value = info_data.loc["baseMVA", "INFO"].item() + self.setattr("baseMVA", value) + + # iterate through the remaining CSV files + for attribute, filepath in csv_data.items(): + # skip the info file + if attribute == info_name: + continue + + # check attribute rule + if attribute not in ATTRIBUTES and not allow_any_keys: + continue + + # read CSV file + sheet_data = pd.read_csv(filepath, index_col=0) + + if attribute in ["bus_name", "branch_name", "gen_name"]: + # convert back to an index + value = pd.Index(sheet_data[attribute].values.tolist(), name=attribute) + else: + value = sheet_data + + self.setattr(attribute, value) + def _get_dataframe(self, attribute, data, n_cols=None, columns_template=None): """ Create a DataFrame with proper columns from raw data. diff --git a/notebooks/load_excel.ipynb b/notebooks/load_excel.ipynb index 9ee1245..70a89c4 100644 --- a/notebooks/load_excel.ipynb +++ b/notebooks/load_excel.ipynb @@ -42,7 +42,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -53,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -69,7 +69,7 @@ " dtype='object', name='bus_name', length=118)" ] }, - "execution_count": 5, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -81,6 +81,30 @@ "# pd.Index(sheet_data[attribute].values.tolist(), name=attribute)" ] }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "path = \"../tests/data/case118_test_to_xlsx.xlsx\"\n", + "prefix = \"\"\n", + "suffix = \"\"\n", + "cf_org.to_excel(path, prefix=prefix, suffix=suffix) # write to .xlsx file" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "path = \"../tests/data/case118_test_to_xlsx_prefix_suffix.xlsx\"\n", + "prefix = \"mpc.\"\n", + "suffix = \"_test\"\n", + "cf_org.to_excel(path, prefix=prefix, suffix=suffix) # write to .xlsx file" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -97,7 +121,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -153,7 +177,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -400,7 +424,7 @@ " dtype='object', name='bus_name', length=118)}" ] }, - "execution_count": 7, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -412,7 +436,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -421,7 +445,7 @@ "dict_keys(['info', 'bus', 'gen', 'branch', 'gencost', 'bus_name'])" ] }, - "execution_count": 8, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -439,7 +463,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -448,7 +472,7 @@ "['version', 'baseMVA', 'bus', 'gen', 'branch', 'gencost', 'bus_name']" ] }, - "execution_count": 9, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -467,7 +491,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -475,13 +499,13 @@ "output_type": "stream", "text": [ "\n", - "MATPOWER Version 8.0, 17-May-2024\n", + "MATPOWER Version 8.1, 12-Jul-2025\n", "Power Flow -- AC-polar-power formulation\n", "\n", "Newton's method converged in 3 iterations.\n", "PF successful\n", "\n", - "Converged in 0.13 seconds\n", + "Converged in 0.06 seconds\n", "================================================================================\n", "| System Summary |\n", "================================================================================\n", @@ -851,7 +875,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -907,7 +931,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -1154,7 +1178,7 @@ " dtype='object', name='bus_name', length=118)}" ] }, - "execution_count": 12, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -1166,7 +1190,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -1175,7 +1199,7 @@ "dict_keys(['mpc.info_test', 'mpc.bus_test', 'mpc.gen_test', 'mpc.branch_test', 'mpc.gencost_test', 'mpc.bus_name_test'])" ] }, - "execution_count": 13, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -1193,28 +1217,7 @@ }, { "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['version', 'baseMVA', 'bus', 'gen', 'branch', 'gencost', 'bus_name']" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cf = CaseFrames(path, prefix=prefix, suffix=suffix)\n", - "cf.attributes" - ] - }, - { - "cell_type": "code", - "execution_count": 15, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -3943,7 +3946,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -3951,13 +3954,13 @@ "output_type": "stream", "text": [ "\n", - "MATPOWER Version 8.0, 17-May-2024\n", + "MATPOWER Version 8.1, 12-Jul-2025\n", "Power Flow -- AC-polar-power formulation\n", "\n", "Newton's method converged in 3 iterations.\n", "PF successful\n", "\n", - "Converged in 0.09 seconds\n", + "Converged in 0.04 seconds\n", "================================================================================\n", "| System Summary |\n", "================================================================================\n", @@ -4320,7 +4323,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -4351,7 +4354,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.8" + "version": "3.12.12" } }, "nbformat": 4, diff --git a/pyproject.toml b/pyproject.toml index 092a7c7..1cb39d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,8 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Mathematics", "License :: OSI Approved :: MIT License", diff --git a/requirements-all.txt b/requirements-all.txt new file mode 100644 index 0000000..4513486 --- /dev/null +++ b/requirements-all.txt @@ -0,0 +1,15 @@ +pandas==2.3.3 +numpy==2.4.0 + +openpyxl==3.1.5 + +oct2py==5.8.0 +matpower==8.1.0.2.2.3 + +pre-commit==3.8.0 +ruff==0.14.10 +setuptools==80.9.0 + +pytest==9.0.2 +pytest-cov==7.0.0 +pytest-xdist==3.8.0 diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 8df6db0..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,15 +0,0 @@ -pandas==2.2.2 -numpy==2.1.1 - -openpyxl==3.1.5 - -oct2py==5.7.2 -matpower==8.0.0.2.1.8 - -pre-commit==3.8.0 -ruff==0.6.4 -setuptools==74.1.2 - -pytest==8.3.2 -pytest-cov==5.0.0 -pytest-xdist==3.6.1 diff --git a/tests/data/case118_test_to_xlsx.xlsx b/tests/data/case118_test_to_xlsx.xlsx deleted file mode 100644 index 227535c..0000000 Binary files a/tests/data/case118_test_to_xlsx.xlsx and /dev/null differ diff --git a/tests/data/case118_test_to_xlsx_prefix_suffix.xlsx b/tests/data/case118_test_to_xlsx_prefix_suffix.xlsx deleted file mode 100644 index f7bf16f..0000000 Binary files a/tests/data/case118_test_to_xlsx_prefix_suffix.xlsx and /dev/null differ diff --git a/tests/data/case9_test_to_xlsx.xlsx b/tests/data/case9_test_to_xlsx.xlsx deleted file mode 100644 index 495d064..0000000 Binary files a/tests/data/case9_test_to_xlsx.xlsx and /dev/null differ diff --git a/tests/data/case9_test_to_xlsx_prefix_suffix.xlsx b/tests/data/case9_test_to_xlsx_prefix_suffix.xlsx deleted file mode 100644 index f3490d6..0000000 Binary files a/tests/data/case9_test_to_xlsx_prefix_suffix.xlsx and /dev/null differ diff --git a/tests/test_core.py b/tests/test_core.py index f6472e0..8fc2116 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -15,32 +15,31 @@ CURDIR = os.path.realpath(os.path.dirname(__file__)) CASE_DIR = os.path.join(os.path.dirname(CURDIR), "data") CASE_PATH_CASE9 = os.path.join(CASE_DIR, CASE_NAME_CASE9) +ATTRIBUTES_CASE9 = ["version", "baseMVA", "bus", "gen", "branch", "gencost"] CASE_NAME_CASE118 = "case118.m" CURDIR = os.path.realpath(os.path.dirname(__file__)) CASE_DIR = os.path.join(os.path.dirname(CURDIR), "data") CASE_PATH_CASE118 = os.path.join(CASE_DIR, CASE_NAME_CASE118) +ATTRIBUTES_CASE118 = [ + "version", + "baseMVA", + "bus", + "gen", + "branch", + "gencost", + "bus_name", +] +ATTRIBUTES_CASE = { + "case9": ATTRIBUTES_CASE9, + "case118": ATTRIBUTES_CASE118, +} def test_input_str_path(): CaseFrames(CASE_PATH_CASE9) -def test_read_excel(): - CASE_NAME = "tests/data/case118_test_to_xlsx.xlsx" - cf = CaseFrames(CASE_NAME) - for attribute in [ - "version", - "baseMVA", - "bus", - "gen", - "branch", - "gencost", - "bus_name", - ]: - assert attribute in cf.attributes - - def test_input_oct2py_io_Struct(): from matpower import start_instance @@ -149,40 +148,107 @@ def test_get_attributes(): # pytest -n auto --durations=0 -def test_to_xlsx(): - cf = CaseFrames(CASE_PATH_CASE9) - cf.to_excel("tests/results/case9/case9_test_to_xlsx.xlsx") - cf.to_excel( - "tests/results/case9_prefix_suffix/case9_test_to_xlsx_prefix_suffix.xlsx", - prefix="mpc.", - suffix="_test", - ) - - cf = CaseFrames(CASE_PATH_CASE118) - cf.to_excel("tests/results/case118/case118_test_to_xlsx.xlsx") - cf.to_excel( - "tests/results/case118_prefix_suffix/case118_test_to_xlsx_prefix_suffix.xlsx", - prefix="mpc.", - suffix="_test", - ) - - -def test_to_csv(): - cf = CaseFrames(CASE_PATH_CASE9) - cf.to_csv("tests/results/case9") - cf.to_csv("tests/results/case9_prefix_suffix", prefix="mpc.", suffix="_test") - - cf = CaseFrames(CASE_PATH_CASE118) - cf.to_csv("tests/results/case118") - cf.to_csv("tests/results/case118_prefix_suffix", prefix="mpc.", suffix="_test") - - -def test_to_schema(): - cf = CaseFrames(CASE_PATH_CASE9) - cf.to_schema("tests/results/case9/schema") - - cf = CaseFrames(CASE_PATH_CASE118) - cf.to_schema("tests/results/case118/schema") +@pytest.mark.parametrize( + "case_path,attributes,output_path,prefix,suffix", + [ + ( + CASE_PATH_CASE9, + ATTRIBUTES_CASE9, + "tests/results/case9/case9_test_to_xlsx.xlsx", + "", + "", + ), + ( + CASE_PATH_CASE9, + ATTRIBUTES_CASE9, + "tests/results/case9_prefix_suffix/case9_test_to_xlsx_prefix_suffix.xlsx", + "mpc.", + "_test", + ), + ( + CASE_PATH_CASE118, + ATTRIBUTES_CASE118, + "tests/results/case118/case118_test_to_xlsx.xlsx", + "", + "", + ), + ( + CASE_PATH_CASE118, + ATTRIBUTES_CASE118, + "tests/results/case118_prefix_suffix/case118_test_to_xlsx_prefix_suffix.xlsx", + "mpc.", + "_test", + ), + ], + ids=["case9", "case9_prefix_suffix", "case118", "case118_prefix_suffix"], +) +def test_to_and_read_xlsx(case_path, attributes, output_path, prefix, suffix): + cf = CaseFrames(case_path) # read .m file + cf.to_excel(output_path, prefix=prefix, suffix=suffix) # write to .xlsx file + cf = CaseFrames( + output_path, prefix=prefix, suffix=suffix + ) # read back from .xlsx file + for attribute in attributes: + assert attribute in cf.attributes, ( + f"Missing attribute '{attribute}' in {cf.attributes}" + ) + + +@pytest.mark.parametrize( + "case_path,attributes,output_dir,prefix,suffix", + [ + (CASE_PATH_CASE9, ATTRIBUTES_CASE9, "tests/results/case9", "", ""), + ( + CASE_PATH_CASE9, + ATTRIBUTES_CASE9, + "tests/results/case9_prefix_suffix", + "mpc.", + "_test", + ), + (CASE_PATH_CASE118, ATTRIBUTES_CASE118, "tests/results/case118", "", ""), + ( + CASE_PATH_CASE118, + ATTRIBUTES_CASE118, + "tests/results/case118_prefix_suffix", + "mpc.", + "_test", + ), + ], + ids=["case9", "case9_prefix_suffix", "case118", "case118_prefix_suffix"], +) +def test_to_and_read_csv(case_path, attributes, output_dir, prefix, suffix): + cf = CaseFrames(case_path) # read .m file + cf.to_csv(output_dir, prefix=prefix, suffix=suffix) # write to .csv directory + cf = CaseFrames( + output_dir, prefix=prefix, suffix=suffix + ) # read back from .csv directory + for attribute in attributes: + assert attribute in cf.attributes, ( + f"Missing attribute '{attribute}' in {cf.attributes}" + ) + + +@pytest.mark.parametrize( + "case_path,schema_dir,case_name", + [ + (CASE_PATH_CASE9, "tests/results/case9/schema", "case9"), + (CASE_PATH_CASE118, "tests/results/case118/schema", "case118"), + ], + ids=["case9", "case118"], +) +def test_to_schema(case_path, schema_dir, case_name): + cf = CaseFrames(case_path) + cf.to_schema(schema_dir) + assert os.path.isdir(schema_dir), f"Schema directory '{schema_dir}' was not created" + + schema_files = os.listdir(schema_dir) + assert len(schema_files) > 0, f"No schema files found in '{schema_dir}'" + + cf = CaseFrames(schema_dir) + for attribute in ATTRIBUTES_CASE[case_name]: + assert attribute in cf.attributes, ( + f"Missing attribute '{attribute}' in {cf.attributes}" + ) def test_to_dict():