From e233e375944d64add870cf63fe27fcff5ff9447b Mon Sep 17 00:00:00 2001 From: jbjd Date: Tue, 16 Dec 2025 22:27:57 -0600 Subject: [PATCH 1/4] poc --- .../parser/skipper.py | 68 ++++++++++++------- personal_python_ast_optimizer/parser/utils.py | 6 ++ tests/parser/test_excludes.py | 9 ++- 3 files changed, 56 insertions(+), 27 deletions(-) diff --git a/personal_python_ast_optimizer/parser/skipper.py b/personal_python_ast_optimizer/parser/skipper.py index 55315d3..129ef0d 100644 --- a/personal_python_ast_optimizer/parser/skipper.py +++ b/personal_python_ast_optimizer/parser/skipper.py @@ -13,6 +13,7 @@ machine_dependent_functions, ) from personal_python_ast_optimizer.parser.utils import ( + exclude_imports, first_occurrence_of_type, get_node_name, is_overload_function, @@ -345,41 +346,21 @@ def visit_AugAssign(self, node: ast.AugAssign) -> ast.AST | None: def visit_Import(self, node: ast.Import) -> ast.AST | None: """Removes imports provided in config, deleting the whole node if no imports are left""" - node.names = [ - alias - for alias in node.names - if alias.name not in self.tokens_config.module_imports_to_skip - ] - - if not node.names: - return None + exclude_imports(node, self.tokens_config.module_imports_to_skip) - return self.generic_visit(node) + return self.generic_visit(node) if node.names else None def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.AST | None: normalized_module_name: str = node.module or "" if normalized_module_name in self.tokens_config.module_imports_to_skip: return None - node.names = [ - alias - for alias in node.names - if alias.name not in self.tokens_config.from_imports_to_skip - and alias.name not in self.optimizations_config.vars_to_fold - and alias.name not in self.optimizations_config.enums_to_fold - ] + exclude_imports(node, self.tokens_config.from_imports_to_skip) if node.module == "__future__" and self._skippable_futures: - node.names = [ - alias - for alias in node.names - if alias.name not in self._skippable_futures - ] + exclude_imports(node, self._skippable_futures) - if not node.names: - return None - - return self.generic_visit(node) + return self.generic_visit(node) if node.names else None def visit_Name(self, node: ast.Name) -> ast.AST: """Extends super's implementation by adding constant folding""" @@ -649,3 +630,40 @@ def _ast_constants_operation( raise ValueError(f"Invalid operation: {operation.__class__.__name__}") return ast.Constant(result) + + +class ImportSkipper: + + def __init__(self, module_imports_to_skip, from_imports_to_skip): + self.module_imports_to_skip = module_imports_to_skip + self.from_imports_to_skip = from_imports_to_skip + + def generic_visit(self, node): + for _, old_value in ast.iter_fields(node): + if isinstance(old_value, list): + new_values = [] + for value in old_value: + if isinstance(value, (ast.Import, ast.ImportFrom)): + value = self.visit_ImportOrImportFrom(value) + + if value is not None: + new_values.append(value) + + old_value[:] = new_values + + # I think below is unneeded? + + # elif isinstance(old_value, ast.AST): + # new_node = self.visit(old_value) + # if new_node is None: + # delattr(node, field) + # else: + # setattr(node, field, new_node) + return node + + def visit_ImportOrImportFrom( + self, node: ast.Import | ast.ImportFrom + ) -> ast.Import | ast.ImportFrom | None: + exclude_imports(node, self.module_imports_to_skip) + + return node if node.names else None diff --git a/personal_python_ast_optimizer/parser/utils.py b/personal_python_ast_optimizer/parser/utils.py index 11ec7a7..9b1a5b8 100644 --- a/personal_python_ast_optimizer/parser/utils.py +++ b/personal_python_ast_optimizer/parser/utils.py @@ -5,6 +5,12 @@ from personal_python_ast_optimizer.parser.config import TokensToSkip +def exclude_imports(node: ast.Import | ast.ImportFrom, exlcudes: Iterable[str]) -> None: + node.names = [ + alias for alias in node.names if (alias.asname or alias.name) not in exlcudes + ] + + def get_node_name(node: ast.AST | None) -> str: """Gets id or attr which both can represent var names""" if isinstance(node, ast.Call): diff --git a/tests/parser/test_excludes.py b/tests/parser/test_excludes.py index 4f65f0a..13c1ee1 100644 --- a/tests/parser/test_excludes.py +++ b/tests/parser/test_excludes.py @@ -160,12 +160,17 @@ def test_exclude_module_imports(): import numpy from numpy._core import uint8 from . import asdf +import a +import a as b +import a as c """, - "", + "import a as c", ) run_minifier_and_assert_correct( before_and_after, - tokens_config=TokensConfig(module_imports_to_skip={"numpy", "numpy._core", ""}), + tokens_config=TokensConfig( + module_imports_to_skip={"numpy", "numpy._core", "", "a", "b"} + ), ) From 10d422001748a5d9e412600ec87bd088fdc5b9c5 Mon Sep 17 00:00:00 2001 From: jbjd Date: Wed, 17 Dec 2025 17:08:18 -0600 Subject: [PATCH 2/4] Remove unused imports --- README.md | 2 +- .../flake_wrapper.py | 17 ---- .../parser/config.py | 3 + .../parser/skipper.py | 90 +++++++++++++------ pyproject.toml | 3 +- requirements.txt | 1 - tests/parser/test_enum_folding.py | 4 +- tests/parser/test_excludes.py | 7 +- tests/parser/test_imports.py | 27 +++++- tests/parser/test_real.py | 4 +- tests/parser/test_script.py | 6 +- version.txt | 2 +- 12 files changed, 106 insertions(+), 60 deletions(-) delete mode 100644 personal_python_ast_optimizer/flake_wrapper.py delete mode 100644 requirements.txt diff --git a/README.md b/README.md index 5e8d3b7..37f9f13 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Personal Python AST Optimizer -This is a project related to minifying or otherwise statically optimizing Python code. It can parse python to output a much smaller output with optionally excluded sections of code. It also includes wrappers for calling autoflake. +This is a project related to minifying or otherwise statically optimizing Python code. It can parse python to output a much smaller output with optionally excluded sections of code. This project is an extension of the builtin ast module. diff --git a/personal_python_ast_optimizer/flake_wrapper.py b/personal_python_ast_optimizer/flake_wrapper.py deleted file mode 100644 index 4c933e0..0000000 --- a/personal_python_ast_optimizer/flake_wrapper.py +++ /dev/null @@ -1,17 +0,0 @@ -import autoflake - - -def run_autoflake(source: str, remove_unused_imports: bool = False) -> str: - """Runs autoflake - remove_unused_imports: defaults to False since it can be destructive - Say some module "foo" imports what another module "bar" imported. - If module "bar" does not use the imports itself, autoflake will remove it - even though "foo" needs it. - """ - return autoflake.fix_code( - source, - remove_all_unused_imports=remove_unused_imports, - remove_duplicate_keys=True, - remove_unused_variables=True, - remove_rhs_for_unused_variables=True, - ) diff --git a/personal_python_ast_optimizer/parser/config.py b/personal_python_ast_optimizer/parser/config.py index 48d4486..dff5cfb 100644 --- a/personal_python_ast_optimizer/parser/config.py +++ b/personal_python_ast_optimizer/parser/config.py @@ -123,6 +123,7 @@ class OptimizationsConfig(_Config): __slots__ = ( "vars_to_fold", "enums_to_fold", + "remove_unused_imports", "fold_constants", "assume_this_machine", ) @@ -132,6 +133,7 @@ def __init__( vars_to_fold: dict[str, int | str | bool] | None = None, enums_to_fold: Iterable[EnumType] | None = None, fold_constants: bool = True, + remove_unused_imports: bool = True, assume_this_machine: bool = False, ) -> None: self.vars_to_fold: dict[str, int | str | bool] = ( @@ -142,6 +144,7 @@ def __init__( if enums_to_fold is None else self._format_enums_to_fold_as_dict(enums_to_fold) ) + self.remove_unused_imports: bool = remove_unused_imports self.assume_this_machine: bool = assume_this_machine self.fold_constants: bool = fold_constants diff --git a/personal_python_ast_optimizer/parser/skipper.py b/personal_python_ast_optimizer/parser/skipper.py index 129ef0d..7c64fe9 100644 --- a/personal_python_ast_optimizer/parser/skipper.py +++ b/personal_python_ast_optimizer/parser/skipper.py @@ -28,6 +28,7 @@ class AstNodeSkipper(ast.NodeTransformer): __slots__ = ( + "_possibly_unused_imports", "_skippable_futures", "_within_class", "_within_function", @@ -61,6 +62,8 @@ def __init__(self, config: SkipConfig) -> None: if self.token_types_config.skip_type_hints: self._skippable_futures.append("annotations") + self._possibly_unused_imports: set[str] = set() + @staticmethod def _within_class_node(function): def wrapper(self: "AstNodeSkipper", *args, **kwargs) -> ast.AST | None: @@ -101,9 +104,9 @@ def generic_visit(self, node) -> ast.AST: new_values.append(value) if ( - field == "body" + not isinstance(node, ast.Module) and not new_values - and not isinstance(node, ast.Module) + and field == "body" ): new_values = [ast.Pass()] @@ -115,6 +118,7 @@ def generic_visit(self, node) -> ast.AST: delattr(node, field) else: setattr(node, field, new_node) + return node @staticmethod @@ -146,13 +150,19 @@ def visit_Module(self, node: ast.Module) -> ast.AST: if not self._has_code_to_skip(): return node + self._possibly_unused_imports = set() + if self.token_types_config.skip_dangling_expressions: skip_dangling_expressions(node) - try: - return self.generic_visit(node) - finally: - self._warn_unused_skips() + module: ast.Module = self.generic_visit(node) # type:ignore + + if self._possibly_unused_imports: + import_skipper = ImportSkipper(self._possibly_unused_imports) + import_skipper.generic_visit(module) + + self._warn_unused_skips() + return module @_within_class_node def visit_ClassDef(self, node: ast.ClassDef) -> ast.AST | None: @@ -233,6 +243,9 @@ def visit_Attribute(self, node: ast.Attribute) -> ast.AST | None: ): return self._get_enum_value_as_AST(node.value.attr, node.attr) + if node.attr in self._possibly_unused_imports: + self._possibly_unused_imports.remove(node.attr) + return self.generic_visit(node) def _get_enum_value_as_AST(self, class_name: str, value_name: str) -> ast.Constant: @@ -344,11 +357,15 @@ def visit_AugAssign(self, node: ast.AugAssign) -> ast.AST | None: return self.generic_visit(node) def visit_Import(self, node: ast.Import) -> ast.AST | None: - """Removes imports provided in config, deleting the whole - node if no imports are left""" exclude_imports(node, self.tokens_config.module_imports_to_skip) - return self.generic_visit(node) if node.names else None + if not node.names: + return None + + if self.optimizations_config.remove_unused_imports: + self._possibly_unused_imports.update(n.asname or n.name for n in node.names) + + return self.generic_visit(node) def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.AST | None: normalized_module_name: str = node.module or "" @@ -359,6 +376,12 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.AST | None: if node.module == "__future__" and self._skippable_futures: exclude_imports(node, self._skippable_futures) + elif ( + node.module != "__future__" + and node.names + and self.optimizations_config.remove_unused_imports + ): + self._possibly_unused_imports.update(n.asname or n.name for n in node.names) return self.generic_visit(node) if node.names else None @@ -366,8 +389,12 @@ def visit_Name(self, node: ast.Name) -> ast.AST: """Extends super's implementation by adding constant folding""" if node.id in self.optimizations_config.vars_to_fold: constant_value = self.optimizations_config.vars_to_fold[node.id] + return ast.Constant(constant_value) else: + if node.id in self._possibly_unused_imports: + self._possibly_unused_imports.remove(node.id) + return self.generic_visit(node) def visit_Dict(self, node: ast.Dict) -> ast.AST: @@ -632,38 +659,43 @@ def _ast_constants_operation( return ast.Constant(result) -class ImportSkipper: +class ImportSkipper(ast.NodeVisitor): - def __init__(self, module_imports_to_skip, from_imports_to_skip): - self.module_imports_to_skip = module_imports_to_skip - self.from_imports_to_skip = from_imports_to_skip + __slots__ = ("unused_imports",) + + def __init__(self, unused_imports: set[str]) -> None: + self.unused_imports = unused_imports def generic_visit(self, node): for _, old_value in ast.iter_fields(node): if isinstance(old_value, list): new_values = [] + ast_removed: bool = False for value in old_value: - if isinstance(value, (ast.Import, ast.ImportFrom)): - value = self.visit_ImportOrImportFrom(value) + if isinstance(value, ast.AST): + value = self.visit(value) + if value is None: + ast_removed = True + continue + elif not isinstance(value, ast.AST): + new_values.extend(value) + continue - if value is not None: - new_values.append(value) + new_values.append(value) - old_value[:] = new_values + if not isinstance(node, ast.Module) and not new_values and ast_removed: + new_values = [ast.Pass()] - # I think below is unneeded? + old_value[:] = new_values - # elif isinstance(old_value, ast.AST): - # new_node = self.visit(old_value) - # if new_node is None: - # delattr(node, field) - # else: - # setattr(node, field, new_node) return node - def visit_ImportOrImportFrom( - self, node: ast.Import | ast.ImportFrom - ) -> ast.Import | ast.ImportFrom | None: - exclude_imports(node, self.module_imports_to_skip) + def visit_Import(self, node: ast.Import) -> ast.Import | None: + exclude_imports(node, self.unused_imports) + + return node if node.names else None + + def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.ImportFrom | None: + exclude_imports(node, self.unused_imports) return node if node.names else None diff --git a/pyproject.toml b/pyproject.toml index 57ab29a..35c6ebe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,7 @@ authors = [ name = "personal-python-ast-optimizer" readme = "README.md" requires-python = ">=3.11" -dynamic = ["dependencies", "version"] +dynamic = ["version"] [tool.setuptools.dynamic] -dependencies = {file = ["requirements.txt"]} version = {file = "version.txt"} diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index f8f27fe..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -autoflake==2.3.1 diff --git a/tests/parser/test_enum_folding.py b/tests/parser/test_enum_folding.py index a720eef..630ee92 100644 --- a/tests/parser/test_enum_folding.py +++ b/tests/parser/test_enum_folding.py @@ -38,9 +38,7 @@ class _SomeIntEnum(IntEnum): import somewhere print(somewhere.someModule._SomeStrEnum.C)""", - """ -import somewhere -print('C')""".strip(), + "print('C')", ), ] diff --git a/tests/parser/test_excludes.py b/tests/parser/test_excludes.py index 13c1ee1..8189d8f 100644 --- a/tests/parser/test_excludes.py +++ b/tests/parser/test_excludes.py @@ -1,6 +1,9 @@ import pytest -from personal_python_ast_optimizer.parser.config import TokensConfig +from personal_python_ast_optimizer.parser.config import ( + OptimizationsConfig, + TokensConfig, +) from tests.utils import BeforeAndAfter, run_minifier_and_assert_correct @@ -171,6 +174,7 @@ def test_exclude_module_imports(): tokens_config=TokensConfig( module_imports_to_skip={"numpy", "numpy._core", "", "a", "b"} ), + optimizations_config=OptimizationsConfig(remove_unused_imports=False), ) @@ -192,4 +196,5 @@ def test_exclude_real_case(): tokens_config=TokensConfig( functions_to_skip={"getLogger"}, variables_to_skip={"TYPE_CHECKING"} ), + optimizations_config=OptimizationsConfig(remove_unused_imports=False), ) diff --git a/tests/parser/test_imports.py b/tests/parser/test_imports.py index 68e5a04..2e7c8cf 100644 --- a/tests/parser/test_imports.py +++ b/tests/parser/test_imports.py @@ -1,6 +1,9 @@ import pytest -from personal_python_ast_optimizer.parser.config import TokenTypesConfig +from personal_python_ast_optimizer.parser.config import ( + OptimizationsConfig, + TokenTypesConfig, +) from tests.utils import BeforeAndAfter, run_minifier_and_assert_correct _futures_imports: str = """ @@ -54,7 +57,10 @@ def i(): """import test,test2 def i():import a,d;from b import c,d as e;from .b import f;print();import e""", ) - run_minifier_and_assert_correct(before_and_after) + run_minifier_and_assert_correct( + before_and_after, + optimizations_config=OptimizationsConfig(remove_unused_imports=False), + ) def test_import_star(): @@ -63,4 +69,21 @@ def test_import_star(): "from ctypes import *", "from ctypes import*", ) + run_minifier_and_assert_correct( + before_and_after, + optimizations_config=OptimizationsConfig(remove_unused_imports=False), + ) + + +def test_remove_unused_imports(): + + before_and_after = BeforeAndAfter( + """ +if a == b: + import foo + import bar + +print(a)""", + "if a==b:pass\nprint(a)", + ) run_minifier_and_assert_correct(before_and_after) diff --git a/tests/parser/test_real.py b/tests/parser/test_real.py index b490378..1017de7 100644 --- a/tests/parser/test_real.py +++ b/tests/parser/test_real.py @@ -11,7 +11,7 @@ def test_image_viewer_constants(): File with constants needed in multiple spots of the codebase \"\"\" -from enum import IntEnum, StrEnum +from enum import StrEnum class ImageFormats(StrEnum): @@ -22,7 +22,7 @@ class ImageFormats(StrEnum): JPEG = "JPEG" PNG = "PNG" WEBP = "WebP\"""", - """from enum import IntEnum,StrEnum + """from enum import StrEnum class ImageFormats(StrEnum):DDS='DDS';GIF='GIF';JPEG='JPEG';PNG='PNG';WEBP='WebP'""", ) run_minifier_and_assert_correct(before_and_after) diff --git a/tests/parser/test_script.py b/tests/parser/test_script.py index 307091d..395826f 100644 --- a/tests/parser/test_script.py +++ b/tests/parser/test_script.py @@ -1,3 +1,4 @@ +from personal_python_ast_optimizer.parser.config import OptimizationsConfig from tests.utils import BeforeAndAfter, run_minifier_and_assert_correct @@ -61,4 +62,7 @@ def test_inline_all(): "from spam import eggs;raise Exception;return 0" ), ) - run_minifier_and_assert_correct(before_and_after) + run_minifier_and_assert_correct( + before_and_after, + optimizations_config=OptimizationsConfig(remove_unused_imports=False), + ) diff --git a/version.txt b/version.txt index 50e2274..831446c 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -5.0.3 +5.1.0 From bece609ad415f974b5dcf1705c714ef14aade905 Mon Sep 17 00:00:00 2001 From: jbjd Date: Wed, 17 Dec 2025 17:18:46 -0600 Subject: [PATCH 3/4] Fix whitespace --- personal_python_ast_optimizer/parser/skipper.py | 2 +- tests/parser/test_imports.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/personal_python_ast_optimizer/parser/skipper.py b/personal_python_ast_optimizer/parser/skipper.py index 7c64fe9..bc641ea 100644 --- a/personal_python_ast_optimizer/parser/skipper.py +++ b/personal_python_ast_optimizer/parser/skipper.py @@ -25,7 +25,7 @@ ) -class AstNodeSkipper(ast.NodeTransformer): +class AstNodeSkipper(ast.NodeVisitor): __slots__ = ( "_possibly_unused_imports", diff --git a/tests/parser/test_imports.py b/tests/parser/test_imports.py index 2e7c8cf..fa2690b 100644 --- a/tests/parser/test_imports.py +++ b/tests/parser/test_imports.py @@ -82,7 +82,7 @@ def test_remove_unused_imports(): if a == b: import foo import bar - + print(a)""", "if a==b:pass\nprint(a)", ) From c6d6d0d1c285bc1ba473ae6379105d636776401f Mon Sep 17 00:00:00 2001 From: jbjd Date: Wed, 17 Dec 2025 17:19:54 -0600 Subject: [PATCH 4/4] Remove requirements in cicd --- .github/workflows/validate-code.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/validate-code.yml b/.github/workflows/validate-code.yml index e8c2277..c48237d 100644 --- a/.github/workflows/validate-code.yml +++ b/.github/workflows/validate-code.yml @@ -37,9 +37,6 @@ jobs: chmod +x ./validate.sh ./validate.sh - - name: Install Dependencies - run: pip install -r requirements.txt - - name: Unit Tests run: | chmod +x ./test.sh