From 125df20236168d93ac48800b2667225fb2f8f369 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Mon, 3 Mar 2025 18:10:02 +0100 Subject: [PATCH 1/7] Use moz.l10n to parse TOML config files --- requirements.txt | 1 + tmx_products/tmx_projectconfig.py | 99 +++++++++++++++++++++++++------ 2 files changed, 82 insertions(+), 18 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5038f3b..20911ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ compare-locales==9.0.* +moz-l10n==0.6.0 diff --git a/tmx_products/tmx_projectconfig.py b/tmx_products/tmx_projectconfig.py index 996ad4b..2e47e9c 100755 --- a/tmx_products/tmx_projectconfig.py +++ b/tmx_products/tmx_projectconfig.py @@ -1,12 +1,13 @@ #!/usr/bin/env python +from compare_locales.parser import getParser from configparser import ConfigParser +from moz.l10n.paths import L10nConfigPaths, get_android_locale import argparse import codecs import json import logging import os -import sys logging.basicConfig() # Get absolute path of ../../config from the current script location (not the @@ -31,17 +32,16 @@ config_parser.read(config_file) storage_path = os.path.join(config_parser.get("config", "root"), "TMX") -try: - from compare_locales import paths - from compare_locales.parser import getParser -except ImportError as e: - print("FATAL: make sure that dependencies are installed") - print(e) - sys.exit(1) - class StringExtraction: - def __init__(self, toml_path, storage_path, reference_locale, repository_name): + def __init__( + self, + toml_path, + storage_path, + reference_locale, + repository_name, + android_project, + ): """Initialize object.""" # Set defaults @@ -54,6 +54,7 @@ def __init__(self, toml_path, storage_path, reference_locale, repository_name): self.storage_path = storage_path self.reference_locale = reference_locale self.repository_name = repository_name + self.android_project = android_project def setStorageAppendMode(self, prefix): """Set storage mode and prefix.""" @@ -82,12 +83,60 @@ def readExistingJSON(locale): def readFiles(locale): """Read files for locale""" - if locale == self.reference_locale: - files = paths.ProjectFiles(None, [project_config]) + if locale != self.reference_locale: + if self.android_project: + file_list = [ + ( + os.path.abspath( + tgt_path.format( + android_locale=get_android_locale(locale) + ) + ), + os.path.abspath(ref_path.format(android_locale=None)), + ) + for ( + ref_path, + tgt_path, + ), locales in project_config_paths.all().items() + if locale in locales + ] + else: + file_list = [ + ( + os.path.abspath(tgt_path.format(locale=locale)), + os.path.abspath(ref_path.format(locale=None)), + ) + for ( + ref_path, + tgt_path, + ), locales in project_config_paths.all().items() + if locale in locales + ] else: - files = paths.ProjectFiles(locale, [project_config]) - - for l10n_file, reference_file, _, _ in files: + if self.android_project: + file_list = [ + ( + os.path.abspath(ref_path.format(android_locale=None)), + os.path.abspath(ref_path.format(android_locale=None)), + ) + for ( + ref_path, + tgt_path, + ), locales in project_config_paths.all().items() + ] + else: + file_list = [ + ( + os.path.abspath(ref_path.format(locale=None)), + os.path.abspath(ref_path.format(locale=None)), + ) + for ( + ref_path, + tgt_path, + ), locales in project_config_paths.all().items() + ] + + for l10n_file, reference_file in file_list: if not os.path.exists(l10n_file): # File not available in localization continue @@ -115,8 +164,12 @@ def readFiles(locale): ) basedir = os.path.dirname(self.toml_path) - project_config = paths.TOMLParser().parse(self.toml_path, env={"l10n_base": ""}) - basedir = os.path.join(basedir, project_config.root) + if self.android_project: + project_config_paths = L10nConfigPaths( + self.toml_path, locale_map={"android_locale": get_android_locale} + ) + else: + project_config_paths = L10nConfigPaths(self.toml_path) # Read strings for reference locale self.translations[self.reference_locale] = ( @@ -124,7 +177,9 @@ def readFiles(locale): ) readFiles(self.reference_locale) - for locale in project_config.all_locales: + locales = list(project_config_paths.all_locales) + locales.sort() + for locale in locales: # If storage mode is append, read existing translations (if available) self.translations[locale] = ( readExistingJSON(locale) if self.storage_append else {} @@ -214,6 +269,13 @@ def main(): action="store_true", help="If set to 'append', translations will be added to an existing cache file", ) + parser.add_argument( + "--android", + dest="android_project", + action="store_true", + help="If passed, the script will parse the config file using Android locale codes", + default=False, + ) parser.add_argument( "--prefix", dest="storage_prefix", @@ -237,6 +299,7 @@ def main(): storage_path, args.reference_code, args.repository_name, + args.android_project, ) if args.append_mode: extracted_strings.setStorageAppendMode(args.storage_prefix) From 6b080391377caef701f7f930d329b96f91e40c30 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Mon, 3 Mar 2025 18:31:46 +0100 Subject: [PATCH 2/7] Add tests --- tests/test_string_projectconfig.py | 18 ++++++++++++++++-- tests/testfiles/toml/de/file.ftl | 1 + tests/testfiles/toml/de/file2.ftl | 1 + tests/testfiles/toml/en/file.ftl | 1 + tests/testfiles/toml/en/file2.ftl | 1 + tests/testfiles/toml/fr/file2.ftl | 0 tests/testfiles/toml/it/file.ftl | 1 + tests/testfiles/toml/l10n.toml | 11 +++++++++++ 8 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 tests/testfiles/toml/de/file.ftl create mode 100644 tests/testfiles/toml/de/file2.ftl create mode 100644 tests/testfiles/toml/en/file.ftl create mode 100644 tests/testfiles/toml/en/file2.ftl create mode 100644 tests/testfiles/toml/fr/file2.ftl create mode 100644 tests/testfiles/toml/it/file.ftl create mode 100644 tests/testfiles/toml/l10n.toml diff --git a/tests/test_string_projectconfig.py b/tests/test_string_projectconfig.py index 3a8d502..984d10c 100644 --- a/tests/test_string_projectconfig.py +++ b/tests/test_string_projectconfig.py @@ -11,10 +11,10 @@ def setUp(self): self.testfiles_path = os.path.join(os.path.dirname(__file__), "testfiles") self.storage_path = os.path.join(self.testfiles_path, "output_android") - def testGetAndroidStringsGerman(self): + def testGetAndroidStrings(self): toml_path = os.path.join(self.testfiles_path, "android", "l10n.toml") extraction = tmx_products.tmx_projectconfig.StringExtraction( - toml_path, self.storage_path, "en-US", "test" + toml_path, self.storage_path, "en-US", "test", True ) extraction.extractStrings() @@ -24,6 +24,20 @@ def testGetAndroidStringsGerman(self): self.assertEqual(len(strings_locale["en-US"]), 17) self.assertEqual(len(strings_locale["es-ES"]), 5) + def testGetProductStrings(self): + toml_path = os.path.join(self.testfiles_path, "toml", "l10n.toml") + extraction = tmx_products.tmx_projectconfig.StringExtraction( + toml_path, self.storage_path, "en", "test", False + ) + extraction.extractStrings() + + strings_locale = extraction.translations + self.assertEqual(len(strings_locale), 4) + self.assertEqual(len(strings_locale["it"]), 1) + self.assertEqual(len(strings_locale["en"]), 2) + self.assertEqual(len(strings_locale["de"]), 2) + self.assertEqual(len(strings_locale["fr"]), 0) + if __name__ == "__main__": unittest.main() diff --git a/tests/testfiles/toml/de/file.ftl b/tests/testfiles/toml/de/file.ftl new file mode 100644 index 0000000..994dcd0 --- /dev/null +++ b/tests/testfiles/toml/de/file.ftl @@ -0,0 +1 @@ +foo = Test diff --git a/tests/testfiles/toml/de/file2.ftl b/tests/testfiles/toml/de/file2.ftl new file mode 100644 index 0000000..170068e --- /dev/null +++ b/tests/testfiles/toml/de/file2.ftl @@ -0,0 +1 @@ +foos = Test 2 diff --git a/tests/testfiles/toml/en/file.ftl b/tests/testfiles/toml/en/file.ftl new file mode 100644 index 0000000..994dcd0 --- /dev/null +++ b/tests/testfiles/toml/en/file.ftl @@ -0,0 +1 @@ +foo = Test diff --git a/tests/testfiles/toml/en/file2.ftl b/tests/testfiles/toml/en/file2.ftl new file mode 100644 index 0000000..170068e --- /dev/null +++ b/tests/testfiles/toml/en/file2.ftl @@ -0,0 +1 @@ +foos = Test 2 diff --git a/tests/testfiles/toml/fr/file2.ftl b/tests/testfiles/toml/fr/file2.ftl new file mode 100644 index 0000000..e69de29 diff --git a/tests/testfiles/toml/it/file.ftl b/tests/testfiles/toml/it/file.ftl new file mode 100644 index 0000000..994dcd0 --- /dev/null +++ b/tests/testfiles/toml/it/file.ftl @@ -0,0 +1 @@ +foo = Test diff --git a/tests/testfiles/toml/l10n.toml b/tests/testfiles/toml/l10n.toml new file mode 100644 index 0000000..620f84b --- /dev/null +++ b/tests/testfiles/toml/l10n.toml @@ -0,0 +1,11 @@ +basepath = "." + +locales = [ + "de", + "fr", + "it" +] + +[[paths]] + reference = "en/*.ftl" + l10n = "{locale}/*.ftl" From d5233cca7cfbc0ff6bbae7bbb6376064dc2d8536 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Mon, 3 Mar 2025 19:11:39 +0100 Subject: [PATCH 3/7] Simplify code for reference locale --- tmx_products/tmx_projectconfig.py | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/tmx_products/tmx_projectconfig.py b/tmx_products/tmx_projectconfig.py index 2e47e9c..ea5cfa5 100755 --- a/tmx_products/tmx_projectconfig.py +++ b/tmx_products/tmx_projectconfig.py @@ -113,28 +113,10 @@ def readFiles(locale): if locale in locales ] else: - if self.android_project: - file_list = [ - ( - os.path.abspath(ref_path.format(android_locale=None)), - os.path.abspath(ref_path.format(android_locale=None)), - ) - for ( - ref_path, - tgt_path, - ), locales in project_config_paths.all().items() - ] - else: - file_list = [ - ( - os.path.abspath(ref_path.format(locale=None)), - os.path.abspath(ref_path.format(locale=None)), - ) - for ( - ref_path, - tgt_path, - ), locales in project_config_paths.all().items() - ] + file_list = [ + (os.path.abspath(ref_path), os.path.abspath(ref_path)) + for ref_path in project_config_paths.ref_paths + ] for l10n_file, reference_file in file_list: if not os.path.exists(l10n_file): From 784d9192a93408cd80fc084b0b11816db6b0fa59 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Wed, 5 Mar 2025 12:45:25 +0100 Subject: [PATCH 4/7] Update requirements --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 20911ee..b46a7d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -compare-locales==9.0.* -moz-l10n==0.6.0 +compare-locales~=9.0.0 +moz-l10n~=0.6.1 From 11b0b6efeb8e9a7f99707c1358116d324b2107b1 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Wed, 5 Mar 2025 12:58:30 +0100 Subject: [PATCH 5/7] Clean up code --- tmx_products/tmx_projectconfig.py | 46 ++++++++++++------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/tmx_products/tmx_projectconfig.py b/tmx_products/tmx_projectconfig.py index ea5cfa5..87e2ae8 100755 --- a/tmx_products/tmx_projectconfig.py +++ b/tmx_products/tmx_projectconfig.py @@ -84,41 +84,29 @@ def readFiles(locale): """Read files for locale""" if locale != self.reference_locale: - if self.android_project: - file_list = [ - ( - os.path.abspath( - tgt_path.format( - android_locale=get_android_locale(locale) - ) - ), - os.path.abspath(ref_path.format(android_locale=None)), - ) - for ( - ref_path, - tgt_path, - ), locales in project_config_paths.all().items() - if locale in locales - ] - else: - file_list = [ - ( - os.path.abspath(tgt_path.format(locale=locale)), - os.path.abspath(ref_path.format(locale=None)), + locale_files = [ + ( + os.path.abspath(ref_path), + os.path.abspath(tgt_path), + ) + for ( + ref_path, + raw_tgt_path, + ), locales in project_config_paths.all().items() + if locale in locales + and os.path.exists( + tgt_path := project_config_paths.format_target_path( + raw_tgt_path, locale ) - for ( - ref_path, - tgt_path, - ), locales in project_config_paths.all().items() - if locale in locales - ] + ) + ] else: - file_list = [ + locale_files = [ (os.path.abspath(ref_path), os.path.abspath(ref_path)) for ref_path in project_config_paths.ref_paths ] - for l10n_file, reference_file in file_list: + for reference_file, l10n_file in locale_files: if not os.path.exists(l10n_file): # File not available in localization continue From 07ee8b0c6ef94bd43b146ab90b5ee07ce1e70fe5 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Wed, 5 Mar 2025 12:58:49 +0100 Subject: [PATCH 6/7] Remove flake8 --- .flake8 | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index f7c0865..0000000 --- a/.flake8 +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -exclude = tests -ignore = E203, E501 From 50c967eec04db22dc37bb6f372dc49a4bc493385 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Wed, 5 Mar 2025 13:06:54 +0100 Subject: [PATCH 7/7] Use uv/ruff in workflows, test 3.12 --- .github/workflows/tests.yaml | 48 +++++++++++++++++++++--------------- requirements.txt | 1 + 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 56d654c..20873fd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -16,42 +16,50 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v4 - - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' - - - name: Install packages - run: | - pip install flake8 black - - - name: Linter (flake8) - run: flake8 . - - - name: Linter (black) - run: black --check . + python-version: '3.12' + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: | + **/requirements.txt + version: "0.6.4" + - name: Install Dependencies + run: uv pip install -r requirements.txt + env: + UV_SYSTEM_PYTHON: 1 + - name: ruff lint + run: ruff check + - name: ruff format + run: ruff format --check tests: runs-on: ubuntu-latest strategy: matrix: python-versions: - - '3.9' - - '3.10' - '3.11' + - '3.12' steps: - name: Check out repository uses: actions/checkout@v4 - - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-versions }} - - - name: Install packages - run: | - pip install -r requirements.txt - + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: | + **/requirements.txt + version: "0.6.4" + - name: Install Dependencies + run: uv pip install -r requirements.txt + env: + UV_SYSTEM_PYTHON: 1 - name: Run tests run: python -m unittest discover diff --git a/requirements.txt b/requirements.txt index b46a7d6..3838258 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ compare-locales~=9.0.0 moz-l10n~=0.6.1 +ruff~=0.9.9