diff --git a/.flake8 b/.flake8 deleted file mode 100644 index f7c0865..0000000 --- a/.flake8 +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -exclude = tests -ignore = E203, E501 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 56d654c..20873fd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -16,42 +16,50 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v4 - - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' - - - name: Install packages - run: | - pip install flake8 black - - - name: Linter (flake8) - run: flake8 . - - - name: Linter (black) - run: black --check . + python-version: '3.12' + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: | + **/requirements.txt + version: "0.6.4" + - name: Install Dependencies + run: uv pip install -r requirements.txt + env: + UV_SYSTEM_PYTHON: 1 + - name: ruff lint + run: ruff check + - name: ruff format + run: ruff format --check tests: runs-on: ubuntu-latest strategy: matrix: python-versions: - - '3.9' - - '3.10' - '3.11' + - '3.12' steps: - name: Check out repository uses: actions/checkout@v4 - - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-versions }} - - - name: Install packages - run: | - pip install -r requirements.txt - + - name: Set up uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: | + **/requirements.txt + version: "0.6.4" + - name: Install Dependencies + run: uv pip install -r requirements.txt + env: + UV_SYSTEM_PYTHON: 1 - name: Run tests run: python -m unittest discover diff --git a/requirements.txt b/requirements.txt index 5038f3b..3838258 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ -compare-locales==9.0.* +compare-locales~=9.0.0 +moz-l10n~=0.6.1 +ruff~=0.9.9 diff --git a/tests/test_string_projectconfig.py b/tests/test_string_projectconfig.py index 3a8d502..984d10c 100644 --- a/tests/test_string_projectconfig.py +++ b/tests/test_string_projectconfig.py @@ -11,10 +11,10 @@ def setUp(self): self.testfiles_path = os.path.join(os.path.dirname(__file__), "testfiles") self.storage_path = os.path.join(self.testfiles_path, "output_android") - def testGetAndroidStringsGerman(self): + def testGetAndroidStrings(self): toml_path = os.path.join(self.testfiles_path, "android", "l10n.toml") extraction = tmx_products.tmx_projectconfig.StringExtraction( - toml_path, self.storage_path, "en-US", "test" + toml_path, self.storage_path, "en-US", "test", True ) extraction.extractStrings() @@ -24,6 +24,20 @@ def testGetAndroidStringsGerman(self): self.assertEqual(len(strings_locale["en-US"]), 17) self.assertEqual(len(strings_locale["es-ES"]), 5) + def testGetProductStrings(self): + toml_path = os.path.join(self.testfiles_path, "toml", "l10n.toml") + extraction = tmx_products.tmx_projectconfig.StringExtraction( + toml_path, self.storage_path, "en", "test", False + ) + extraction.extractStrings() + + strings_locale = extraction.translations + self.assertEqual(len(strings_locale), 4) + self.assertEqual(len(strings_locale["it"]), 1) + self.assertEqual(len(strings_locale["en"]), 2) + self.assertEqual(len(strings_locale["de"]), 2) + self.assertEqual(len(strings_locale["fr"]), 0) + if __name__ == "__main__": unittest.main() diff --git a/tests/testfiles/toml/de/file.ftl b/tests/testfiles/toml/de/file.ftl new file mode 100644 index 0000000..994dcd0 --- /dev/null +++ b/tests/testfiles/toml/de/file.ftl @@ -0,0 +1 @@ +foo = Test diff --git a/tests/testfiles/toml/de/file2.ftl b/tests/testfiles/toml/de/file2.ftl new file mode 100644 index 0000000..170068e --- /dev/null +++ b/tests/testfiles/toml/de/file2.ftl @@ -0,0 +1 @@ +foos = Test 2 diff --git a/tests/testfiles/toml/en/file.ftl b/tests/testfiles/toml/en/file.ftl new file mode 100644 index 0000000..994dcd0 --- /dev/null +++ b/tests/testfiles/toml/en/file.ftl @@ -0,0 +1 @@ +foo = Test diff --git a/tests/testfiles/toml/en/file2.ftl b/tests/testfiles/toml/en/file2.ftl new file mode 100644 index 0000000..170068e --- /dev/null +++ b/tests/testfiles/toml/en/file2.ftl @@ -0,0 +1 @@ +foos = Test 2 diff --git a/tests/testfiles/toml/fr/file2.ftl b/tests/testfiles/toml/fr/file2.ftl new file mode 100644 index 0000000..e69de29 diff --git a/tests/testfiles/toml/it/file.ftl b/tests/testfiles/toml/it/file.ftl new file mode 100644 index 0000000..994dcd0 --- /dev/null +++ b/tests/testfiles/toml/it/file.ftl @@ -0,0 +1 @@ +foo = Test diff --git a/tests/testfiles/toml/l10n.toml b/tests/testfiles/toml/l10n.toml new file mode 100644 index 0000000..620f84b --- /dev/null +++ b/tests/testfiles/toml/l10n.toml @@ -0,0 +1,11 @@ +basepath = "." + +locales = [ + "de", + "fr", + "it" +] + +[[paths]] + reference = "en/*.ftl" + l10n = "{locale}/*.ftl" diff --git a/tmx_products/tmx_projectconfig.py b/tmx_products/tmx_projectconfig.py index 996ad4b..87e2ae8 100755 --- a/tmx_products/tmx_projectconfig.py +++ b/tmx_products/tmx_projectconfig.py @@ -1,12 +1,13 @@ #!/usr/bin/env python +from compare_locales.parser import getParser from configparser import ConfigParser +from moz.l10n.paths import L10nConfigPaths, get_android_locale import argparse import codecs import json import logging import os -import sys logging.basicConfig() # Get absolute path of ../../config from the current script location (not the @@ -31,17 +32,16 @@ config_parser.read(config_file) storage_path = os.path.join(config_parser.get("config", "root"), "TMX") -try: - from compare_locales import paths - from compare_locales.parser import getParser -except ImportError as e: - print("FATAL: make sure that dependencies are installed") - print(e) - sys.exit(1) - class StringExtraction: - def __init__(self, toml_path, storage_path, reference_locale, repository_name): + def __init__( + self, + toml_path, + storage_path, + reference_locale, + repository_name, + android_project, + ): """Initialize object.""" # Set defaults @@ -54,6 +54,7 @@ def __init__(self, toml_path, storage_path, reference_locale, repository_name): self.storage_path = storage_path self.reference_locale = reference_locale self.repository_name = repository_name + self.android_project = android_project def setStorageAppendMode(self, prefix): """Set storage mode and prefix.""" @@ -82,12 +83,30 @@ def readExistingJSON(locale): def readFiles(locale): """Read files for locale""" - if locale == self.reference_locale: - files = paths.ProjectFiles(None, [project_config]) + if locale != self.reference_locale: + locale_files = [ + ( + os.path.abspath(ref_path), + os.path.abspath(tgt_path), + ) + for ( + ref_path, + raw_tgt_path, + ), locales in project_config_paths.all().items() + if locale in locales + and os.path.exists( + tgt_path := project_config_paths.format_target_path( + raw_tgt_path, locale + ) + ) + ] else: - files = paths.ProjectFiles(locale, [project_config]) + locale_files = [ + (os.path.abspath(ref_path), os.path.abspath(ref_path)) + for ref_path in project_config_paths.ref_paths + ] - for l10n_file, reference_file, _, _ in files: + for reference_file, l10n_file in locale_files: if not os.path.exists(l10n_file): # File not available in localization continue @@ -115,8 +134,12 @@ def readFiles(locale): ) basedir = os.path.dirname(self.toml_path) - project_config = paths.TOMLParser().parse(self.toml_path, env={"l10n_base": ""}) - basedir = os.path.join(basedir, project_config.root) + if self.android_project: + project_config_paths = L10nConfigPaths( + self.toml_path, locale_map={"android_locale": get_android_locale} + ) + else: + project_config_paths = L10nConfigPaths(self.toml_path) # Read strings for reference locale self.translations[self.reference_locale] = ( @@ -124,7 +147,9 @@ def readFiles(locale): ) readFiles(self.reference_locale) - for locale in project_config.all_locales: + locales = list(project_config_paths.all_locales) + locales.sort() + for locale in locales: # If storage mode is append, read existing translations (if available) self.translations[locale] = ( readExistingJSON(locale) if self.storage_append else {} @@ -214,6 +239,13 @@ def main(): action="store_true", help="If set to 'append', translations will be added to an existing cache file", ) + parser.add_argument( + "--android", + dest="android_project", + action="store_true", + help="If passed, the script will parse the config file using Android locale codes", + default=False, + ) parser.add_argument( "--prefix", dest="storage_prefix", @@ -237,6 +269,7 @@ def main(): storage_path, args.reference_code, args.repository_name, + args.android_project, ) if args.append_mode: extracted_strings.setStorageAppendMode(args.storage_prefix)