Use moz.l10n to parse TOML config files

.github/workflows/tests.yaml

@@ -16,42 +16,50 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
-
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
-
-      - name: Install packages
-        run: |
-          pip install flake8 black
-
-      - name: Linter (flake8)
-        run: flake8 .
-
-      - name: Linter (black)
-        run: black --check .
+          python-version: '3.12'
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-dependency-glob: |
+            **/requirements.txt
+          version: "0.6.4"
+      - name: Install Dependencies
+        run: uv pip install -r requirements.txt
+        env:
+          UV_SYSTEM_PYTHON: 1
+      - name: ruff lint
+        run: ruff check
+      - name: ruff format
+        run: ruff format --check
 
   tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-versions:
-          - '3.9'
-          - '3.10'
           - '3.11'
+          - '3.12'
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
-
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-versions }}
-
-      - name: Install packages
-        run: |
-          pip install -r requirements.txt
-
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-dependency-glob: |
+            **/requirements.txt
+          version: "0.6.4"
+      - name: Install Dependencies
+        run: uv pip install -r requirements.txt
+        env:
+          UV_SYSTEM_PYTHON: 1
       - name: Run tests
         run: python -m unittest discover

requirements.txt

@@ -1 +1,3 @@
-compare-locales==9.0.*
+compare-locales~=9.0.0
+moz-l10n~=0.6.1
+ruff~=0.9.9

tests/test_string_projectconfig.py

@@ -11,10 +11,10 @@ def setUp(self):
         self.testfiles_path = os.path.join(os.path.dirname(__file__), "testfiles")
         self.storage_path = os.path.join(self.testfiles_path, "output_android")
 
-    def testGetAndroidStringsGerman(self):
+    def testGetAndroidStrings(self):
         toml_path = os.path.join(self.testfiles_path, "android", "l10n.toml")
         extraction = tmx_products.tmx_projectconfig.StringExtraction(
-            toml_path, self.storage_path, "en-US", "test"
+            toml_path, self.storage_path, "en-US", "test", True
         )
         extraction.extractStrings()
 
@@ -24,6 +24,20 @@ def testGetAndroidStringsGerman(self):
         self.assertEqual(len(strings_locale["en-US"]), 17)
         self.assertEqual(len(strings_locale["es-ES"]), 5)
 
+    def testGetProductStrings(self):
+        toml_path = os.path.join(self.testfiles_path, "toml", "l10n.toml")
+        extraction = tmx_products.tmx_projectconfig.StringExtraction(
+            toml_path, self.storage_path, "en", "test", False
+        )
+        extraction.extractStrings()
+
+        strings_locale = extraction.translations
+        self.assertEqual(len(strings_locale), 4)
+        self.assertEqual(len(strings_locale["it"]), 1)
+        self.assertEqual(len(strings_locale["en"]), 2)
+        self.assertEqual(len(strings_locale["de"]), 2)
+        self.assertEqual(len(strings_locale["fr"]), 0)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/testfiles/toml/de/file.ftl

@@ -0,0 +1 @@
+foo = Test

tests/testfiles/toml/de/file2.ftl

@@ -0,0 +1 @@
+foos = Test 2

tests/testfiles/toml/en/file.ftl

@@ -0,0 +1 @@
+foo = Test

tests/testfiles/toml/en/file2.ftl

@@ -0,0 +1 @@
+foos = Test 2

tests/testfiles/toml/it/file.ftl

@@ -0,0 +1 @@
+foo = Test

tests/testfiles/toml/l10n.toml

@@ -0,0 +1,11 @@
+basepath = "."
+
+locales = [
+    "de",
+    "fr",
+    "it"
+]
+
+[[paths]]
+    reference = "en/*.ftl"
+    l10n = "{locale}/*.ftl"

tmx_products/tmx_projectconfig.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python
 
+from compare_locales.parser import getParser
 from configparser import ConfigParser
+from moz.l10n.paths import L10nConfigPaths, get_android_locale
 import argparse
 import codecs
 import json
 import logging
 import os
-import sys
 
 logging.basicConfig()
 # Get absolute path of ../../config from the current script location (not the
@@ -31,17 +32,16 @@
     config_parser.read(config_file)
     storage_path = os.path.join(config_parser.get("config", "root"), "TMX")
 
-try:
-    from compare_locales import paths
-    from compare_locales.parser import getParser
-except ImportError as e:
-    print("FATAL: make sure that dependencies are installed")
-    print(e)
-    sys.exit(1)
-
 
 class StringExtraction:
-    def __init__(self, toml_path, storage_path, reference_locale, repository_name):
+    def __init__(
+        self,
+        toml_path,
+        storage_path,
+        reference_locale,
+        repository_name,
+        android_project,
+    ):
         """Initialize object."""
 
         # Set defaults
@@ -54,6 +54,7 @@ def __init__(self, toml_path, storage_path, reference_locale, repository_name):
         self.storage_path = storage_path
         self.reference_locale = reference_locale
         self.repository_name = repository_name
+        self.android_project = android_project
 
     def setStorageAppendMode(self, prefix):
         """Set storage mode and prefix."""
@@ -82,12 +83,30 @@ def readExistingJSON(locale):
         def readFiles(locale):
             """Read files for locale"""
 
-            if locale == self.reference_locale:
-                files = paths.ProjectFiles(None, [project_config])
+            if locale != self.reference_locale:
+                locale_files = [
+                    (
+                        os.path.abspath(ref_path),
+                        os.path.abspath(tgt_path),
+                    )
+                    for (
+                        ref_path,
+                        raw_tgt_path,
+                    ), locales in project_config_paths.all().items()
+                    if locale in locales
+                    and os.path.exists(
+                        tgt_path := project_config_paths.format_target_path(
+                            raw_tgt_path, locale
+                        )
+                    )
+                ]
             else:
-                files = paths.ProjectFiles(locale, [project_config])
+                locale_files = [
+                    (os.path.abspath(ref_path), os.path.abspath(ref_path))
+                    for ref_path in project_config_paths.ref_paths
+                ]
 
-            for l10n_file, reference_file, _, _ in files:
+            for reference_file, l10n_file in locale_files:
                 if not os.path.exists(l10n_file):
                     # File not available in localization
                     continue
@@ -115,16 +134,22 @@ def readFiles(locale):
                 )
 
         basedir = os.path.dirname(self.toml_path)
-        project_config = paths.TOMLParser().parse(self.toml_path, env={"l10n_base": ""})
-        basedir = os.path.join(basedir, project_config.root)
+        if self.android_project:
+            project_config_paths = L10nConfigPaths(
+                self.toml_path, locale_map={"android_locale": get_android_locale}
+            )
+        else:
+            project_config_paths = L10nConfigPaths(self.toml_path)
 
         # Read strings for reference locale
         self.translations[self.reference_locale] = (
             readExistingJSON(self.reference_locale) if self.storage_append else {}
         )
         readFiles(self.reference_locale)
 
-        for locale in project_config.all_locales:
+        locales = list(project_config_paths.all_locales)
+        locales.sort()
+        for locale in locales:
             # If storage mode is append, read existing translations (if available)
             self.translations[locale] = (
                 readExistingJSON(locale) if self.storage_append else {}
@@ -214,6 +239,13 @@ def main():
         action="store_true",
         help="If set to 'append', translations will be added to an existing cache file",
     )
+    parser.add_argument(
+        "--android",
+        dest="android_project",
+        action="store_true",
+        help="If passed, the script will parse the config file using Android locale codes",
+        default=False,
+    )
     parser.add_argument(
         "--prefix",
         dest="storage_prefix",
@@ -237,6 +269,7 @@ def main():
         storage_path,
         args.reference_code,
         args.repository_name,
+        args.android_project,
     )
     if args.append_mode:
         extracted_strings.setStorageAppendMode(args.storage_prefix)