Use moz.l10n for parsing

.github/workflows/tests.yaml

@@ -1,11 +1,8 @@
 name: Run Tests
 on:
-  # Trigger on push to default branch
   push:
     branches:
       - main
-  # Trigger the workflow on pull request events
-  # but only for the master branch
   pull_request:
     branches:
       - main
@@ -15,20 +12,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
-          python-version: '3.12'
+          python-version: '3.11'
       - name: Set up uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v6
         with:
           enable-cache: true
           cache-dependency-glob: |
-            **/requirements.txt
-          version: "0.6.4"
+            requirements.*
+          version: "0.8.22"
       - name: Install Dependencies
-        run: uv pip install -r requirements.txt
+        run: uv pip install -r requirements.txt -r requirements-dev.txt
         env:
           UV_SYSTEM_PYTHON: 1
       - name: ruff lint
@@ -43,22 +40,23 @@ jobs:
         python-versions:
           - '3.11'
           - '3.12'
+          - '3.12'
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
-          python-version: ${{ matrix.python-versions }}
+          python-version: '3.11'
       - name: Set up uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v6
         with:
           enable-cache: true
           cache-dependency-glob: |
-            **/requirements.txt
-          version: "0.6.4"
+            requirements.*
+          version: "0.8.22"
       - name: Install Dependencies
-        run: uv pip install -r requirements.txt
+        run: uv pip install -r requirements.txt -r requirements-dev.txt
         env:
           UV_SYSTEM_PYTHON: 1
       - name: Run tests

pyrightconfig.json

@@ -0,0 +1,4 @@
+{
+  "include": ["tmx_products", "tests"],
+  "extraPaths": ["tmx_products"]
+}

requirements-dev.txt

@@ -0,0 +1 @@
+ruff~=0.13.1

requirements.txt

@@ -1,3 +1 @@
-compare-locales~=9.0.0
-moz-l10n~=0.6.1
-ruff~=0.9.9
+moz-l10n[xml]~=0.9.1

tests/__init__.py

@@ -0,0 +1,6 @@
+import sys
+from pathlib import Path
+
+# Add .../tmx_folder to sys.path so tests can do "import functions"
+root = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(root / "tmx_products"))

tests/test_string_projectconfig.py

@@ -3,7 +3,7 @@
 import os
 import unittest
 
-import tmx_products.tmx_projectconfig
+import tmx_projectconfig as tmx_proj
 
 
 class TestStringExtraction(unittest.TestCase):
@@ -13,7 +13,7 @@ def setUp(self):
 
     def testGetAndroidStrings(self):
         toml_path = os.path.join(self.testfiles_path, "android", "l10n.toml")
-        extraction = tmx_products.tmx_projectconfig.StringExtraction(
+        extraction = tmx_proj.StringExtraction(
             toml_path, self.storage_path, "en-US", "test", True
         )
         extraction.extractStrings()
@@ -40,7 +40,7 @@ def testGetAndroidStrings(self):
 
     def testGetProductStrings(self):
         toml_path = os.path.join(self.testfiles_path, "toml", "l10n.toml")
-        extraction = tmx_products.tmx_projectconfig.StringExtraction(
+        extraction = tmx_proj.StringExtraction(
             toml_path, self.storage_path, "en", "test", False
         )
         extraction.extractStrings()

tests/test_string_extraction.py → tests/test_string_repository.py

@@ -4,7 +4,7 @@
 import os
 import unittest
 
-import tmx_products.tmx_products
+import tmx_repository as tmx_prod
 
 
 class TestStringExtraction(unittest.TestCase):
@@ -14,7 +14,7 @@ def setUp(self):
 
     def testGetProductStringsChinese(self):
         repo_path = os.path.join(self.testfiles_path, "product", "zh-CN")
-        extraction = tmx_products.tmx_products.StringExtraction(
+        extraction = tmx_prod.StringExtraction(
             self.storage_path, "zh-CN", "en-US", "test"
         )
         extraction.setRepositoryPath(repo_path)
@@ -98,9 +98,7 @@ def testGetProductStringsChinese(self):
 
     def testGetProductStringsItalian(self):
         repo_path = os.path.join(self.testfiles_path, "product", "it")
-        extraction = tmx_products.tmx_products.StringExtraction(
-            self.storage_path, "it", "en-US", "test"
-        )
+        extraction = tmx_prod.StringExtraction(self.storage_path, "it", "en-US", "test")
         extraction.setRepositoryPath(repo_path)
         extraction.extractStrings()
 
@@ -125,19 +123,17 @@ def testGetProductStringsItalian(self):
         )
 
         self.assertEqual(
-            strings_locale["browser/chrome/updater/updater.ini:TitleText"],
+            strings_locale["browser/chrome/updater/updater.ini:Strings.TitleText"],
             "Aggiornamento %MOZ_APP_DISPLAYNAME%",
         )
         self.assertEqual(
-            strings_locale["browser/chrome/updater/updater.ini:InfoText"],
+            strings_locale["browser/chrome/updater/updater.ini:Strings.InfoText"],
             "%MOZ_APP_DISPLAYNAME% sta installando gli aggiornamenti e si avvierà fra qualche istante…",
         )
 
     def testGetProductStringsBulgarian(self):
         repo_path = os.path.join(self.testfiles_path, "product", "bg")
-        extraction = tmx_products.tmx_products.StringExtraction(
-            self.storage_path, "bg", "en-US", "test"
-        )
+        extraction = tmx_prod.StringExtraction(self.storage_path, "bg", "en-US", "test")
         extraction.setRepositoryPath(repo_path)
         extraction.extractStrings()
 
@@ -149,9 +145,7 @@ def testGetProductStringsBulgarian(self):
         )
 
     def testEscape(self):
-        extraction = tmx_products.tmx_products.StringExtraction(
-            self.storage_path, "", "", ""
-        )
+        extraction = tmx_prod.StringExtraction(self.storage_path, "", "", "")
         extraction.translations = {
             "This is a simple test.": "This is a simple test.",
             "您的電腦中已儲存下列的 Cookie:": "您的電腦中已儲存下列的 Cookie:",
@@ -167,9 +161,7 @@ def testEscape(self):
             self.assertEqual(extraction.escape(string), result)
 
     def testRelativePath(self):
-        extraction = tmx_products.tmx_products.StringExtraction(
-            self.storage_path, "", "", ""
-        )
+        extraction = tmx_prod.StringExtraction(self.storage_path, "", "", "")
 
         extraction.setRepositoryPath("/home/test")
         paths = {
@@ -203,17 +195,15 @@ def testRelativePath(self):
 
     def testOutput(self):
         repo_path = os.path.join(self.testfiles_path, "tmx", "en-US")
-        extraction = tmx_products.tmx_products.StringExtraction(
+        extraction = tmx_prod.StringExtraction(
             self.storage_path, "en-US", "en-US", "test"
         )
         extraction.setRepositoryPath(repo_path)
         extraction.extractStrings()
         extraction.storeTranslations("")
 
         repo_path = os.path.join(self.testfiles_path, "tmx", "it")
-        extraction = tmx_products.tmx_products.StringExtraction(
-            self.storage_path, "it", "en-US", "test"
-        )
+        extraction = tmx_prod.StringExtraction(self.storage_path, "it", "en-US", "test")
         extraction.setRepositoryPath(repo_path)
         extraction.extractStrings()
         extraction.storeTranslations("")
@@ -252,7 +242,7 @@ def testOutput(self):
 
     def testOutputAppend(self):
         repo_path = os.path.join(self.testfiles_path, "tmx", "en-US")
-        extraction = tmx_products.tmx_products.StringExtraction(
+        extraction = tmx_prod.StringExtraction(
             self.storage_path, "en-US", "en-US", "appendtest"
         )
         extraction.setRepositoryPath(repo_path)
@@ -261,7 +251,7 @@ def testOutputAppend(self):
 
         # Do a new extraction, but append to existing translations
         repo_path = os.path.join(self.testfiles_path, "tmx", "en-US", "mail")
-        extraction = tmx_products.tmx_products.StringExtraction(
+        extraction = tmx_prod.StringExtraction(
             self.storage_path, "en-US", "en-US", "appendtest"
         )
         extraction.setRepositoryPath(repo_path)
@@ -302,20 +292,6 @@ def testOutputAppend(self):
         self.assertTrue(cmp_result_php)
         self.assertTrue(cmp_result_json)
 
-    def testBrokenEnconding(self):
-        repo_path = os.path.join(self.testfiles_path, "tmx", "oc")
-        extraction = tmx_products.tmx_products.StringExtraction(
-            self.storage_path, "oc", "en-US", "test"
-        )
-        extraction.setRepositoryPath(repo_path)
-        extraction.extractStrings()
-
-        self.assertEqual(
-            extraction.translations["test/test.dtd:test1"], "Test with one \ slash"
-        )
-        self.assertFalse("test/test.dtd:test_missing" in extraction.translations)
-        self.assertFalse("test/test.dtd:test_empty" in extraction.translations)
-
 
 if __name__ == "__main__":
     unittest.main()

tests/testfiles/product/zh-CN/browser/chrome/browser/main.ftl

@@ -18,7 +18,8 @@ timeDiffHoursAgo = { $number ->
 # Basic string
 sample = Just a test
 
-some junk text, should be ignored
+# TODO: restore this?
+# some junk text, should be ignored
 
 # Parameterized term
 onboarding-fxa-text = Зарегистрируйте { -fxaccount-brand-name(case: "nominative") } test.

tests/testfiles/tmx/en-US/test/test.dtd

@@ -1,3 +1,7 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
 <!ENTITY test1 "Test with one \ slash">
 <!ENTITY test2 "Test with single 'quotes'">
 <!ENTITY test2b "Test with escaped single 'quotes'">

tests/testfiles/tmx/it/test/test.dtd

@@ -1,3 +1,7 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
 <!ENTITY test1 "Prova con uno \ slash">
 <!ENTITY test2 "Prova con 'virgolette'">
 <!ENTITY test2b "Prova con \'virgolette\' escaped">

tmx_products/functions.py

@@ -0,0 +1,93 @@
+from configparser import ConfigParser
+import argparse
+import os
+
+
+def get_config() -> str:
+    # Get absolute path of ../../config from the current script location (not the
+    # current folder)
+    config_folder = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "config")
+    )
+    # Read Transvision's configuration file from ../../config/config.ini
+    # If not available use a default storage folder to store data
+    config_file = os.path.join(config_folder, "config.ini")
+    if not os.path.isfile(config_file):
+        print(
+            "Configuration file /app/config/config.ini is missing. "
+            "Default settings will be used."
+        )
+        root_folder = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), os.pardir)
+        )
+        storage_path = os.path.join(root_folder, "TMX")
+        os.makedirs(storage_path, exist_ok=True)
+    else:
+        config_parser = ConfigParser()
+        config_parser.read(config_file)
+        storage_path = os.path.join(config_parser.get("config", "root"), "TMX")
+
+    return storage_path
+
+
+def get_cli_parameters(config: bool = False) -> argparse.Namespace:
+    # Read command line input parameters
+    parser = argparse.ArgumentParser()
+
+    if config:
+        parser.add_argument("toml_path", help="Path to root l10n.toml file")
+        parser.add_argument(
+            "--android",
+            dest="android_project",
+            action="store_true",
+            help="If passed, the script will parse the config file using Android locale codes",
+            default=False,
+        )
+    else:
+        parser.add_argument(
+            "--path",
+            dest="repo_path",
+            help="Path to locale files",
+            required=True,
+        )
+        parser.add_argument(
+            "--locale",
+            dest="locale_code",
+            help="Locale code",
+            required=True,
+        )
+
+    # Common parameters
+    parser.add_argument(
+        "--ref",
+        dest="reference_code",
+        help="Reference locale code",
+        required=True,
+    )
+    parser.add_argument(
+        "--repo", dest="repository_name", help="Repository name", required=True
+    )
+    parser.add_argument(
+        "--append",
+        dest="append_mode",
+        action="store_true",
+        help="If set to 'append', translations will be added to an existing cache file",
+    )
+    parser.add_argument(
+        "--prefix",
+        dest="storage_prefix",
+        nargs="?",
+        help="This prefix will be prependended to the identified "
+        "path in string IDs (e.g. extensions/irc for Chatzilla)",
+        default="",
+    )
+    parser.add_argument(
+        "--output",
+        nargs="?",
+        type=str,
+        choices=["json", "php"],
+        help="Store only one type of output.",
+        default="",
+    )
+
+    return parser.parse_args()