Change behavior for .ini files, refactor code

tests/test_string_repository.py

@@ -103,7 +103,7 @@ def testGetProductStringsItalian(self):
         extraction.extractStrings()
 
         strings_locale = extraction.translations
-        self.assertEqual(len(strings_locale), 5)
+        self.assertEqual(len(strings_locale), 13)
 
         self.assertEqual(
             strings_locale["browser/chrome/browser/whitespaces.dtd:whitespaces"],
@@ -122,12 +122,44 @@ def testGetProductStringsItalian(self):
             "Test 3  ",
         )
 
+        # .ini files
         self.assertEqual(
-            strings_locale["browser/chrome/updater/updater.ini:Strings.TitleText"],
+            strings_locale["browser/chrome/browser/crash.ini:CrashReporterTitle"],
+            "Crash Reporter",
+        )
+        self.assertEqual(
+            strings_locale["browser/chrome/browser/crash.ini:isRTL"],
+            "",
+        )
+
+        # .ftl files
+        # A string with attributes but no value should store only the attributes
+        self.assertTrue(
+            "browser/chrome/browser/file.ftl:attr-but-no-value" not in strings_locale
+        )
+        self.assertEqual(
+            strings_locale["browser/chrome/browser/file.ftl:attr-but-no-value.label"],
+            "Label with no value",
+        )
+        self.assertEqual(
+            strings_locale["browser/chrome/browser/file.ftl:attr-with-value"],
+            "Value",
+        )
+        self.assertEqual(
+            strings_locale["browser/chrome/browser/file.ftl:attr-with-value.label"],
+            "Label with value",
+        )
+        self.assertEqual(
+            strings_locale["browser/chrome/browser/file.ftl:empty-string"],
+            '{ "" }',
+        )
+
+        self.assertEqual(
+            strings_locale["browser/chrome/updater/updater.ini:TitleText"],
             "Aggiornamento %MOZ_APP_DISPLAYNAME%",
         )
         self.assertEqual(
-            strings_locale["browser/chrome/updater/updater.ini:Strings.InfoText"],
+            strings_locale["browser/chrome/updater/updater.ini:InfoText"],
             "%MOZ_APP_DISPLAYNAME% sta installando gli aggiornamenti e si avvierà fra qualche istante…",
         )
 

tests/testfiles/product/it/browser/chrome/browser/crash.ini

@@ -0,0 +1,14 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file is in the UTF-8 encoding
+[Strings]
+# LOCALIZATION NOTE (isRTL):
+# Leave this entry empty unless your language requires right-to-left layout,
+# for example like Arabic, Hebrew, Persian. If your language needs RTL, please
+# use the untranslated English word "yes" as value
+isRTL=
+CrashReporterTitle=Crash Reporter
+# LOCALIZATION NOTE (CrashReporterVendorTitle): %s is replaced with the vendor name.  (i.e. "Mozilla")
+CrashReporterVendorTitle=%s Crash Reporter

tests/testfiles/product/it/browser/chrome/browser/file.ftl

@@ -0,0 +1,8 @@
+foo = Test
+
+attr-but-no-value =
+    .label = Label with no value
+attr-with-value = Value
+    .label = Label with value
+
+empty-string = { "" }

tmx_products/functions.py

@@ -1,4 +1,7 @@
 from configparser import ConfigParser
+from moz.l10n.formats import Format
+from moz.l10n.message import serialize_message
+from moz.l10n.model import Entry, Message, Resource
 import argparse
 import os
 
@@ -91,3 +94,41 @@ def get_cli_parameters(config: bool = False) -> argparse.Namespace:
     )
 
     return parser.parse_args()
+
+
+def parse_file(
+    resource: Resource,
+    storage: dict[str, str],
+    filename: str,
+    id_format: str,
+) -> None:
+    def get_entry_value(value: Message) -> str:
+        entry_value = serialize_message(resource.format, value)
+        if resource.format == Format.android:
+            # In Android resources, unescape quotes
+            entry_value = entry_value.replace('\\"', '"').replace("\\'", "'")
+
+        return entry_value
+
+    try:
+        for section in resource.sections:
+            for entry in section.entries:
+                if isinstance(entry, Entry):
+                    if resource.format == Format.ini:
+                        entry_id = ".".join(entry.id)
+                    else:
+                        entry_id = ".".join(section.id + entry.id)
+                    string_id = f"{id_format}:{entry_id}"
+                    if entry.properties:
+                        # Store the value of an entry with attributes only
+                        # if the value is not empty.
+                        if not entry.value.is_empty():
+                            storage[string_id] = get_entry_value(entry.value)
+                        for attribute, attr_value in entry.properties.items():
+                            attr_id = f"{string_id}.{attribute}"
+                            storage[attr_id] = get_entry_value(attr_value)
+                    else:
+                        storage[string_id] = get_entry_value(entry.value)
+    except Exception as e:
+        print(f"Error parsing file: {filename}")
+        print(e)

tmx_products/tmx_projectconfig.py

@@ -1,9 +1,6 @@
 #!/usr/bin/env python
 
-from functions import get_cli_parameters, get_config
-from moz.l10n.formats import Format
-from moz.l10n.message import serialize_message
-from moz.l10n.model import Entry
+from functions import get_cli_parameters, get_config, parse_file
 from moz.l10n.paths import L10nConfigPaths, get_android_locale
 from moz.l10n.resource import parse_resource
 import codecs
@@ -58,14 +55,6 @@ def readExistingJSON(locale):
 
             return translations
 
-        def getEntryValue(resource, value):
-            entry_value = serialize_message(resource.format, value)
-            if resource.format == Format.android:
-                # In Android resources, unescape quotes
-                entry_value = entry_value.replace('\\"', '"').replace("\\'", "'")
-
-            return entry_value
-
         def readFiles(locale):
             """Read files for locale"""
 
@@ -116,34 +105,15 @@ def readFiles(locale):
                         resource = parse_resource(
                             l10n_file, android_literal_quotes=True
                         )
-                    for section in resource.sections:
-                        for entry in section.entries:
-                            if isinstance(entry, Entry):
-                                entry_id = ".".join(section.id + entry.id)
-                                string_id = (
-                                    f"{self.repository_name}/{key_path}:{entry_id}"
-                                )
-                                if entry.properties:
-                                    # Store the value of an entry with attributes only
-                                    # if the value is not empty.
-                                    if not entry.value.is_empty():
-                                        self.translations[locale][string_id] = (
-                                            getEntryValue(resource, entry.value)
-                                        )
-                                    for (
-                                        attribute,
-                                        attr_value,
-                                    ) in entry.properties.items():
-                                        attr_id = f"{string_id}.{attribute}"
-                                        self.translations[locale][attr_id] = (
-                                            getEntryValue(resource, attr_value)
-                                        )
-                                else:
-                                    self.translations[locale][string_id] = (
-                                        getEntryValue(resource, entry.value)
-                                    )
+
+                    parse_file(
+                        resource,
+                        self.translations[locale],
+                        l10n_file,
+                        f"{self.repository_name}/{key_path}",
+                    )
                 except Exception as e:
-                    print(f"Error parsing file: {reference_file}")
+                    print(f"Error parsing resource: {reference_file}")
                     print(e)
 
         basedir = os.path.dirname(self.toml_path)

tmx_products/tmx_repository.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 
-from functions import get_cli_parameters, get_config
+from functions import get_cli_parameters, get_config, parse_file
 from moz.l10n.resource import parse_resource
-from moz.l10n.message import serialize_message
-from moz.l10n.model import Entry
 import codecs
 import json
 import os
@@ -84,49 +82,30 @@ def extractStrings(self):
         # If storage mode is append, read existing translations (if available)
         # before overriding them
         if self.storage_append:
-            file_name = f"{self.storage_file}.json"
-            if os.path.isfile(file_name):
-                with open(file_name) as f:
+            filename = f"{self.storage_file}.json"
+            if os.path.isfile(filename):
+                with open(filename) as f:
                     self.translations = json.load(f)
                 f.close()
 
         # Create a list of files to analyze
         self.extractFileList()
 
-        for file_name in self.file_list:
-            resource = parse_resource(file_name)
+        for filename in self.file_list:
             try:
-                for section in resource.sections:
-                    for entry in section.entries:
-                        if isinstance(entry, Entry):
-                            entry_id = ".".join(section.id + entry.id)
-                            string_id = f"{self.getRelativePath(file_name)}:{entry_id}"
-                            if entry.properties:
-                                # Store the value of an entry with attributes only
-                                # if the value is not empty.
-                                if not entry.value.is_empty():
-                                    self.translations[string_id] = serialize_message(
-                                        resource.format, entry.value
-                                    )
-                                for attribute, attr_value in entry.properties.items():
-                                    attr_id = f"{string_id}.{attribute}"
-                                    self.translations[attr_id] = serialize_message(
-                                        resource.format, attr_value
-                                    )
-                            else:
-                                self.translations[string_id] = serialize_message(
-                                    resource.format, entry.value
-                                )
+                resource = parse_resource(filename)
+                rel_filename = self.getRelativePath(filename)
+                parse_file(resource, self.translations, filename, f"{rel_filename}")
             except Exception as e:
-                print(f"Error parsing file: {file_name}")
+                print(f"Error parsing resource: {filename}")
                 print(e)
 
         # Remove extra strings from locale
         if self.reference_locale != self.locale:
             # Read the JSON cache for reference locale if available
-            file_name = f"{self.reference_storage_file}.json"
-            if os.path.isfile(file_name):
-                with open(file_name) as f:
+            filename = f"{self.reference_storage_file}.json"
+            if os.path.isfile(filename):
+                with open(filename) as f:
                     reference_strings = json.load(f)
                 f.close()