From 234caf4eea180e0dc455e0a48bae992ea07d5930 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Fri, 26 Sep 2025 07:30:48 +0200 Subject: [PATCH 1/4] Update tests to document current behavior --- tests/test_string_repository.py | 36 ++++++++++++++++++- .../it/browser/chrome/browser/crash.ini | 14 ++++++++ .../it/browser/chrome/browser/file.ftl | 8 +++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 tests/testfiles/product/it/browser/chrome/browser/crash.ini create mode 100644 tests/testfiles/product/it/browser/chrome/browser/file.ftl diff --git a/tests/test_string_repository.py b/tests/test_string_repository.py index 95c44dd..5f2ddf1 100644 --- a/tests/test_string_repository.py +++ b/tests/test_string_repository.py @@ -103,7 +103,7 @@ def testGetProductStringsItalian(self): extraction.extractStrings() strings_locale = extraction.translations - self.assertEqual(len(strings_locale), 5) + self.assertEqual(len(strings_locale), 13) self.assertEqual( strings_locale["browser/chrome/browser/whitespaces.dtd:whitespaces"], @@ -122,6 +122,40 @@ def testGetProductStringsItalian(self): "Test 3 ", ) + # .ini files + self.assertEqual( + strings_locale[ + "browser/chrome/browser/crash.ini:Strings.CrashReporterTitle" + ], + "Crash Reporter", + ) + self.assertEqual( + strings_locale["browser/chrome/browser/crash.ini:Strings.isRTL"], + "", + ) + + # .ftl files + # A string with attributes but no value should store only the attributes + self.assertTrue( + "browser/chrome/browser/file.ftl:attr-but-no-value" not in strings_locale + ) + self.assertEqual( + strings_locale["browser/chrome/browser/file.ftl:attr-but-no-value.label"], + "Label with no value", + ) + self.assertEqual( + strings_locale["browser/chrome/browser/file.ftl:attr-with-value"], + "Value", + ) + self.assertEqual( + strings_locale["browser/chrome/browser/file.ftl:attr-with-value.label"], + "Label with value", + ) + self.assertEqual( + strings_locale["browser/chrome/browser/file.ftl:empty-string"], + '{ "" }', + ) + self.assertEqual( strings_locale["browser/chrome/updater/updater.ini:Strings.TitleText"], "Aggiornamento %MOZ_APP_DISPLAYNAME%", diff --git a/tests/testfiles/product/it/browser/chrome/browser/crash.ini b/tests/testfiles/product/it/browser/chrome/browser/crash.ini new file mode 100644 index 0000000..8151db7 --- /dev/null +++ b/tests/testfiles/product/it/browser/chrome/browser/crash.ini @@ -0,0 +1,14 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file is in the UTF-8 encoding +[Strings] +# LOCALIZATION NOTE (isRTL): +# Leave this entry empty unless your language requires right-to-left layout, +# for example like Arabic, Hebrew, Persian. If your language needs RTL, please +# use the untranslated English word "yes" as value +isRTL= +CrashReporterTitle=Crash Reporter +# LOCALIZATION NOTE (CrashReporterVendorTitle): %s is replaced with the vendor name. (i.e. "Mozilla") +CrashReporterVendorTitle=%s Crash Reporter diff --git a/tests/testfiles/product/it/browser/chrome/browser/file.ftl b/tests/testfiles/product/it/browser/chrome/browser/file.ftl new file mode 100644 index 0000000..de9fdcb --- /dev/null +++ b/tests/testfiles/product/it/browser/chrome/browser/file.ftl @@ -0,0 +1,8 @@ +foo = Test + +attr-but-no-value = + .label = Label with no value +attr-with-value = Value + .label = Label with value + +empty-string = { "" } From 78c277bc3fa4e0bd5fef0f33ee478dc2e0bcccf6 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Fri, 26 Sep 2025 07:37:58 +0200 Subject: [PATCH 2/4] Don't store section in ID for .ini strings Matching Pontoon behavior. --- tests/test_string_repository.py | 10 ++++------ tmx_products/tmx_projectconfig.py | 5 ++++- tmx_products/tmx_repository.py | 8 ++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/test_string_repository.py b/tests/test_string_repository.py index 5f2ddf1..7282407 100644 --- a/tests/test_string_repository.py +++ b/tests/test_string_repository.py @@ -124,13 +124,11 @@ def testGetProductStringsItalian(self): # .ini files self.assertEqual( - strings_locale[ - "browser/chrome/browser/crash.ini:Strings.CrashReporterTitle" - ], + strings_locale["browser/chrome/browser/crash.ini:CrashReporterTitle"], "Crash Reporter", ) self.assertEqual( - strings_locale["browser/chrome/browser/crash.ini:Strings.isRTL"], + strings_locale["browser/chrome/browser/crash.ini:isRTL"], "", ) @@ -157,11 +155,11 @@ def testGetProductStringsItalian(self): ) self.assertEqual( - strings_locale["browser/chrome/updater/updater.ini:Strings.TitleText"], + strings_locale["browser/chrome/updater/updater.ini:TitleText"], "Aggiornamento %MOZ_APP_DISPLAYNAME%", ) self.assertEqual( - strings_locale["browser/chrome/updater/updater.ini:Strings.InfoText"], + strings_locale["browser/chrome/updater/updater.ini:InfoText"], "%MOZ_APP_DISPLAYNAME% sta installando gli aggiornamenti e si avvierà fra qualche istante…", ) diff --git a/tmx_products/tmx_projectconfig.py b/tmx_products/tmx_projectconfig.py index a72f351..b5fd426 100755 --- a/tmx_products/tmx_projectconfig.py +++ b/tmx_products/tmx_projectconfig.py @@ -119,7 +119,10 @@ def readFiles(locale): for section in resource.sections: for entry in section.entries: if isinstance(entry, Entry): - entry_id = ".".join(section.id + entry.id) + if resource.format == Format.ini: + entry_id = ".".join(entry.id) + else: + entry_id = ".".join(section.id + entry.id) string_id = ( f"{self.repository_name}/{key_path}:{entry_id}" ) diff --git a/tmx_products/tmx_repository.py b/tmx_products/tmx_repository.py index 79a1e99..1138a14 100755 --- a/tmx_products/tmx_repository.py +++ b/tmx_products/tmx_repository.py @@ -1,9 +1,10 @@ #!/usr/bin/env python from functions import get_cli_parameters, get_config -from moz.l10n.resource import parse_resource +from moz.l10n.formats import Format from moz.l10n.message import serialize_message from moz.l10n.model import Entry +from moz.l10n.resource import parse_resource import codecs import json import os @@ -99,7 +100,10 @@ def extractStrings(self): for section in resource.sections: for entry in section.entries: if isinstance(entry, Entry): - entry_id = ".".join(section.id + entry.id) + if resource.format == Format.ini: + entry_id = ".".join(entry.id) + else: + entry_id = ".".join(section.id + entry.id) string_id = f"{self.getRelativePath(file_name)}:{entry_id}" if entry.properties: # Store the value of an entry with attributes only From 7829821e1e3cb2d6f48e00753dc56c95004f9bd9 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Fri, 26 Sep 2025 07:49:07 +0200 Subject: [PATCH 3/4] Refactor tmx_repository --- tmx_products/functions.py | 36 ++++++++++++++++++++++++ tmx_products/tmx_repository.py | 51 ++++++++++------------------------ 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/tmx_products/functions.py b/tmx_products/functions.py index 4721dd5..cd4a79c 100644 --- a/tmx_products/functions.py +++ b/tmx_products/functions.py @@ -1,4 +1,7 @@ from configparser import ConfigParser +from moz.l10n.formats import Format +from moz.l10n.message import serialize_message +from moz.l10n.model import Entry, Resource import argparse import os @@ -91,3 +94,36 @@ def get_cli_parameters(config: bool = False) -> argparse.Namespace: ) return parser.parse_args() + + +def parse_file( + resource: Resource, storage: dict[str, str], filename: str, id_format: str, +) -> None: + try: + for section in resource.sections: + for entry in section.entries: + if isinstance(entry, Entry): + if resource.format == Format.ini: + entry_id = ".".join(entry.id) + else: + entry_id = ".".join(section.id + entry.id) + string_id = f"{id_format}:{entry_id}" + if entry.properties: + # Store the value of an entry with attributes only + # if the value is not empty. + if not entry.value.is_empty(): + storage[string_id] = serialize_message( + resource.format, entry.value + ) + for attribute, attr_value in entry.properties.items(): + attr_id = f"{string_id}.{attribute}" + storage[attr_id] = serialize_message( + resource.format, attr_value + ) + else: + storage[string_id] = serialize_message( + resource.format, entry.value + ) + except Exception as e: + print(f"Error parsing file: {filename}") + print(e) diff --git a/tmx_products/tmx_repository.py b/tmx_products/tmx_repository.py index 1138a14..83a5c1b 100755 --- a/tmx_products/tmx_repository.py +++ b/tmx_products/tmx_repository.py @@ -1,9 +1,6 @@ #!/usr/bin/env python -from functions import get_cli_parameters, get_config -from moz.l10n.formats import Format -from moz.l10n.message import serialize_message -from moz.l10n.model import Entry +from functions import get_cli_parameters, get_config, parse_file from moz.l10n.resource import parse_resource import codecs import json @@ -85,52 +82,32 @@ def extractStrings(self): # If storage mode is append, read existing translations (if available) # before overriding them if self.storage_append: - file_name = f"{self.storage_file}.json" - if os.path.isfile(file_name): - with open(file_name) as f: + filename = f"{self.storage_file}.json" + if os.path.isfile(filename): + with open(filename) as f: self.translations = json.load(f) f.close() # Create a list of files to analyze self.extractFileList() - for file_name in self.file_list: - resource = parse_resource(file_name) + for filename in self.file_list: try: - for section in resource.sections: - for entry in section.entries: - if isinstance(entry, Entry): - if resource.format == Format.ini: - entry_id = ".".join(entry.id) - else: - entry_id = ".".join(section.id + entry.id) - string_id = f"{self.getRelativePath(file_name)}:{entry_id}" - if entry.properties: - # Store the value of an entry with attributes only - # if the value is not empty. - if not entry.value.is_empty(): - self.translations[string_id] = serialize_message( - resource.format, entry.value - ) - for attribute, attr_value in entry.properties.items(): - attr_id = f"{string_id}.{attribute}" - self.translations[attr_id] = serialize_message( - resource.format, attr_value - ) - else: - self.translations[string_id] = serialize_message( - resource.format, entry.value - ) + resource = parse_resource(filename) + rel_filename = self.getRelativePath(filename) + parse_file( + resource, self.translations, filename, f"{rel_filename}" + ) except Exception as e: - print(f"Error parsing file: {file_name}") + print(f"Error parsing resource: {filename}") print(e) # Remove extra strings from locale if self.reference_locale != self.locale: # Read the JSON cache for reference locale if available - file_name = f"{self.reference_storage_file}.json" - if os.path.isfile(file_name): - with open(file_name) as f: + filename = f"{self.reference_storage_file}.json" + if os.path.isfile(filename): + with open(filename) as f: reference_strings = json.load(f) f.close() From 23cfbf24118f983264ee7a5f33c7e56eabecbe79 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Fri, 26 Sep 2025 08:05:34 +0200 Subject: [PATCH 4/4] Reformat tmx_projectconfig --- tmx_products/functions.py | 27 +++++++++------- tmx_products/tmx_projectconfig.py | 51 ++++++------------------------- tmx_products/tmx_repository.py | 4 +-- 3 files changed, 26 insertions(+), 56 deletions(-) diff --git a/tmx_products/functions.py b/tmx_products/functions.py index cd4a79c..32dc589 100644 --- a/tmx_products/functions.py +++ b/tmx_products/functions.py @@ -1,7 +1,7 @@ from configparser import ConfigParser from moz.l10n.formats import Format from moz.l10n.message import serialize_message -from moz.l10n.model import Entry, Resource +from moz.l10n.model import Entry, Message, Resource import argparse import os @@ -97,8 +97,19 @@ def get_cli_parameters(config: bool = False) -> argparse.Namespace: def parse_file( - resource: Resource, storage: dict[str, str], filename: str, id_format: str, + resource: Resource, + storage: dict[str, str], + filename: str, + id_format: str, ) -> None: + def get_entry_value(value: Message) -> str: + entry_value = serialize_message(resource.format, value) + if resource.format == Format.android: + # In Android resources, unescape quotes + entry_value = entry_value.replace('\\"', '"').replace("\\'", "'") + + return entry_value + try: for section in resource.sections: for entry in section.entries: @@ -112,18 +123,12 @@ def parse_file( # Store the value of an entry with attributes only # if the value is not empty. if not entry.value.is_empty(): - storage[string_id] = serialize_message( - resource.format, entry.value - ) + storage[string_id] = get_entry_value(entry.value) for attribute, attr_value in entry.properties.items(): attr_id = f"{string_id}.{attribute}" - storage[attr_id] = serialize_message( - resource.format, attr_value - ) + storage[attr_id] = get_entry_value(attr_value) else: - storage[string_id] = serialize_message( - resource.format, entry.value - ) + storage[string_id] = get_entry_value(entry.value) except Exception as e: print(f"Error parsing file: {filename}") print(e) diff --git a/tmx_products/tmx_projectconfig.py b/tmx_products/tmx_projectconfig.py index b5fd426..4d8ac40 100755 --- a/tmx_products/tmx_projectconfig.py +++ b/tmx_products/tmx_projectconfig.py @@ -1,9 +1,6 @@ #!/usr/bin/env python -from functions import get_cli_parameters, get_config -from moz.l10n.formats import Format -from moz.l10n.message import serialize_message -from moz.l10n.model import Entry +from functions import get_cli_parameters, get_config, parse_file from moz.l10n.paths import L10nConfigPaths, get_android_locale from moz.l10n.resource import parse_resource import codecs @@ -58,14 +55,6 @@ def readExistingJSON(locale): return translations - def getEntryValue(resource, value): - entry_value = serialize_message(resource.format, value) - if resource.format == Format.android: - # In Android resources, unescape quotes - entry_value = entry_value.replace('\\"', '"').replace("\\'", "'") - - return entry_value - def readFiles(locale): """Read files for locale""" @@ -116,37 +105,15 @@ def readFiles(locale): resource = parse_resource( l10n_file, android_literal_quotes=True ) - for section in resource.sections: - for entry in section.entries: - if isinstance(entry, Entry): - if resource.format == Format.ini: - entry_id = ".".join(entry.id) - else: - entry_id = ".".join(section.id + entry.id) - string_id = ( - f"{self.repository_name}/{key_path}:{entry_id}" - ) - if entry.properties: - # Store the value of an entry with attributes only - # if the value is not empty. - if not entry.value.is_empty(): - self.translations[locale][string_id] = ( - getEntryValue(resource, entry.value) - ) - for ( - attribute, - attr_value, - ) in entry.properties.items(): - attr_id = f"{string_id}.{attribute}" - self.translations[locale][attr_id] = ( - getEntryValue(resource, attr_value) - ) - else: - self.translations[locale][string_id] = ( - getEntryValue(resource, entry.value) - ) + + parse_file( + resource, + self.translations[locale], + l10n_file, + f"{self.repository_name}/{key_path}", + ) except Exception as e: - print(f"Error parsing file: {reference_file}") + print(f"Error parsing resource: {reference_file}") print(e) basedir = os.path.dirname(self.toml_path) diff --git a/tmx_products/tmx_repository.py b/tmx_products/tmx_repository.py index 83a5c1b..672ce8c 100755 --- a/tmx_products/tmx_repository.py +++ b/tmx_products/tmx_repository.py @@ -95,9 +95,7 @@ def extractStrings(self): try: resource = parse_resource(filename) rel_filename = self.getRelativePath(filename) - parse_file( - resource, self.translations, filename, f"{rel_filename}" - ) + parse_file(resource, self.translations, filename, f"{rel_filename}") except Exception as e: print(f"Error parsing resource: {filename}") print(e)