diff --git a/poetry.lock b/poetry.lock index d1b8608c..e80f63bd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1100,6 +1100,35 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "ruff" +version = "0.14.4" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "ruff-0.14.4-py3-none-linux_armv6l.whl", hash = "sha256:e6604613ffbcf2297cd5dcba0e0ac9bd0c11dc026442dfbb614504e87c349518"}, + {file = "ruff-0.14.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d99c0b52b6f0598acede45ee78288e5e9b4409d1ce7f661f0fa36d4cbeadf9a4"}, + {file = "ruff-0.14.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9358d490ec030f1b51d048a7fd6ead418ed0826daf6149e95e30aa67c168af33"}, + {file = "ruff-0.14.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81b40d27924f1f02dfa827b9c0712a13c0e4b108421665322218fc38caf615c2"}, + {file = "ruff-0.14.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f5e649052a294fe00818650712083cddc6cc02744afaf37202c65df9ea52efa5"}, + {file = "ruff-0.14.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa082a8f878deeba955531f975881828fd6afd90dfa757c2b0808aadb437136e"}, + {file = "ruff-0.14.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:1043c6811c2419e39011890f14d0a30470f19d47d197c4858b2787dfa698f6c8"}, + {file = "ruff-0.14.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a9f3a936ac27fb7c2a93e4f4b943a662775879ac579a433291a6f69428722649"}, + {file = "ruff-0.14.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:95643ffd209ce78bc113266b88fba3d39e0461f0cbc8b55fb92505030fb4a850"}, + {file = "ruff-0.14.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:456daa2fa1021bc86ca857f43fe29d5d8b3f0e55e9f90c58c317c1dcc2afc7b5"}, + {file = "ruff-0.14.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:f911bba769e4a9f51af6e70037bb72b70b45a16db5ce73e1f72aefe6f6d62132"}, + {file = "ruff-0.14.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:76158a7369b3979fa878612c623a7e5430c18b2fd1c73b214945c2d06337db67"}, + {file = "ruff-0.14.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f3b8f3b442d2b14c246e7aeca2e75915159e06a3540e2f4bed9f50d062d24469"}, + {file = "ruff-0.14.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c62da9a06779deecf4d17ed04939ae8b31b517643b26370c3be1d26f3ef7dbde"}, + {file = "ruff-0.14.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5a443a83a1506c684e98acb8cb55abaf3ef725078be40237463dae4463366349"}, + {file = "ruff-0.14.4-py3-none-win32.whl", hash = "sha256:643b69cb63cd996f1fc7229da726d07ac307eae442dd8974dbc7cf22c1e18fff"}, + {file = "ruff-0.14.4-py3-none-win_amd64.whl", hash = "sha256:26673da283b96fe35fa0c939bf8411abec47111644aa9f7cfbd3c573fb125d2c"}, + {file = "ruff-0.14.4-py3-none-win_arm64.whl", hash = "sha256:dd09c292479596b0e6fec8cd95c65c3a6dc68e9ad17b8f2382130f87ff6a75bb"}, + {file = "ruff-0.14.4.tar.gz", hash = "sha256:f459a49fe1085a749f15414ca76f61595f1a2cc8778ed7c279b6ca2e1fd19df3"}, +] + [[package]] name = "selenium" version = "4.37.0" @@ -1363,4 +1392,4 @@ h11 = ">=0.9.0,<1" [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "0c33d68a88f63080120c52ca56e9726c0f509ba2823d21196b96f09d78832486" +content-hash = "5c0bf36adfd6df57cb03c37d00cf72239f29a7d636bbee6d1f12d95f51c718bd" diff --git a/pyproject.toml b/pyproject.toml index 0c5aae68..f08b3061 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ pytest-repeat = "^0.9.3" flask = "^3.1.0" python-dateutil = "^2.9.0.post0" tox = "^4.31.0" +ruff = "^0.14.4" [build-system] requires = ["poetry-core"] diff --git a/tests/android/conftest.py b/tests/android/conftest.py index a48f0676..21892fc5 100644 --- a/tests/android/conftest.py +++ b/tests/android/conftest.py @@ -119,7 +119,9 @@ def fixture_experiment_feature(request): @pytest.fixture(name="check_ping_for_experiment") def fixture_check_ping_for_experiment(experiment_slug, variables): - def _check_ping_for_experiment(branch=None, experiment=experiment_slug, reason=None): + def _check_ping_for_experiment( + branch=None, experiment=experiment_slug, reason=None + ): model = TelemetryModel(branch=branch, experiment=experiment) timeout = time.time() + 60 * 5 @@ -142,7 +144,8 @@ def _check_ping_for_experiment(branch=None, experiment=experiment_slug, reason=N for event in events: event_name = event.get("name") if (reason == "enrollment" and event_name == "enrollment") or ( - reason == "unenrollment" and event_name in ["unenrollment", "disqualification"] + reason == "unenrollment" + and event_name in ["unenrollment", "disqualification"] ): telemetry_model = TelemetryModel( branch=event["extra"]["branch"], @@ -278,7 +281,9 @@ def fixture_dismiss_system_dialogs(): @pytest.fixture(name="setup_experiment") -def fixture_setup_experiment(run_nimbus_cli_command, experiment_slug, experiment_branch): +def fixture_setup_experiment( + run_nimbus_cli_command, experiment_slug, experiment_branch +): def setup_experiment(): logging.info("====== Beginning Test ======") command = [ @@ -294,6 +299,8 @@ def setup_experiment(): "--reset-app", ] run_nimbus_cli_command(" ".join(command)) - time.sleep(10) # Wait a while as there's no real way to know when the app has started + time.sleep( + 10 + ) # Wait a while as there's no real way to know when the app has started return setup_experiment diff --git a/tests/android/generate_smoke_tests.py b/tests/android/generate_smoke_tests.py index 37fc5fc5..b365bc8e 100644 --- a/tests/android/generate_smoke_tests.py +++ b/tests/android/generate_smoke_tests.py @@ -10,7 +10,9 @@ parser = argparse.ArgumentParser("Options for android apk downloader") -parser.add_argument("--test-files", nargs="+", help="List of test files to generate tests from") +parser.add_argument( + "--test-files", nargs="+", help="List of test files to generate tests from" +) args = parser.parse_args() diff --git a/tests/android/tests/test_generic_scenarios.py b/tests/android/tests/test_generic_scenarios.py index 4ca50648..1cbddda5 100644 --- a/tests/android/tests/test_generic_scenarios.py +++ b/tests/android/tests/test_generic_scenarios.py @@ -2,7 +2,9 @@ @pytest.mark.generic_test -def test_experiment_unenrolls_via_studies_toggle(setup_experiment, gradlewbuild, open_app): +def test_experiment_unenrolls_via_studies_toggle( + setup_experiment, gradlewbuild, open_app +): setup_experiment() open_app() gradlewbuild.test("GenericExperimentIntegrationTest#disableStudiesViaStudiesToggle") @@ -25,6 +27,8 @@ def test_experiment_unenrolls_via_secret_menu( ): setup_experiment() open_app() - gradlewbuild.test("GenericExperimentIntegrationTest#testExperimentUnenrolledViaSecretMenu") + gradlewbuild.test( + "GenericExperimentIntegrationTest#testExperimentUnenrolledViaSecretMenu" + ) gradlewbuild.test("GenericExperimentIntegrationTest#testExperimentUnenrolled") assert check_ping_for_experiment(reason="unenrollment", branch=load_branches[0]) diff --git a/tests/android/tests/test_survey_messages.py b/tests/android/tests/test_survey_messages.py index 3db0fd1f..7bddd6d6 100644 --- a/tests/android/tests/test_survey_messages.py +++ b/tests/android/tests/test_survey_messages.py @@ -14,16 +14,22 @@ def test_survey_navigates_correctly(setup_experiment, gradlewbuild): @pytest.mark.messaging_survey def test_survey_no_thanks_navigates_correctly(setup_experiment, gradlewbuild): setup_experiment() - gradlewbuild.test("SurveyExperimentIntegrationTest#checkSurveyNoThanksNavigatesCorrectly") + gradlewbuild.test( + "SurveyExperimentIntegrationTest#checkSurveyNoThanksNavigatesCorrectly" + ) @pytest.mark.messaging_homescreen def test_homescreen_survey_dismisses_correctly(setup_experiment, gradlewbuild): setup_experiment() - gradlewbuild.test("SurveyExperimentIntegrationTest#checkHomescreenSurveyDismissesCorrectly") + gradlewbuild.test( + "SurveyExperimentIntegrationTest#checkHomescreenSurveyDismissesCorrectly" + ) @pytest.mark.messaging_survey def test_survey_landscape_looks_correct(setup_experiment, gradlewbuild): setup_experiment() - gradlewbuild.test("SurveyExperimentIntegrationTest#checkSurveyLandscapeLooksCorrect") + gradlewbuild.test( + "SurveyExperimentIntegrationTest#checkSurveyLandscapeLooksCorrect" + ) diff --git a/tests/conftest.py b/tests/conftest.py index 4ea98b8c..2b970311 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,49 +25,63 @@ def pytest_addoption(parser) -> None: - parser.addoption( - "--experiment-recipe", - action="store", - default=None, - help="Recipe JSON for experiment", - ), - parser.addoption( - "--experiment-branch", - action="store", - default=None, - help="Experiment Branch to test", - ), - parser.addoption( - "--run-update-test", - action="store_true", - default=None, - help="Run older version of firefox", - ), - parser.addoption( - "--private-browsing-enabled", - action="store_true", - default=None, - help="Run private browsing test", - ), - parser.addoption( - "--experiment-slug", - action="store", - default=None, - help="Experiment slug from Experimenter", - ), - parser.addoption( - "--experiment-server", - action="store", - default="prod", - choices=("prod", "stage", "stage/preview"), - help="The server where the experiment is located, either stage or prod", - ), - parser.addoption( - "--experiment-json", - action="store", - default=None, - help="The experiments JSON file.", - ), + ( + parser.addoption( + "--experiment-recipe", + action="store", + default=None, + help="Recipe JSON for experiment", + ), + ) + ( + parser.addoption( + "--experiment-branch", + action="store", + default=None, + help="Experiment Branch to test", + ), + ) + ( + parser.addoption( + "--run-update-test", + action="store_true", + default=None, + help="Run older version of firefox", + ), + ) + ( + parser.addoption( + "--private-browsing-enabled", + action="store_true", + default=None, + help="Run private browsing test", + ), + ) + ( + parser.addoption( + "--experiment-slug", + action="store", + default=None, + help="Experiment slug from Experimenter", + ), + ) + ( + parser.addoption( + "--experiment-server", + action="store", + default="prod", + choices=("prod", "stage", "stage/preview"), + help="The server where the experiment is located, either stage or prod", + ), + ) + ( + parser.addoption( + "--experiment-json", + action="store", + default=None, + help="The experiments JSON file.", + ), + ) parser.addoption( "--firefox-path", action="store", @@ -76,6 +90,10 @@ def pytest_addoption(parser) -> None: ) +MIN_FAILURES_FOR_EXIT_CODE = 6 +FAILURE_RATIO_THRESHOLD = 0.1 + + def start_process(path, command): module_path = Path(path) @@ -163,7 +181,9 @@ def fixture_enroll_experiment( try: with selenium.context(selenium.CONTEXT_CHROME): time.sleep(5) - result = selenium.execute_async_script(script, experiment_json, experiment_branch) + result = selenium.execute_async_script( + script, experiment_json, experiment_branch + ) logging.info(f"Force Enrolling: {result}") except JavascriptException as e: if "slug" in str(e): @@ -201,16 +221,16 @@ def setup_profile(pytestconfig: typing.Any, request: typing.Any) -> typing.Any: dirs_exist_ok=True, ignore_dangling_symlinks=True, ) - return f'{Path("utilities/klaatu-profile").absolute()}' - if request.node.get_closest_marker("reuse_profile") and not request.config.getoption( - "--run-update-test" - ): + return f"{Path('utilities/klaatu-profile').absolute()}" + if request.node.get_closest_marker( + "reuse_profile" + ) and not request.config.getoption("--run-update-test"): shutil.copytree( Path("utilities/klaatu-profile-firefox-base").absolute(), Path("utilities/klaatu-profile-current-base").absolute(), dirs_exist_ok=True, ) - return f'{Path("utilities/klaatu-profile-current-base").absolute()}' + return f"{Path('utilities/klaatu-profile-current-base').absolute()}" @pytest.fixture @@ -227,20 +247,22 @@ def firefox_options( logging.info(f"Using firefox at {firefox_path}") firefox_options.log.level = "trace" if request.config.getoption("--run-update-test"): - if request.node.get_closest_marker("update_test"): # disable test needs different firefox + if request.node.get_closest_marker( + "update_test" + ): # disable test needs different firefox binary = Path( "utilities/firefox-old-nightly-disable-test/firefox/firefox-bin" ).absolute() firefox_options.binary = f"{binary}" firefox_options.add_argument("-profile") firefox_options.add_argument( - f'{Path("utilities/klaatu-profile-disable-test").absolute()}' + f"{Path('utilities/klaatu-profile-disable-test').absolute()}" ) firefox_options.add_argument("-profile") firefox_options.add_argument(setup_profile) - if request.node.get_closest_marker("reuse_profile") and not request.config.getoption( - "--run-update-test" - ): + if request.node.get_closest_marker( + "reuse_profile" + ) and not request.config.getoption("--run-update-test"): firefox_options.add_argument("-profile") firefox_options.add_argument(setup_profile) firefox_options.add_argument("-remote-allow-system-access") @@ -261,7 +283,9 @@ def firefox_options( firefox_options.set_preference("toolkit.telemetry.log.level", "Trace") firefox_options.set_preference("toolkit.telemetry.log.dump", True) firefox_options.set_preference("toolkit.telemetry.send.overrideOfficialCheck", True) - firefox_options.set_preference("toolkit.telemetry.testing.disableFuzzingDelay", True) + firefox_options.set_preference( + "toolkit.telemetry.testing.disableFuzzingDelay", True + ) firefox_options.set_preference("nimbus.debug", True) firefox_options.set_preference("app.normandy.run_interval_seconds", 30) firefox_options.set_preference( @@ -269,7 +293,9 @@ def firefox_options( "5E:36:F2:14:DE:82:3F:8B:29:96:89:23:5F:03:41:AC:AF:A0:75:AF:82:CB:4C:D4:30:7C:3D:B3:43:39:2A:FE", # noqa: E501 ) firefox_options.set_preference("datareporting.healthreport.service.enabled", True) - firefox_options.set_preference("datareporting.healthreport.logging.consoleEnabled", True) + firefox_options.set_preference( + "datareporting.healthreport.logging.consoleEnabled", True + ) firefox_options.set_preference("datareporting.healthreport.service.firstRun", True) firefox_options.set_preference( "datareporting.healthreport.documentServerURI", @@ -278,7 +304,9 @@ def firefox_options( firefox_options.set_preference( "app.normandy.api_url", "https://normandy.cdn.mozilla.net/api/v1" ) - firefox_options.set_preference("app.normandy.user_id", "7ef5ab6d-42d6-4c4e-877d-c3174438050a") + firefox_options.set_preference( + "app.normandy.user_id", "7ef5ab6d-42d6-4c4e-877d-c3174438050a" + ) firefox_options.set_preference("messaging-system.log", "debug") firefox_options.set_preference("toolkit.telemetry.scheduler.tickInterval", 15) firefox_options.set_preference("toolkit.telemetry.collectInterval", 10) @@ -427,9 +455,11 @@ def _navigate_function(text=None, use_clipboard=False): el = selenium.find_element(By.CSS_SELECTOR, "#urlbar-input") WebDriverWait(selenium, 60).until(EC.element_to_be_clickable(el)) if use_clipboard: - ActionChains(selenium).move_to_element(el).pause(1).click().pause(1).key_down( + ActionChains(selenium).move_to_element(el).pause(1).click().pause( + 1 + ).key_down(cmd_or_ctrl_button).send_keys("v").key_up( cmd_or_ctrl_button - ).send_keys("v").key_up(cmd_or_ctrl_button).send_keys(Keys.ENTER).perform() + ).send_keys(Keys.ENTER).perform() return else: el.click() @@ -449,7 +479,9 @@ def _navigate_function(text=None, use_clipboard=False): @pytest.fixture(name="find_impression") def fixture_find_impression(selenium, find_telemetry): - def fixture_find_impression_runner(source: str, provider: str, tagged: bool) -> bool: + def fixture_find_impression_runner( + source: str, provider: str, tagged: bool + ) -> bool: control = True timeout = time.time() + 120 script = "return Glean.serp.impression.testGetValue()" @@ -718,6 +750,15 @@ def pytest_sessionfinish(session, exitstatus): session.config.stash[metadata_key]["Firefox Version"] = os.environ.get( "FIREFOX_VERSION", "N/A" ) + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter: + failed_count = len(reporter.stats.get("failed", [])) + passed_count = len(reporter.stats.get("passed", [])) + total_count = failed_count + passed_count + if failed_count > 0 and (failed_count / total_count) > FAILURE_RATIO_THRESHOLD: + session.exitstatus = 1 + else: + session.exitstatus = 0 @then("Firefox should be allowed to open a new tab") @@ -737,7 +778,8 @@ def check_new_tab(selenium): @given( - "Firefox is launched enrolled in an Experiment with custom search", target_fixture="selenium" + "Firefox is launched enrolled in an Experiment with custom search", + target_fixture="selenium", ) def setup_browser(selenium, setup_search_test): selenium.implicitly_wait(5) diff --git a/tests/ios/conftest.py b/tests/ios/conftest.py index c892046b..a90bcadc 100644 --- a/tests/ios/conftest.py +++ b/tests/ios/conftest.py @@ -20,8 +20,12 @@ def pytest_addoption(parser): - parser.addoption("--experiment-slug", action="store", help="The experiments experimenter URL") - parser.addoption("--stage", action="store_true", default=None, help="Use the stage server") + parser.addoption( + "--experiment-slug", action="store", help="The experiments experimenter URL" + ) + parser.addoption( + "--stage", action="store_true", default=None, help="Use the stage server" + ) parser.addoption( "--build-dev", action="store_true", @@ -29,7 +33,9 @@ def pytest_addoption(parser): help="Build the developer edition of Firefox", ) parser.addoption( - "--experiment-feature", action="store", help="Feature name you want to test against" + "--experiment-feature", + action="store", + help="Feature name you want to test against", ) parser.addoption( "--experiment-branch", @@ -132,7 +138,9 @@ def fixture_build_fennec(request): @pytest.fixture() def xcodebuild(xcodebuild_log): - yield XCodeBuild(xcodebuild_log, scheme="Fennec", test_plan="ExperimentIntegrationTests") + yield XCodeBuild( + xcodebuild_log, scheme="Fennec", test_plan="ExperimentIntegrationTests" + ) @pytest.fixture(scope="session") @@ -230,7 +238,9 @@ def fixture_set_env_variables(experiment_data): @pytest.fixture(name="check_ping_for_experiment") def fixture_check_ping_for_experiment(experiment_slug, variables): - def _check_ping_for_experiment(branch=None, experiment=experiment_slug, reason=None): + def _check_ping_for_experiment( + branch=None, experiment=experiment_slug, reason=None + ): model = TelemetryModel(branch=branch, experiment=experiment) timeout = time.time() + 60 * 5 @@ -252,7 +262,8 @@ def _check_ping_for_experiment(branch=None, experiment=experiment_slug, reason=N for event in events: event_name = event.get("name") if (reason == "enrollment" and event_name == "enrollment") or ( - reason == "unenrollment" and event_name in ["unenrollment", "disqualification"] + reason == "unenrollment" + and event_name in ["unenrollment", "disqualification"] ): telemetry_model = TelemetryModel( branch=event["extra"]["branch"], @@ -282,10 +293,16 @@ def _run_nimbus_cli_command(command): @pytest.fixture(name="setup_experiment") def setup_experiment( - experiment_slug, experiment_server, experiment_branch, run_nimbus_cli_command, nimbus_cli_args + experiment_slug, + experiment_server, + experiment_branch, + run_nimbus_cli_command, + nimbus_cli_args, ): def _setup_experiment(): - logging.info(f"Testing experiment {experiment_slug}, BRANCH: {experiment_branch}") + logging.info( + f"Testing experiment {experiment_slug}, BRANCH: {experiment_branch}" + ) command = [ "nimbus-cli", "--app firefox_ios", diff --git a/tests/ios/generate_smoke_tests.py b/tests/ios/generate_smoke_tests.py index 8fefd42c..7b899826 100644 --- a/tests/ios/generate_smoke_tests.py +++ b/tests/ios/generate_smoke_tests.py @@ -6,7 +6,9 @@ parser = argparse.ArgumentParser("Options for android apk downloader") -parser.add_argument("--test-files", nargs="+", help="List of test files to generate tests from") +parser.add_argument( + "--test-files", nargs="+", help="List of test files to generate tests from" +) args = parser.parse_args() @@ -36,7 +38,9 @@ def search_for_smoke_tests(tests_name): test_names.append(code[locations[0] + 1].strip(":")) for location in locations: - for count in range(5): # loop forward to get 'func' location and then test name + for count in range( + 5 + ): # loop forward to get 'func' location and then test name if "func" in code[location + count]: test_name = code[location + count + 1] test_names.append(test_name) diff --git a/tests/ios/get_specific_device_udid.py b/tests/ios/get_specific_device_udid.py index a0057604..a250817e 100644 --- a/tests/ios/get_specific_device_udid.py +++ b/tests/ios/get_specific_device_udid.py @@ -16,6 +16,10 @@ filtered_devices = {} for runtime, runtime_devices in devices["devices"].items(): for device in runtime_devices: - if device["isAvailable"] and json_version in runtime and device_name == device["name"]: + if ( + device["isAvailable"] + and json_version in runtime + and device_name == device["name"] + ): print(device["udid"]) break diff --git a/tests/ios/test_messaging.py b/tests/ios/test_messaging.py index 3a0c3589..f5fae646 100644 --- a/tests/ios/test_messaging.py +++ b/tests/ios/test_messaging.py @@ -8,7 +8,11 @@ @pytest.mark.messaging_survey def test_survey_navigates_correctly( - xcodebuild, setup_experiment, start_app, experiment_branch, check_ping_for_experiment + xcodebuild, + setup_experiment, + start_app, + experiment_branch, + check_ping_for_experiment, ): xcodebuild.install(boot=False) setup_experiment() @@ -29,7 +33,11 @@ def test_survey_navigates_correctly( @pytest.mark.messaging_survey def test_survey_no_thanks_navigates_correctly( - xcodebuild, setup_experiment, start_app, experiment_branch, check_ping_for_experiment + xcodebuild, + setup_experiment, + start_app, + experiment_branch, + check_ping_for_experiment, ): xcodebuild.install(boot=False) setup_experiment() @@ -50,7 +58,11 @@ def test_survey_no_thanks_navigates_correctly( @pytest.mark.messaging_survey def test_survey_landscape_looks_correct( - xcodebuild, setup_experiment, start_app, experiment_branch, check_ping_for_experiment + xcodebuild, + setup_experiment, + start_app, + experiment_branch, + check_ping_for_experiment, ): xcodebuild.install(boot=False) setup_experiment() @@ -71,7 +83,11 @@ def test_survey_landscape_looks_correct( @pytest.mark.messaging_new_tab_card def test_homescreen_survey_navigates_correctly( - xcodebuild, setup_experiment, start_app, experiment_branch, check_ping_for_experiment + xcodebuild, + setup_experiment, + start_app, + experiment_branch, + check_ping_for_experiment, ): xcodebuild.install(boot=False) setup_experiment() @@ -92,7 +108,11 @@ def test_homescreen_survey_navigates_correctly( @pytest.mark.messaging_new_tab_card def test_homescreen_survey_dismisses_correctly( - xcodebuild, setup_experiment, start_app, experiment_branch, check_ping_for_experiment + xcodebuild, + setup_experiment, + start_app, + experiment_branch, + check_ping_for_experiment, ): xcodebuild.install(boot=False) setup_experiment() diff --git a/tests/ios/xcodebuild.py b/tests/ios/xcodebuild.py index 5aa69739..5519122a 100644 --- a/tests/ios/xcodebuild.py +++ b/tests/ios/xcodebuild.py @@ -14,7 +14,9 @@ def __init__(self, log, **kwargs): self.device = os.getenv("SIMULATOR_DEVICE", "iPhone 17") self.ios_version = os.getenv("IOS_VERSION", "26.0") self.binary = "xcodebuild" - self.destination = f"platform=iOS Simulator,name={self.device},OS={self.ios_version}" + self.destination = ( + f"platform=iOS Simulator,name={self.device},OS={self.ios_version}" + ) self.scheme = "Fennec" self.testPlan = "ExperimentIntegrationTests" self.xcrun = XCRun() @@ -66,7 +68,10 @@ def test(self, identifier, build=True, erase=True): self.logger.info("Running: {}".format(" ".join(args))) try: out = subprocess.check_output( - args, cwd=f"{here.parents[3]}", stderr=subprocess.STDOUT, universal_newlines=True + args, + cwd=f"{here.parents[3]}", + stderr=subprocess.STDOUT, + universal_newlines=True, ) except subprocess.CalledProcessError as e: out = e.output diff --git a/tests/scenarios/test_functionality_nimbus.py b/tests/scenarios/test_functionality_nimbus.py index 2afcf9a9..a2438b5e 100644 --- a/tests/scenarios/test_functionality_nimbus.py +++ b/tests/scenarios/test_functionality_nimbus.py @@ -60,7 +60,6 @@ def open_a_new_tab_via_keyboard(cmd_or_ctrl_button, selenium): @then("The user will install a language pack") def install_acholi_language_pack(selenium, request): - add_button_locator = (By.CSS_SELECTOR, "#add") addon_installed_locator = (By.CSS_SELECTOR, "#appMenu-addon-installed-notification") add_to_firefox_locator = ( @@ -74,17 +73,24 @@ def install_acholi_language_pack(selenium, request): ) language_button_locator = (By.CSS_SELECTOR, "#manageBrowserLanguagesButton") language_search_locator = (By.CSS_SELECTOR, ".in-menulist menuitem label") - menu_list_locator = (By.CSS_SELECTOR, ".languages-grid #availableLocales .in-menulist") + menu_list_locator = ( + By.CSS_SELECTOR, + ".languages-grid #availableLocales .in-menulist", + ) if not request.config.getoption("--run-update-test"): pytest.skip("needs --run-update-test option to run") return # install language pack - selenium.get("https://addons.mozilla.org/en-US/firefox/addon/acholi-ug-language-pack/") + selenium.get( + "https://addons.mozilla.org/en-US/firefox/addon/acholi-ug-language-pack/" + ) selenium.find_element(By.CSS_SELECTOR, ".AMInstallButton-button").click() with selenium.context(selenium.CONTEXT_CHROME): - WebDriverWait(selenium, 60).until(EC.element_to_be_clickable(add_to_firefox_locator)) + WebDriverWait(selenium, 60).until( + EC.element_to_be_clickable(add_to_firefox_locator) + ) time.sleep(5) # need to sleep as the waits sometimes don't work selenium.find_element(*add_to_firefox_locator).click() WebDriverWait(selenium, 60).until( @@ -95,14 +101,18 @@ def install_acholi_language_pack(selenium, request): selenium.get("about:preferences") button = selenium.find_element(*language_button_locator) button.click() - WebDriverWait(selenium, 60).until(EC.visibility_of_element_located(root_dialog_box_locator)) + WebDriverWait(selenium, 60).until( + EC.visibility_of_element_located(root_dialog_box_locator) + ) dialog = selenium.find_element(*root_dialog_box_locator) selenium.switch_to.frame(dialog) dialog = selenium.find_element(*browser_dialog_box_locator) menu_list = selenium.find_element(*menu_list_locator) menu_list.click() - WebDriverWait(menu_list, 60).until(EC.visibility_of_element_located(language_search_locator)) + WebDriverWait(menu_list, 60).until( + EC.visibility_of_element_located(language_search_locator) + ) el = menu_list.find_element(*language_search_locator) ActionChains(selenium).move_to_element(el).pause(1).click().perform() ActionBuilder(selenium).clear_actions() @@ -111,7 +121,9 @@ def install_acholi_language_pack(selenium, request): for item in language_list: if "Acholi" in item.get_attribute("label"): selenium.execute_script("arguments[0].scrollIntoView(true);", item) - ActionChains(selenium).move_to_element(item).pause(1).click().pause(1).perform() + ActionChains(selenium).move_to_element(item).pause(1).click().pause( + 1 + ).perform() break WebDriverWait(dialog, 60).until( EC.element_to_be_clickable(add_button_locator), message="Language was not added" @@ -132,7 +144,9 @@ def check_for_localized_firefox(selenium): ) for item in list: if "Acholi" in item.get_attribute("label"): - ActionChains(selenium).move_to_element(item).pause(1).click().pause(1).perform() + ActionChains(selenium).move_to_element(item).pause(1).click().pause( + 1 + ).perform() break """ This translates from: diff --git a/tests/scenarios/test_generic_nimbus.py b/tests/scenarios/test_generic_nimbus.py index 45d841f1..74e87446 100644 --- a/tests/scenarios/test_generic_nimbus.py +++ b/tests/scenarios/test_generic_nimbus.py @@ -14,7 +14,9 @@ @then("The experiment branch should be correctly reported") -def check_branch_in_telemetry(telemetry_event_check, experiment_json, request, experiment_slug): +def check_branch_in_telemetry( + telemetry_event_check, experiment_json, request, experiment_slug +): experiment_branch = request.config.getoption("--experiment-branch") telemetry_event_check(f"optin-{experiment_slug}") assert experiment_branch in experiment_json["branch"] @@ -27,9 +29,13 @@ def unenroll_via_studies_page(selenium, experiment_json): timeout = timeout = time.time() + 60 while time.time() < timeout: selenium.get("about:studies") - WebDriverWait(selenium, 30).until(EC.presence_of_element_located(study_name_locator)) + WebDriverWait(selenium, 30).until( + EC.presence_of_element_located(study_name_locator) + ) items = selenium.find_elements(*study_name_locator) - if any(item for item in items if experiment_json["userFacingName"] in item.text): + if any( + item for item in items if experiment_json["userFacingName"] in item.text + ): logging.info("Experiment unenrolled") return True time.sleep(2) diff --git a/tests/scenarios/test_generic_telemetry.py b/tests/scenarios/test_generic_telemetry.py index e82b3292..1830afe5 100644 --- a/tests/scenarios/test_generic_telemetry.py +++ b/tests/scenarios/test_generic_telemetry.py @@ -19,12 +19,18 @@ ) -@scenario("../features/generic_telemetry.feature", "Report correct telemetry for organic searches") +@scenario( + "../features/generic_telemetry.feature", + "Report correct telemetry for organic searches", +) def test_report_correct_telemetry_for_organic_searches(): pass -@scenario("../features/generic_telemetry.feature", "Report correct telemetry for tagged searches") +@scenario( + "../features/generic_telemetry.feature", + "Report correct telemetry for tagged searches", +) def test_report_correct_telemetry_for_tagged_searches(): pass @@ -101,7 +107,9 @@ def search_using_search_bar_to_return_ads(selenium, enable_search_bar): # perform search with selenium.context(selenium.CONTEXT_CHROME): - WebDriverWait(selenium, 60).until(EC.visibility_of_element_located(search_box_locator)) + WebDriverWait(selenium, 60).until( + EC.visibility_of_element_located(search_box_locator) + ) search_bar = selenium.find_element(*search_box_locator) search_bar.send_keys("buy stocks") search_bar.send_keys(Keys.ENTER) @@ -112,9 +120,9 @@ def search_using_context_click_menu(selenium, static_server, find_telemetry): selenium.get(static_server) el = selenium.find_element(By.CSS_SELECTOR, "#search-to-return-ads") - ActionChains(selenium).move_to_element(el).pause(1).double_click(el).pause(1).context_click( - el - ).perform() + ActionChains(selenium).move_to_element(el).pause(1).double_click(el).pause( + 1 + ).context_click(el).perform() with selenium.context(selenium.CONTEXT_CHROME): menu = selenium.find_element(By.CSS_SELECTOR, "#contentAreaContextMenu") menu.find_element(By.CSS_SELECTOR, "#context-searchselect").click() @@ -137,7 +145,9 @@ def search_using_context_click_menu_full(selenium, static_server, find_telemetry with selenium.context(selenium.CONTEXT_CHROME): menu = selenium.find_element(By.CSS_SELECTOR, "#contentAreaContextMenu") menu.find_element(By.CSS_SELECTOR, "#context-searchselect").click() - WebDriverWait(selenium, 60).until(EC.number_of_windows_to_be(current_windows + 1)) + WebDriverWait(selenium, 60).until( + EC.number_of_windows_to_be(current_windows + 1) + ) try: assert find_telemetry( "browser.search.withads.contextmenu", scalar="klaatu:tagged", value=1 @@ -150,12 +160,20 @@ def search_using_context_click_menu_full(selenium, static_server, find_telemetry assert False -@then(parsers.parse("The browser reports correct telemetry for the {search:w} search event")) +@then( + parsers.parse( + "The browser reports correct telemetry for the {search:w} search event" + ) +) def check_telemetry_for_with_ads_search(find_impression, search): assert find_impression(source=search, provider="klaatu", tagged="true") -@then(parsers.parse("The browser reports correct telemetry for the {search:w} adclick event")) +@then( + parsers.parse( + "The browser reports correct telemetry for the {search:w} adclick event" + ) +) def check_telemetry_for_ad_click_search(find_impression, search): assert find_impression(source=search, provider="klaatu", tagged="true") @@ -246,7 +264,9 @@ def close_browser(selenium): @then("The page loads") def wait_for_ad_click_page_to_load(selenium): - WebDriverWait(selenium, 60).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "body"))) + WebDriverWait(selenium, 60).until( + EC.visibility_of_element_located((By.CSS_SELECTOR, "body")) + ) @then("The user goes back to the search page") diff --git a/tests/scenarios/test_user_interface.py b/tests/scenarios/test_user_interface.py index ee7bbacd..1a7ec16e 100644 --- a/tests/scenarios/test_user_interface.py +++ b/tests/scenarios/test_user_interface.py @@ -32,12 +32,16 @@ def test_experiment_does_not_drastically_slow_firefox(): @pytest.mark.xfail(reason="Weird behavior with firefox and displaying the string") -@scenario("../features/user_interface.feature", "The experiment shows on the studies page") +@scenario( + "../features/user_interface.feature", "The experiment shows on the studies page" +) def test_experiment_shows_on_studies_page(): pass -@scenario("../features/user_interface.feature", "The experiment shows on the support page") +@scenario( + "../features/user_interface.feature", "The experiment shows on the support page" +) def test_experiment_shows_on_support_page(): pass @@ -104,9 +108,9 @@ def firefox_speed(selenium, firefox_startup_time): return perfData.loadEventEnd - perfData.navigationStart """ ) - assert ( - (int(firefox_startup_time) * 0.2) + int(firefox_startup_time) - ) >= startup, "This experiment caused a slowdown within Firefox." + assert ((int(firefox_startup_time) * 0.2) + int(firefox_startup_time)) >= startup, ( + "This experiment caused a slowdown within Firefox." + ) @then("A user chooses to update Firefox") @@ -138,7 +142,7 @@ def start_updated_firefox(): # Build new Firefox Instance options = Options() options.add_argument("-profile") - options.add_argument(f'{Path("utilities/klaatu-profile").absolute()}') + options.add_argument(f"{Path('utilities/klaatu-profile').absolute()}") options.add_argument("-headless") binary = f"{Path('utilities/firefox-old-nightly/firefox/firefox-bin').absolute()}" options.binary_location = f"{binary}" diff --git a/tests/toolbar.py b/tests/toolbar.py index ec564db9..a4d8c6a4 100644 --- a/tests/toolbar.py +++ b/tests/toolbar.py @@ -29,7 +29,7 @@ def __init__(self, root: typing.Any, selenium: typing.Any) -> None: def _id(self) -> str: """Extension name.""" with self.selenium.context(self.selenium.CONTEXT_CHROME): - return f'{self.root.get_attribute("data-extensionid")}' + return f"{self.root.get_attribute('data-extensionid')}" @property def widget_id(self) -> str: diff --git a/tox.ini b/tox.ini index 7610a595..f8948267 100644 --- a/tox.ini +++ b/tox.ini @@ -31,19 +31,13 @@ commands = basepython = py312 commands = poetry install --no-root - poetry -V - poetry install --no-root - poetry run isort --sp pyproject.toml -c tests/ - poetry run black --quiet --diff --config pyproject.toml --check tests/ - poetry run flake8 --config .flake8 tests/ - # poetry run mypy tests/ --config-file=pyproject.toml - Scope too big for now + poetry run ruff check tests/ utilities/ [testenv:fix-formatting] basepython = py312 commands = poetry install --no-root - poetry run black --config pyproject.toml tests/ - poetry run isort tests/ + poetry run ruff format tests/ utilities/ [testenv:mypy] basepython = py312 diff --git a/utilities/CIRCLECI_APKS.md b/utilities/CIRCLECI_APKS.md new file mode 100644 index 00000000..cc9a7b56 --- /dev/null +++ b/utilities/CIRCLECI_APKS.md @@ -0,0 +1,161 @@ +# CircleCI APK Fetcher + +Scripts to automatically find and download APK artifacts from CircleCI jobs in the mozilla/experimenter repository. + +## Files + +- **get_circleci_apks.py** - Python script that uses CircleCI API to find and download APKs +- **fetch_circleci_apks.sh** - Bash wrapper for easy CI usage + +## Prerequisites + +- Python 3.6+ +- `requests` library (`pip install requests`) +- CircleCI API token (optional for public repos, required for private repos) + +## Usage + +### Python Script + +Basic usage (defaults to mozilla/experimenter, main branch): +```bash +python utilities/get_circleci_apks.py +``` + +With custom options: +```bash +python utilities/get_circleci_apks.py \ + --branch main \ + --job-name build-fenix \ + --output-dir ./apks +``` + +All options: +```bash +python utilities/get_circleci_apks.py \ + --org mozilla \ + --repo experimenter \ + --branch main \ + --workflow-name "Build" \ + --job-name "build-fenix" \ + --output-dir ./apks \ + --token YOUR_CIRCLECI_TOKEN +``` + +### Bash Script + +The bash script provides a simpler interface using environment variables: + +```bash +# Basic usage +./utilities/fetch_circleci_apks.sh + +# With environment variables +CIRCLECI_BRANCH=main \ +CIRCLECI_OUTPUT_DIR=./apks \ +CIRCLECI_JOB_NAME=build-fenix \ +./utilities/fetch_circleci_apks.sh +``` + +### GitHub Actions Integration + +Add this step to your GitHub Actions workflow: + +```yaml +- name: Fetch APKs from CircleCI + env: + CIRCLECI_TOKEN: ${{ secrets.CIRCLECI_TOKEN }} # Optional + CIRCLECI_BRANCH: main + CIRCLECI_OUTPUT_DIR: ${{ github.workspace }}/klaatu + run: | + ./utilities/fetch_circleci_apks.sh +``` + +Or use the Python script directly: + +```yaml +- name: Install Python dependencies + run: pip install requests + +- name: Fetch APKs from CircleCI + env: + CIRCLECI_TOKEN: ${{ secrets.CIRCLECI_TOKEN }} # Optional + run: | + python utilities/get_circleci_apks.py \ + --branch main \ + --output-dir ${{ github.workspace }}/klaatu \ + --job-name build-fenix +``` + +### Example for Android Workflow + +Here's how to integrate it into the android_manual.yml workflow: + +```yaml +- name: Fetch Fenix APKs from Experimenter CircleCI + env: + CIRCLECI_TOKEN: ${{ secrets.CIRCLECI_TOKEN }} + CIRCLECI_BRANCH: main + CIRCLECI_OUTPUT_DIR: ${{ github.workspace }}/klaatu + working-directory: klaatu + run: | + pip install requests + python utilities/get_circleci_apks.py --job-name fenix + +- name: Install APKs + working-directory: klaatu + run: | + adb install *.apk +``` + +## Environment Variables + +The bash script supports these environment variables: + +- `CIRCLECI_TOKEN` - CircleCI API token (optional for public repos) +- `CIRCLECI_ORG` - GitHub organization (default: mozilla) +- `CIRCLECI_REPO` - GitHub repository (default: experimenter) +- `CIRCLECI_BRANCH` - Branch name (default: main) +- `CIRCLECI_OUTPUT_DIR` - Output directory (default: current directory) +- `CIRCLECI_WORKFLOW_NAME` - Filter by workflow name (optional) +- `CIRCLECI_JOB_NAME` - Filter by job name (optional) + +## How It Works + +1. Fetches recent pipelines from the specified repository and branch +2. For each pipeline, gets all workflows +3. For each successful workflow, gets all jobs +4. For each successful job, checks for artifacts +5. Downloads any APK artifacts found +6. Stops after downloading from the first successful job with APKs + +This ensures you always get the most recent successful build's APKs without any manual intervention. + +## CircleCI API Token + +To access private repositories or increase rate limits, you'll need a CircleCI API token: + +1. Go to https://app.circleci.com/settings/user/tokens +2. Create a new Personal API Token +3. Set it as `CIRCLECI_TOKEN` environment variable or use `--token` flag + +For GitHub Actions, add it as a repository secret: +- Go to repository Settings → Secrets and variables → Actions +- Add new secret named `CIRCLECI_TOKEN` + +## Troubleshooting + +**No APKs found:** +- Check that the branch name is correct +- Verify there are recent successful builds with APK artifacts +- Try without job/workflow filters first +- Check if you need a CircleCI token for the repository + +**Download fails:** +- Ensure you have the `requests` library installed +- Check your CircleCI token has proper permissions +- Verify the artifact URLs are accessible + +**Rate limiting:** +- Add a CircleCI API token to increase rate limits +- The script checks up to 20 recent pipelines by default diff --git a/utilities/WORKFLOW_EXAMPLE.yml b/utilities/WORKFLOW_EXAMPLE.yml new file mode 100644 index 00000000..f2571a6c --- /dev/null +++ b/utilities/WORKFLOW_EXAMPLE.yml @@ -0,0 +1,88 @@ +# Example: How to integrate CircleCI APK fetching into your GitHub Actions workflow +# +# This shows how to replace the Docker build step with fetching pre-built APKs +# from CircleCI mozilla/experimenter builds + +# Add this step after "Setup Python" and before "Run Tests" + + - name: Install Python dependencies for APK fetcher + run: pip install requests + + - name: Fetch Fenix APKs from CircleCI + env: + # Optional: Add CIRCLECI_TOKEN to your GitHub secrets for private repos + CIRCLECI_TOKEN: ${{ secrets.CIRCLECI_TOKEN }} + working-directory: klaatu + run: | + # Fetch APKs from mozilla/experimenter CircleCI builds + python utilities/get_circleci_apks.py \ + --org mozilla \ + --repo experimenter \ + --branch main \ + --output-dir . \ + --job-name fenix + + # List downloaded files + echo "Downloaded APKs:" + ls -lh *.apk + + # Then you can install and run tests as normal + - name: Run Tests + run: | + cd klaatu + + # Install the APKs (adjust filenames as needed based on what CircleCI produces) + adb install *androidTest*.apk + adb install *x86_64*.apk + + # Continue with your test commands... + cd ../firefox/mobile/android/fenix/app/src/androidTest/java/org/mozilla/fenix/experimentintegration + pip3 install virtualenv poetry pipenv + pipenv install + pipenv install pytest-rerunfailures + pipenv run python generate_smoke_tests.py + pipenv run pytest --experiment-slug ${{ inputs.slug }} ... + + +# Alternative: Use the bash wrapper script +# This is simpler and uses environment variables + + - name: Fetch Fenix APKs from CircleCI (using bash script) + env: + CIRCLECI_TOKEN: ${{ secrets.CIRCLECI_TOKEN }} + CIRCLECI_BRANCH: update_firefox_versions + CIRCLECI_OUTPUT_DIR: ${{ github.workspace }}/klaatu + CIRCLECI_JOB_NAME: Build Fenix APKs + working-directory: klaatu + run: ./utilities/fetch_circleci_apks.sh + + +# Alternative: Conditional fetching - use CircleCI if available, fallback to Docker build + + - name: Try to fetch APKs from CircleCI + id: fetch_circleci + continue-on-error: true + env: + CIRCLECI_TOKEN: ${{ secrets.CIRCLECI_TOKEN }} + working-directory: klaatu + run: | + pip install requests + python utilities/get_circleci_apks.py \ + --branch main \ + --output-dir . \ + --job-name fenix + + - name: Build APKs with Docker (if CircleCI fetch failed) + if: steps.fetch_circleci.outcome != 'success' + working-directory: klaatu + run: | + docker build -t fenix-builder -f android-build.Dockerfile . + docker run -d --name fenix-builder fenix-builder + docker cp fenix-builder:mozilla-central/mobile/android/fenix/app-fenix-debug-androidTest.apk ./ + docker cp fenix-builder:mozilla-central/mobile/android/fenix/app-fenix-x86_64-debug.apk ./ + + - name: Install APKs + working-directory: klaatu + run: | + adb install *androidTest*.apk + adb install *x86_64*.apk diff --git a/utilities/check_experimenter_and_start_jobs.py b/utilities/check_experimenter_and_start_jobs.py index 41d70003..d3328bb3 100644 --- a/utilities/check_experimenter_and_start_jobs.py +++ b/utilities/check_experimenter_and_start_jobs.py @@ -12,51 +12,50 @@ import requests -experimenter_url = "https://experimenter.services.mozilla.com/api/v6/experiments/?=status=Preview" +experimenter_url = ( + "https://experimenter.services.mozilla.com/api/v6/experiments/?=status=Preview" +) run_flag = False testing_list = {} path = Path().cwd() versions = requests.get("https://whattrainisitnow.com/api/firefox/releases/").json() + def trigger_github_action(slug, branch, firefox_version, workflow_id): - url = f'https://api.github.com/repos/jrbenny35/klaatu/actions/workflows/{workflow_id}/dispatches' - inputs = { - 'slug': slug, - 'branch': branch, - 'firefox-version': f"{firefox_version}" - } + url = f"https://api.github.com/repos/jrbenny35/klaatu/actions/workflows/{workflow_id}/dispatches" + inputs = {"slug": slug, "branch": branch, "firefox-version": f"{firefox_version}"} headers = { - 'Accept': 'application/vnd.github.v3+json', - 'Authorization': f"Bearer {os.getenv('BEARER_TOKEN')}" , - 'X-GitHub-Api-Version': '2022-11-28' - } - - data = { - 'ref': 'main', - 'inputs': inputs or {} + "Accept": "application/vnd.github.v3+json", + "Authorization": f"Bearer {os.getenv('BEARER_TOKEN')}", + "X-GitHub-Api-Version": "2022-11-28", } - print(f"Running tests for {inputs['slug']}, with data {data}, on workflow {workflow_id}") + + data = {"ref": "main", "inputs": inputs or {}} + print( + f"Running tests for {inputs['slug']}, with data {data}, on workflow {workflow_id}" + ) response = requests.post(url, headers=headers, data=json.dumps(data)) - + if response.status_code == 204: - print('Workflow triggered successfully!') + print("Workflow triggered successfully!") else: - print(f'Failed to trigger workflow: {response.status_code}') + print(f"Failed to trigger workflow: {response.status_code}") print(response.text) + def get_latest_versions(versions, min_version): # Parse versions and group by major.minor version_list = [] from packaging.version import Version - + for version in versions.keys(): if Version(version) >= Version(min_version[0]): - version_list.append(version) + version_list.append(version) version_groups = defaultdict(list) for version_str in version_list: @@ -64,27 +63,31 @@ def get_latest_versions(versions, min_version): major_minor = f"{version.major}.{version.minor}" version_groups[major_minor].append(version) - # Determine the latest version in each group latest_versions = {} for major_minor, versions in version_groups.items(): - latest_versions[major_minor] = max(versions, key=lambda v: (v.major, v.minor, v.micro)) - + latest_versions[major_minor] = max( + versions, key=lambda v: (v.major, v.minor, v.micro) + ) + # Return the latest versions sorted by major.minor return [f"{version}" for version in sorted(latest_versions.values())] + def get_firefox_verions(app_name, channel, min_version): test_versions = set() - non_desktop_beta = [f"{Version(list(versions.keys())[-1]).major +1}.0b"] - + non_desktop_beta = [f"{Version(list(versions.keys())[-1]).major + 1}.0b"] + if "firefox_ios" in app_name: # Get list of versions from requested to current based on whattrainisitnow for version in reversed(versions.keys()): version = Version(version) if version.major > Version(min_version).major: test_versions.add(version.major) - if not test_versions: # if the version doesn't exist in whattrainisitnow just return it - return [f"{Version(min_version)}"] + if ( + not test_versions + ): # if the version doesn't exist in whattrainisitnow just return it + return [f"{Version(min_version)}"] else: return [f"{_}" for _ in test_versions if _ >= 128] else: @@ -92,7 +95,7 @@ def get_firefox_verions(app_name, channel, min_version): case "release": test_versions = get_latest_versions(versions, min_version) if "desktop" in app_name: - test_versions.extend(['latest', 'latest-beta']) + test_versions.extend(["latest", "latest-beta"]) return test_versions case "nightly": return "['latest']" @@ -101,9 +104,10 @@ def get_firefox_verions(app_name, channel, min_version): return "['latest-beta']" return non_desktop_beta + # Load string of last experiment try: - with open('previous_experiment.txt') as f: + with open("previous_experiment.txt") as f: previous_experiment = f.read() except FileNotFoundError: subprocess.run([f"touch {path}"], encoding="utf8", shell=True) @@ -121,13 +125,15 @@ def get_firefox_verions(app_name, channel, min_version): exit # Sorted list of experiments that have a publishedDate field -experiments = [_ for _ in current_experiments if _['publishedDate'] is not None] -experiments = sorted(experiments, key=lambda _: parser.isoparse(_.get('publishedDate'))) +experiments = [_ for _ in current_experiments if _["publishedDate"] is not None] +experiments = sorted(experiments, key=lambda _: parser.isoparse(_.get("publishedDate"))) current_experiments = [] for experiment in experiments: try: - if parser.parse(experiment.get("startDate")) >= datetime.now() - timedelta(days=7): + if parser.parse(experiment.get("startDate")) >= datetime.now() - timedelta( + days=7 + ): current_experiments.append(experiment) except TypeError: continue @@ -135,7 +141,7 @@ def get_firefox_verions(app_name, channel, min_version): temp_experiments_list = [] for count, item in enumerate(current_experiments): - if not item.get('isRollout'): + if not item.get("isRollout"): temp_experiments_list.append(item) current_experiments = temp_experiments_list @@ -154,32 +160,45 @@ def get_firefox_verions(app_name, channel, min_version): desktop_workflows = ["windows_manual.yml", "linux_manual.yml"] try: - ff_version = [re.search(r"versionCompare\('(\d+).!'\)", data["targeting"]).group(1)] + ff_version = [ + re.search(r"versionCompare\('(\d+).!'\)", data["targeting"]).group(1) + ] except AttributeError: continue # Don't test experiments with no target version - branches = f"{[item["slug"] for item in data["branches"]]}" + branches = f"{[item['slug'] for item in data['branches']]}" match data["appName"]: case "firefox_desktop": for workflow_id in desktop_workflows: trigger_github_action( - slug, branches, get_firefox_verions(data["appName"], data["channel"], ff_version), workflow_id + slug, + branches, + get_firefox_verions(data["appName"], data["channel"], ff_version), + workflow_id, ) case "firefox_ios": workflow_id = "ios_manual.yml" _ff_version = Version(ff_version[0]) trigger_github_action( - slug, branches, get_firefox_verions(data["appName"], data["channel"], f"{_ff_version.major}"), workflow_id + slug, + branches, + get_firefox_verions( + data["appName"], data["channel"], f"{_ff_version.major}" + ), + workflow_id, ) case "fenix": workflow_id = "android_manual.yml" _ff_version = Version(ff_version[0]) trigger_github_action( - slug, branches, get_firefox_verions(data["appName"], data["channel"], ff_version), workflow_id + slug, + branches, + get_firefox_verions(data["appName"], data["channel"], ff_version), + workflow_id, ) time.sleep(30) # Write last experiment to file for next cron run -with open('previous_experiment.txt', 'w') as f: +with open("previous_experiment.txt", "w") as f: f.writelines(experiments[-1]["slug"]) print(f"Last experiment checked was {experiments[-1]['slug']}") diff --git a/utilities/download_old_firefox.py b/utilities/download_old_firefox.py index 7bcabf6a..f99b85a8 100644 --- a/utilities/download_old_firefox.py +++ b/utilities/download_old_firefox.py @@ -13,7 +13,7 @@ _date = { "year": download_date.strftime("%Y"), "month": download_date.strftime("%m"), - "day": download_date.strftime("%d") + "day": download_date.strftime("%d"), } download_dir = f"{base_url}/pub/firefox/nightly/{_date['year']}/{_date['month']}/" @@ -27,11 +27,11 @@ ) page_link = page_link[1] - html = requests.get(f'{base_url}{page_link["href"]}') + html = requests.get(f"{base_url}{page_link['href']}") soup = BeautifulSoup(html.text, "html.parser") - download_link = soup.find(href=re.compile(f"firefox-.*.en-US.linux-x86_64.tar.xz")) - complete_download_url = f'{base_url}{download_link["href"]}' + download_link = soup.find(href=re.compile("firefox-.*.en-US.linux-x86_64.tar.xz")) + complete_download_url = f"{base_url}{download_link['href']}" # print to stdout for wget or curl print(complete_download_url) diff --git a/utilities/download_release_firefox.py b/utilities/download_release_firefox.py index 76026eff..3f7c1a85 100644 --- a/utilities/download_release_firefox.py +++ b/utilities/download_release_firefox.py @@ -1,6 +1,3 @@ -from datetime import date -import re - import requests from bs4 import BeautifulSoup diff --git a/utilities/get_android_apks.py b/utilities/get_android_apks.py index 194d73db..6614f0ee 100644 --- a/utilities/get_android_apks.py +++ b/utilities/get_android_apks.py @@ -8,13 +8,13 @@ parser = argparse.ArgumentParser("Options for android apk downloader") -parser.add_argument('--firefox-version', help="The firefox version to download") +parser.add_argument("--firefox-version", help="The firefox version to download") args = parser.parse_args() headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', - 'Accept': 'application/json', - 'Content-Type': 'application/json', + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Accept": "application/json", + "Content-Type": "application/json", } jobs_filter = ["signing-apk-fenix-debug", "signing-apk-fenix-android-test-debug"] download_ids = {} @@ -25,9 +25,9 @@ firefox_version = args.firefox_version.replace(".", "_") hg_url = requests.get("https://hg.mozilla.org/releases/mozilla-release/tags").content -soup = BeautifulSoup(hg_url, 'html.parser') +soup = BeautifulSoup(hg_url, "html.parser") -versions = soup.find_all('b', string=re.compile("FIREFOX-ANDROID")) +versions = soup.find_all("b", string=re.compile("FIREFOX-ANDROID")) beta_versions = sorted([_.get_text() for _ in versions if "b" in _.get_text()]) release_versions = sorted([_.get_text() for _ in versions if "b" not in _.get_text()]) @@ -49,19 +49,27 @@ print(f"https://hg.mozilla.org/releases/mozilla-release/rev/{final_version}") -hg_url = requests.get(f"https://hg.mozilla.org/releases/mozilla-release/rev/{final_version}").content +hg_url = requests.get( + f"https://hg.mozilla.org/releases/mozilla-release/rev/{final_version}" +).content -soup = BeautifulSoup(hg_url, 'html.parser') +soup = BeautifulSoup(hg_url, "html.parser") -treeherder_link = soup.find("a", string='default view') -treeherder_link = treeherder_link.get('href') +treeherder_link = soup.find("a", string="default view") +treeherder_link = treeherder_link.get("href") -revision = treeherder_link.split('=')[-1] +revision = treeherder_link.split("=")[-1] -result_id_json = requests.get(f"https://treeherder.mozilla.org/api/project/mozilla-release/push/?full=true&count=10&revision={revision}", headers=headers).json() +result_id_json = requests.get( + f"https://treeherder.mozilla.org/api/project/mozilla-release/push/?full=true&count=10&revision={revision}", + headers=headers, +).json() result_id = result_id_json.get("results")[0].get("id") -fenix_jobs = requests.get("https://treeherder.mozilla.org/api/project/mozilla-release/jobs/?job_group_symbol=fenix-debug&count=1000", headers=headers).json() +fenix_jobs = requests.get( + "https://treeherder.mozilla.org/api/project/mozilla-release/jobs/?job_group_symbol=fenix-debug&count=1000", + headers=headers, +).json() for item in fenix_jobs["results"]: for job in jobs_filter: @@ -73,19 +81,24 @@ exit for job, id in download_ids.items(): - data = requests.get(f"https://treeherder.mozilla.org/api/project/mozilla-release/jobs/{id}/", headers=headers).json() + data = requests.get( + f"https://treeherder.mozilla.org/api/project/mozilla-release/jobs/{id}/", + headers=headers, + ).json() task_ids[job] = data.get("task_id") test_apk = requests.get( f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_ids.get('signing-apk-fenix-android-test-debug')}/runs/0/artifacts/public/build/target.noarch.apk", - headers=headers + headers=headers, ) fenix_apk = requests.get( f"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_ids.get('signing-apk-fenix-debug')}/runs/0/artifacts/public/build/target.x86_64.apk", - headers=headers + headers=headers, ) -with open(path.resolve() / f"android-debug-test-v{args.firefox_version}.apk", "wb") as file: +with open( + path.resolve() / f"android-debug-test-v{args.firefox_version}.apk", "wb" +) as file: file.write(test_apk.content) with open(path.resolve() / f"fenix-debug-v{args.firefox_version}.apk", "wb") as file: file.write(fenix_apk.content) diff --git a/utilities/get_circleci_apks.py b/utilities/get_circleci_apks.py index d8951928..7e9c4091 100644 --- a/utilities/get_circleci_apks.py +++ b/utilities/get_circleci_apks.py @@ -45,11 +45,7 @@ def __init__(self, token: Optional[str] = None): self.headers["Circle-Token"] = self.token def get_recent_pipelines( - self, - org: str, - repo: str, - branch: Optional[str] = None, - limit: int = 20 + self, org: str, repo: str, branch: Optional[str] = None, limit: int = 20 ) -> List[Dict[str, Any]]: """Get recent pipelines for a project.""" url = f"{self.BASE_URL}/project/github/{org}/{repo}/pipeline" @@ -81,7 +77,9 @@ def get_workflow_jobs(self, workflow_id: str) -> List[Dict[str, Any]]: data = response.json() return data.get("items", []) - def get_job_artifacts(self, job_number: int, org: str, repo: str) -> List[Dict[str, Any]]: + def get_job_artifacts( + self, job_number: int, org: str, repo: str + ) -> List[Dict[str, Any]]: """Get artifacts for a specific job.""" url = f"{self.BASE_URL}/project/github/{org}/{repo}/{job_number}/artifacts" response = requests.get(url, headers=self.headers) @@ -148,7 +146,9 @@ def find_and_download_apks( continue if workflow_status != "success": - print(f" Skipping workflow '{workflow_name_actual}' (status: {workflow_status})") + print( + f" Skipping workflow '{workflow_name_actual}' (status: {workflow_status})" + ) continue print(f" Checking workflow: {workflow_name_actual}") @@ -172,8 +172,7 @@ def find_and_download_apks( artifacts = self.get_job_artifacts(job_number, org, repo) apk_artifacts = [ - a for a in artifacts - if a.get("path", "").endswith(".apk") + a for a in artifacts if a.get("path", "").endswith(".apk") ] if apk_artifacts: @@ -198,13 +197,15 @@ def find_and_download_apks( # If we downloaded APKs, we're done if downloaded_count > 0: - print(f"\n{'='*80}") - print(f"Successfully downloaded {downloaded_count} APK(s) from:") + print(f"\n{'=' * 80}") + print( + f"Successfully downloaded {downloaded_count} APK(s) from:" + ) print(f"Pipeline #{pipeline_number}") print(f"Workflow: {workflow_name_actual}") print(f"Job: {job_name_actual}") print(f"Saved to: {output_dir.resolve()}") - print(f"{'='*80}") + print(f"{'=' * 80}") return downloaded_count except Exception as e: @@ -223,37 +224,28 @@ def main(): description="Automatically find and download APK artifacts from CircleCI" ) parser.add_argument( - "--org", - default="mozilla", - help="GitHub organization name (default: mozilla)" + "--org", default="mozilla", help="GitHub organization name (default: mozilla)" ) parser.add_argument( "--repo", default="experimenter", - help="GitHub repository name (default: experimenter)" - ) - parser.add_argument( - "--branch", - default="main", - help="Branch name (default: main)" + help="GitHub repository name (default: experimenter)", ) + parser.add_argument("--branch", default="main", help="Branch name (default: main)") parser.add_argument( - "--workflow-name", - help="Workflow name filter (optional, matches substring)" + "--workflow-name", help="Workflow name filter (optional, matches substring)" ) parser.add_argument( - "--job-name", - help="Job name filter (optional, matches substring)" + "--job-name", help="Job name filter (optional, matches substring)" ) parser.add_argument( "--output-dir", type=Path, default=Path.cwd(), - help="Directory to save downloaded APKs (default: current directory)" + help="Directory to save downloaded APKs (default: current directory)", ) parser.add_argument( - "--token", - help="CircleCI API token (or set CIRCLECI_TOKEN env var)" + "--token", help="CircleCI API token (or set CIRCLECI_TOKEN env var)" ) args = parser.parse_args()