From 7eda54cfab880dc1e21439d4795916221708185a Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Mon, 10 Nov 2025 17:39:45 -0800 Subject: [PATCH 01/29] read manifests in chooser script --- choose_ci_set.py | 116 ++++++++++++++++++++--------- modules/page_object_about_pages.py | 4 +- 2 files changed, 85 insertions(+), 35 deletions(-) diff --git a/choose_ci_set.py b/choose_ci_set.py index 01e67cf36..7776b2aed 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -1,24 +1,18 @@ import os +import platform import re import sys from subprocess import check_output -import pytest +import yaml -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +CI_MANIFEST = "manifests/ci.yaml" CI_MARK = "@pytest.mark.ci" HEADED_MARK = "@pytest.mark.headed" +MIN_RUN_SIZE = 7 OUTPUT_FILE = "selected_tests" - - -class CollectionPlugin: - """Mini plugin to get test names""" - - def __init__(self): - self.tests = [] - - def pytest_report_collectionfinish(self, items): - self.tests = [item.nodeid for item in items] +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +SLASH = "/" if "/" in SCRIPT_DIR else "\\" def snakify(pascal: str) -> str: @@ -72,7 +66,7 @@ def get_tests_by_model( return matching_tests -def dedupe(run_list: list, slash: str) -> list: +def dedupe(run_list: list) -> list: """For a run list, remove entries that are covered by more general entries.""" run_list = list(set(run_list)) dotslashes = [] @@ -87,7 +81,7 @@ def dedupe(run_list: list, slash: str) -> list: dotslashes.append(i) for dotslash in dotslashes: - run_list[dotslash] = f".{slash}{run_list[dotslash]}" + run_list[dotslash] = f".{SLASH}{run_list[dotslash]}" for i, entry_a in enumerate(run_list): for j, entry_b in enumerate(run_list): @@ -105,6 +99,59 @@ def dedupe(run_list: list, slash: str) -> list: return run_list +def sysname(): + sys_platform = platform.system().lower() + if sys_platform.startswith("darwin"): + return "mac" + elif sys_platform.startswith("win"): + return "win" + elif sys_platform.startswith("linux"): + return "linux" + raise OSError("Unsupported system.") + + +def convert_manifest_to_list(manifest_loc): + manifest = yaml.safe_load(open(manifest_loc)) + toplevel = [".", "tests"] + tests = [] + for suite in manifest: + print(suite) + if suite == "add_manifest": + for linked_manifest in manifest[suite]: + tests.extend( + convert_manifest_to_list(SLASH.join(["manifests", linked_manifest])) + ) + for test in manifest[suite]: + print(f" {test}") + addtest = False + test_name = f"{test}.py" + if manifest[suite][test] == "pass": + addtest = True + elif isinstance(manifest[suite][test], dict): + print(f"==={manifest[suite][test].get(sysname())}===") + if manifest[suite][test].get(sysname()) == "pass": + addtest = True + else: + for subtest in manifest[suite][test]: + if subtest in ["mac", "win", "linux"]: + continue + print(f" {subtest}") + test_name = f"{test}.py::{subtest}" + if isinstance(manifest[suite][test][subtest], dict): + if manifest[suite][test][subtest].get(sysname()) == "pass": + addtest = True + elif manifest[suite][test][subtest] == "pass": + addtest = True + if addtest: + test_to_add = SLASH.join(toplevel + [suite, test_name]) + assert os.path.exists(test_to_add.split("::")[0]), ( + f"{test_to_add} could not be found" + ) + tests.append(test_to_add) + addtest = False + return tests + + if __name__ == "__main__": if os.path.exists(".env"): with open(".env") as fh: @@ -120,7 +167,12 @@ def dedupe(run_list: list, slash: str) -> list: fh.write("tests") sys.exit(0) - slash = "/" if "/" in SCRIPT_DIR else "\\" + if os.environ.get("STARFOX_MANIFEST"): + run_list = convert_manifest_to_list(os.environ["STARFOX_MANIFEST"]) + run_list = dedupe(run_list) + with open(OUTPUT_FILE, "w") as fh: + fh.write("\n".join(run_list)) + sys.exit(0) re_obj = { "test_re_string": r".*/.*/test_.*\.py", @@ -130,7 +182,7 @@ def dedupe(run_list: list, slash: str) -> list: "class_re_string": r"\s*class (\w+)[(A-Za-z0-9_)]*:", } for k in list(re_obj.keys()): - if slash == "\\": + if SLASH == "\\": re_obj[k] = re_obj.get(k).replace("/", r"\\") short_name = "_".join(k.split("_")[:-1]) re_obj[short_name] = re.compile(re_obj.get(k)) @@ -140,7 +192,7 @@ def dedupe(run_list: list, slash: str) -> list: committed_files = ( check_output(["git", "--no-pager", "diff", "--name-only", "origin/main"]) .decode() - .replace("/", slash) + .replace("/", SLASH) .splitlines() ) @@ -164,13 +216,6 @@ def dedupe(run_list: list, slash: str) -> list: lines = fh.readlines() test_paths_and_contents[this_file] = "".join(lines) - p = CollectionPlugin() - pytest.main(["--collect-only", "-m", "ci", "-s"], plugins=[p]) - ci_paths = [f".{slash}{test}" for test in p.tests] - - # Dedupe just in case - ci_paths = list(set(ci_paths)) - changed_suite_conftests = [ f for f in committed_files if re_obj.get("suite_conftest_re").match(f) ] @@ -184,7 +229,7 @@ def dedupe(run_list: list, slash: str) -> list: if changed_suite_conftests: run_list = [ - "." + slash + os.path.join(*suite.split(slash)[-3:-1]) + "." + SLASH + os.path.join(*suite.split(SLASH)[-3:-1]) for suite in changed_suite_conftests ] @@ -212,7 +257,7 @@ def dedupe(run_list: list, slash: str) -> list: found = False for file in run_list: # Don't add if already exists in suite changes - pieces = file.split(slash) + pieces = file.split(SLASH) if len(pieces) == 3 and pieces[-1] in changed_test: found = True @@ -223,15 +268,18 @@ def dedupe(run_list: list, slash: str) -> list: run_list.append(changed_test) if not run_list: + ci_paths = convert_manifest_to_list(CI_MANIFEST) with open(OUTPUT_FILE, "w") as fh: fh.write("\n".join(ci_paths)) - else: + sys.exit(0) + + if len(run_list) < MIN_RUN_SIZE: run_list.extend(ci_paths) - # Dedupe just in case - if slash == "\\": - run_list = [entry.replace("/", slash) for entry in run_list] - run_list = dedupe(run_list, slash) - run_list = [entry for entry in run_list if os.path.exists(entry.split("::")[0])] - with open(OUTPUT_FILE, "w") as fh: - fh.write("\n".join(run_list)) + # Dedupe just in case + if SLASH == "\\": + run_list = [entry.replace("/", SLASH) for entry in run_list] + run_list = dedupe(run_list) + run_list = [entry for entry in run_list if os.path.exists(entry.split("::")[0])] + with open(OUTPUT_FILE, "w") as fh: + fh.write("\n".join(run_list)) diff --git a/modules/page_object_about_pages.py b/modules/page_object_about_pages.py index 8a98fa5b2..95224294e 100644 --- a/modules/page_object_about_pages.py +++ b/modules/page_object_about_pages.py @@ -195,7 +195,9 @@ def remove_password_csv(self, downloads_dir): if delete_files_regex.match(file): os.remove(passwords_csv) - def verify_csv_export(self, downloads_folder: str, filename: str, timeout: int = 20): + def verify_csv_export( + self, downloads_folder: str, filename: str, timeout: int = 20 + ): """ Wait until the exported CSV file is present, non-empty, and readable. """ From 84f6f57fa9f957ea0660308703ca9f4ea01fe7a6 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Wed, 12 Nov 2025 11:50:33 -0800 Subject: [PATCH 02/29] update CI beta flows to take manifests --- .github/workflows/smoke.yml | 6 ++ manifests/ci.yaml | 54 ++++++++++ manifests/incident.yaml | 194 ++++++++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 manifests/ci.yaml create mode 100644 manifests/incident.yaml diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index ec0da223b..2ba824bcb 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -93,6 +93,10 @@ jobs: - name: Download Executable if: ${{ inputs.win_installer_link }} run: | + $env_contents = @" + STARFOX_MANIFEST='.\manifests\incident.yaml' + "@ + New-Item -Name .env -Value $env_contents -ItemType File -Force Invoke-WebRequest -Uri ${{ inputs.win_installer_link }} -OutFile "${{ github.workspace }}\setup.exe" New-Item -ItemType Directory -Path "C:\Program Files\Custom Firefox" -Force shell: pwsh @@ -165,6 +169,7 @@ jobs: if: ${{ inputs.mac_installer_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; + echo "STARFOX_MANIFEST='manifest/incident.yaml'" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; - name: Install dependencies run: | @@ -228,6 +233,7 @@ jobs: MANUAL_DOWNLOAD_LINK: ${{ inputs.linux_tarball_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; + echo "STARFOX_MANIFEST='manifest/incident.yaml'" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; sudo apt install gnome-screenshot uname -m; diff --git a/manifests/ci.yaml b/manifests/ci.yaml new file mode 100644 index 000000000..920179845 --- /dev/null +++ b/manifests/ci.yaml @@ -0,0 +1,54 @@ +address_bar_and_search: + test_default_search_provider_change_awesome_bar: + test_default_search_provider_change_awesome_bar: pass +bookmarks_and_history: + test_bookmark_website_via_star_button: + test_bookmark_website_via_star: pass +downloads: + test_set_always_ask_file_type: + test_set_always_ask_file_type: pass +find_toolbar: + test_find_toolbar_search: + test_find_toolbar_search: pass +form_autofill: + test_telephone_autofill_attribute: + test_telephone_attribute_autofill: pass +menus: + test_copy_paste_actions: + test_login_form_copy_paste: pass + test_image_context_menu_actions: + test_save_image_as: pass +networking: + test_http_site: + test_http_site: pass +notifications: + test_notifications_displayed: + test_notifications_displayed: pass +password_manager: + test_about_logins_navigation_from_context_menu: + test_about_logins_navigation_from_login_form_context_menu: pass +pdf_viewer: + test_pdf_data_can_be_cleared: + test_pdf_data_can_be_cleared: pass +printing_ui: + test_print_preview: + test_print_preview_keys: pass +reader_view: + test_improved_type_control_panel: + test_type_control_panel_font[sans-serif]: pass + test_improved_type_control_panel: + test_type_control_panel_font[serif]: pass + test_improved_type_control_panel: + test_type_control_panel_font[monospace]: pass +scrolling_panning_zooming: + test_zoom_text_only: + test_zoom_text_only_from_settings: pass +security_and_privacy: + test_cross_site_tracking_cookies_blocked: + test_cross_site_tracking_cookies_blocked: pass +tabs: + test_active_tab: + test_active_tab: pass +theme_and_toolbar: + test_customize_themes_and_redirect: + test_redirect_to_addons: pass diff --git a/manifests/incident.yaml b/manifests/incident.yaml new file mode 100644 index 000000000..7a0291a6c --- /dev/null +++ b/manifests/incident.yaml @@ -0,0 +1,194 @@ +--- +address_bar_and_search: + test_add_engine_address_bar: pass + test_addon_suggestion: pass + test_default_search_provider_change_awesome_bar: pass + test_google_search_counts_us: + win: pass + linux: pass + mac: unstable + test_google_withads_url_bar_us: + win: pass + linux: pass + mac: unstable + test_sap_google_adclick: unstable + test_search_code_google_non_us: pass + test_search_code_google_us: pass + test_search_engine_selector: pass + test_search_suggestions: pass + test_search_term_persists: pass + test_server_not_found_error: pass + test_suggestion_engine_selection: pass +audio_video: + test_allow_audio_video_functionality: + win: unstable + mac: pass + linux: unstable + test_block_audio_video_functionality: pass + test_users_actions_saved_on_reload: pass +bookmarks_and_history: + test_add_new_other_bookmark: pass + test_bookmark_via_bookmark_menu: pass + test_bookmark_website_via_star_button: pass + test_clear_all_history: pass + test_clear_recent_history_displayed: pass + test_delete_bookmarks_from_toolbar: pass + test_delete_other_bookmarks: pass + test_deleted_page_not_remembered: pass + test_open_bookmark_in_new_window_via_toolbar_context_menu: pass + test_open_bookmark_in_private_window_via_toolbar_context_menu: pass + test_open_bookmarks_from_toolbar: pass + test_open_websites_from_history: pass + test_private_window_website_not_in_history: pass + test_toggle_bookmarks_toolbar: pass + test_user_can_forget_history: pass +downloads: + test_add_zip_type: pass + test_download_pdf: pass + test_download_pdf_from_context_menu: unstable +drag_and_drop: + test_copy_entire_row_column: pass +find_toolbar: + test_find_in_pdf: pass + test_find_toolbar_nav: pass + test_find_toolbar_search: pass +form_autofill: + test_address_autofill_attribute: pass + test_autofill_cc_cvv: pass + test_autofill_credit_card: pass + test_autofill_credit_card_doorhanger: pass + test_autofill_credit_card_enable: pass + test_autofill_credit_card_four_fields: pass + test_cc_clear_form: pass + test_clear_form: pass + test_create_new_cc_profile: pass + test_create_profile_autofill: pass + test_delete_cc_profile: pass + test_enable_disable_autofill: pass + test_form_autofill_suggestions: pass + test_name_autofill_attribute: pass + test_private_mode_info_not_saved: pass + test_telephone_autofill_attribute: pass + test_updating_address: pass +geolocation: + test_geolocation_shared_via_html5: + win: unstable + mac: pass + linux: pass + test_geolocation_shared_via_w3c_api: + win: unstable + mac: pass + linux: pass +language_packs: + test_language_pack_install_addons: pass + test_language_pack_install_preferences: pass +menus: + test_frequently_used_context_menu: + win: unstable + mac: pass + linux: pass + test_hyperlink_context_menu: pass + test_tab_context_menu_actions: pass + test_tab_context_menu_close: pass +networking: + test_default_dns_protection: pass + test_http_site: pass +notifications: + test_audio_video_permissions_notification: + win: pass + mac: unstable + linux: pass + test_camera_permissions_notification: + win: pass + mac: unstable + linux: pass + test_deny_geolocation: pass + test_deny_screen_capture: pass + test_geolocation_prompt_presence: pass + test_microphone_permissions_notification: + win: pass + mac: unstable + linux: pass + test_notifications_displayed: pass + test_screen_share_permission_prompt: pass + test_webextension_completed_installation_successfully_displayed: pass +password_manager: + test_add_password_non_ascii_chars: pass + test_add_password_save_valid_data: pass + test_auto_saved_generated_password_context_menu: pass + test_can_view_password_when_PP_enabled: pass + test_changes_made_in_edit_mode_are_saved: pass + test_delete_login: pass + test_multiple_saved_logins: pass + test_never_save_login_via_doorhanger: pass + test_primary_password_triggered_on_about_logins_access: pass + test_save_login_via_doorhanger: pass + test_update_login_via_doorhanger: pass +pdf_viewer: + test_download_pdf_with_form_fields: pass + test_download_triggered_on_content_disposition_attachment: pass + test_open_pdf_in_FF: pass + test_pdf_navigation: pass + test_zoom_pdf_viewer: pass +preferences: + test_check_for_updates: pass + test_clear_cookie_data: + win: unstable + mac: pass + linux: pass + test_firefox_home_new_tabs: pass + test_firefox_home_on_launch: pass + test_lang_pack_changed_from_about_prefs: pass + test_never_remember_history: pass + test_notifications_change_set: pass +printing_ui: + test_print_preview: pass +profile: + test_set_default_profile: pass +reader_view: + test_improved_type_control_panel: pass + test_reader_view_close_sidebar: pass + test_reader_view_location_bar: pass +scrolling_panning_zooming: + test_default_zoom_persists: pass + test_mouse_wheel_zoom: + win: pass + mac: unstable + linux: pass + test_zoom_text_only: pass +security_and_privacy: + test_blocking_cryptominers: pass + test_blocking_fingerprinters: pass + test_blocking_social_media_trackers: unstable + test_cookies_not_saved_private_browsing: pass + test_cross_site_tracking_cookies_blocked: pass + test_cryptominers_blocked_and_shown_in_info_panel: pass + test_detected_blocked_trackers_found: pass + test_downloads_from_private_not_leaked: pass + test_https_enabled_private_browsing: pass + test_never_remember_browsing_history: pass + test_no_trackers_detected: pass + test_open_link_in_private_window: pass + test_phishing_and_malware_warnings: pass + test_private_browser_password_doorhanger: pass + test_private_session_awesome_bar_exclusion: pass + test_private_session_history_exclusion: pass + test_private_window_from_panelui: pass + test_third_party_content_blocked_private_browsing: pass + test_tls_v1_2_protocol: pass + test_trackers_crypto_fingerprint_blocked: pass + test_tracking_content_custom_mode: pass + test_undo_close_tab_private_browsing: pass +tabs: + test_active_tab: pass + test_navigation_multiple_tabs: pass + test_open_bookmark_in_new_tab: pass + test_open_new_tab: pass + test_open_new_tab_keys: pass + test_open_new_tab_via_hyperlink: pass + test_pin_tab: pass + test_reopen_tab_through_context_menu: pass + test_reopen_tab_through_history_menu: pass + test_reopen_tabs_through_keys: pass +theme_and_toolbar: + test_customize_themes_and_redirect: pass From e75bdb028c16134f34bcb410abca7f775ac68479 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Wed, 12 Nov 2025 11:52:01 -0800 Subject: [PATCH 03/29] typo --- .github/workflows/smoke.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 2ba824bcb..5d16580c4 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -169,7 +169,7 @@ jobs: if: ${{ inputs.mac_installer_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; - echo "STARFOX_MANIFEST='manifest/incident.yaml'" >> "$GITHUB_ENV" + echo "STARFOX_MANIFEST='manifests/incident.yaml'" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; - name: Install dependencies run: | @@ -233,7 +233,7 @@ jobs: MANUAL_DOWNLOAD_LINK: ${{ inputs.linux_tarball_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; - echo "STARFOX_MANIFEST='manifest/incident.yaml'" >> "$GITHUB_ENV" + echo "STARFOX_MANIFEST='manifests/incident.yaml'" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; sudo apt install gnome-screenshot uname -m; From 77d1c440e5168fb6f47fb76593e6d4b606e6e4a8 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Mon, 10 Nov 2025 17:39:45 -0800 Subject: [PATCH 04/29] read manifests in chooser script --- choose_ci_set.py | 116 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 82 insertions(+), 34 deletions(-) diff --git a/choose_ci_set.py b/choose_ci_set.py index 01e67cf36..7776b2aed 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -1,24 +1,18 @@ import os +import platform import re import sys from subprocess import check_output -import pytest +import yaml -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +CI_MANIFEST = "manifests/ci.yaml" CI_MARK = "@pytest.mark.ci" HEADED_MARK = "@pytest.mark.headed" +MIN_RUN_SIZE = 7 OUTPUT_FILE = "selected_tests" - - -class CollectionPlugin: - """Mini plugin to get test names""" - - def __init__(self): - self.tests = [] - - def pytest_report_collectionfinish(self, items): - self.tests = [item.nodeid for item in items] +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +SLASH = "/" if "/" in SCRIPT_DIR else "\\" def snakify(pascal: str) -> str: @@ -72,7 +66,7 @@ def get_tests_by_model( return matching_tests -def dedupe(run_list: list, slash: str) -> list: +def dedupe(run_list: list) -> list: """For a run list, remove entries that are covered by more general entries.""" run_list = list(set(run_list)) dotslashes = [] @@ -87,7 +81,7 @@ def dedupe(run_list: list, slash: str) -> list: dotslashes.append(i) for dotslash in dotslashes: - run_list[dotslash] = f".{slash}{run_list[dotslash]}" + run_list[dotslash] = f".{SLASH}{run_list[dotslash]}" for i, entry_a in enumerate(run_list): for j, entry_b in enumerate(run_list): @@ -105,6 +99,59 @@ def dedupe(run_list: list, slash: str) -> list: return run_list +def sysname(): + sys_platform = platform.system().lower() + if sys_platform.startswith("darwin"): + return "mac" + elif sys_platform.startswith("win"): + return "win" + elif sys_platform.startswith("linux"): + return "linux" + raise OSError("Unsupported system.") + + +def convert_manifest_to_list(manifest_loc): + manifest = yaml.safe_load(open(manifest_loc)) + toplevel = [".", "tests"] + tests = [] + for suite in manifest: + print(suite) + if suite == "add_manifest": + for linked_manifest in manifest[suite]: + tests.extend( + convert_manifest_to_list(SLASH.join(["manifests", linked_manifest])) + ) + for test in manifest[suite]: + print(f" {test}") + addtest = False + test_name = f"{test}.py" + if manifest[suite][test] == "pass": + addtest = True + elif isinstance(manifest[suite][test], dict): + print(f"==={manifest[suite][test].get(sysname())}===") + if manifest[suite][test].get(sysname()) == "pass": + addtest = True + else: + for subtest in manifest[suite][test]: + if subtest in ["mac", "win", "linux"]: + continue + print(f" {subtest}") + test_name = f"{test}.py::{subtest}" + if isinstance(manifest[suite][test][subtest], dict): + if manifest[suite][test][subtest].get(sysname()) == "pass": + addtest = True + elif manifest[suite][test][subtest] == "pass": + addtest = True + if addtest: + test_to_add = SLASH.join(toplevel + [suite, test_name]) + assert os.path.exists(test_to_add.split("::")[0]), ( + f"{test_to_add} could not be found" + ) + tests.append(test_to_add) + addtest = False + return tests + + if __name__ == "__main__": if os.path.exists(".env"): with open(".env") as fh: @@ -120,7 +167,12 @@ def dedupe(run_list: list, slash: str) -> list: fh.write("tests") sys.exit(0) - slash = "/" if "/" in SCRIPT_DIR else "\\" + if os.environ.get("STARFOX_MANIFEST"): + run_list = convert_manifest_to_list(os.environ["STARFOX_MANIFEST"]) + run_list = dedupe(run_list) + with open(OUTPUT_FILE, "w") as fh: + fh.write("\n".join(run_list)) + sys.exit(0) re_obj = { "test_re_string": r".*/.*/test_.*\.py", @@ -130,7 +182,7 @@ def dedupe(run_list: list, slash: str) -> list: "class_re_string": r"\s*class (\w+)[(A-Za-z0-9_)]*:", } for k in list(re_obj.keys()): - if slash == "\\": + if SLASH == "\\": re_obj[k] = re_obj.get(k).replace("/", r"\\") short_name = "_".join(k.split("_")[:-1]) re_obj[short_name] = re.compile(re_obj.get(k)) @@ -140,7 +192,7 @@ def dedupe(run_list: list, slash: str) -> list: committed_files = ( check_output(["git", "--no-pager", "diff", "--name-only", "origin/main"]) .decode() - .replace("/", slash) + .replace("/", SLASH) .splitlines() ) @@ -164,13 +216,6 @@ def dedupe(run_list: list, slash: str) -> list: lines = fh.readlines() test_paths_and_contents[this_file] = "".join(lines) - p = CollectionPlugin() - pytest.main(["--collect-only", "-m", "ci", "-s"], plugins=[p]) - ci_paths = [f".{slash}{test}" for test in p.tests] - - # Dedupe just in case - ci_paths = list(set(ci_paths)) - changed_suite_conftests = [ f for f in committed_files if re_obj.get("suite_conftest_re").match(f) ] @@ -184,7 +229,7 @@ def dedupe(run_list: list, slash: str) -> list: if changed_suite_conftests: run_list = [ - "." + slash + os.path.join(*suite.split(slash)[-3:-1]) + "." + SLASH + os.path.join(*suite.split(SLASH)[-3:-1]) for suite in changed_suite_conftests ] @@ -212,7 +257,7 @@ def dedupe(run_list: list, slash: str) -> list: found = False for file in run_list: # Don't add if already exists in suite changes - pieces = file.split(slash) + pieces = file.split(SLASH) if len(pieces) == 3 and pieces[-1] in changed_test: found = True @@ -223,15 +268,18 @@ def dedupe(run_list: list, slash: str) -> list: run_list.append(changed_test) if not run_list: + ci_paths = convert_manifest_to_list(CI_MANIFEST) with open(OUTPUT_FILE, "w") as fh: fh.write("\n".join(ci_paths)) - else: + sys.exit(0) + + if len(run_list) < MIN_RUN_SIZE: run_list.extend(ci_paths) - # Dedupe just in case - if slash == "\\": - run_list = [entry.replace("/", slash) for entry in run_list] - run_list = dedupe(run_list, slash) - run_list = [entry for entry in run_list if os.path.exists(entry.split("::")[0])] - with open(OUTPUT_FILE, "w") as fh: - fh.write("\n".join(run_list)) + # Dedupe just in case + if SLASH == "\\": + run_list = [entry.replace("/", SLASH) for entry in run_list] + run_list = dedupe(run_list) + run_list = [entry for entry in run_list if os.path.exists(entry.split("::")[0])] + with open(OUTPUT_FILE, "w") as fh: + fh.write("\n".join(run_list)) From 46b3a7a7b16801e88bbc07a203fb8d769356f039 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Wed, 12 Nov 2025 11:50:33 -0800 Subject: [PATCH 05/29] update CI beta flows to take manifests --- .github/workflows/smoke.yml | 6 ++ manifests/ci.yaml | 54 ++++++++++ manifests/incident.yaml | 194 ++++++++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 manifests/ci.yaml create mode 100644 manifests/incident.yaml diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index ec0da223b..2ba824bcb 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -93,6 +93,10 @@ jobs: - name: Download Executable if: ${{ inputs.win_installer_link }} run: | + $env_contents = @" + STARFOX_MANIFEST='.\manifests\incident.yaml' + "@ + New-Item -Name .env -Value $env_contents -ItemType File -Force Invoke-WebRequest -Uri ${{ inputs.win_installer_link }} -OutFile "${{ github.workspace }}\setup.exe" New-Item -ItemType Directory -Path "C:\Program Files\Custom Firefox" -Force shell: pwsh @@ -165,6 +169,7 @@ jobs: if: ${{ inputs.mac_installer_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; + echo "STARFOX_MANIFEST='manifest/incident.yaml'" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; - name: Install dependencies run: | @@ -228,6 +233,7 @@ jobs: MANUAL_DOWNLOAD_LINK: ${{ inputs.linux_tarball_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; + echo "STARFOX_MANIFEST='manifest/incident.yaml'" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; sudo apt install gnome-screenshot uname -m; diff --git a/manifests/ci.yaml b/manifests/ci.yaml new file mode 100644 index 000000000..920179845 --- /dev/null +++ b/manifests/ci.yaml @@ -0,0 +1,54 @@ +address_bar_and_search: + test_default_search_provider_change_awesome_bar: + test_default_search_provider_change_awesome_bar: pass +bookmarks_and_history: + test_bookmark_website_via_star_button: + test_bookmark_website_via_star: pass +downloads: + test_set_always_ask_file_type: + test_set_always_ask_file_type: pass +find_toolbar: + test_find_toolbar_search: + test_find_toolbar_search: pass +form_autofill: + test_telephone_autofill_attribute: + test_telephone_attribute_autofill: pass +menus: + test_copy_paste_actions: + test_login_form_copy_paste: pass + test_image_context_menu_actions: + test_save_image_as: pass +networking: + test_http_site: + test_http_site: pass +notifications: + test_notifications_displayed: + test_notifications_displayed: pass +password_manager: + test_about_logins_navigation_from_context_menu: + test_about_logins_navigation_from_login_form_context_menu: pass +pdf_viewer: + test_pdf_data_can_be_cleared: + test_pdf_data_can_be_cleared: pass +printing_ui: + test_print_preview: + test_print_preview_keys: pass +reader_view: + test_improved_type_control_panel: + test_type_control_panel_font[sans-serif]: pass + test_improved_type_control_panel: + test_type_control_panel_font[serif]: pass + test_improved_type_control_panel: + test_type_control_panel_font[monospace]: pass +scrolling_panning_zooming: + test_zoom_text_only: + test_zoom_text_only_from_settings: pass +security_and_privacy: + test_cross_site_tracking_cookies_blocked: + test_cross_site_tracking_cookies_blocked: pass +tabs: + test_active_tab: + test_active_tab: pass +theme_and_toolbar: + test_customize_themes_and_redirect: + test_redirect_to_addons: pass diff --git a/manifests/incident.yaml b/manifests/incident.yaml new file mode 100644 index 000000000..7a0291a6c --- /dev/null +++ b/manifests/incident.yaml @@ -0,0 +1,194 @@ +--- +address_bar_and_search: + test_add_engine_address_bar: pass + test_addon_suggestion: pass + test_default_search_provider_change_awesome_bar: pass + test_google_search_counts_us: + win: pass + linux: pass + mac: unstable + test_google_withads_url_bar_us: + win: pass + linux: pass + mac: unstable + test_sap_google_adclick: unstable + test_search_code_google_non_us: pass + test_search_code_google_us: pass + test_search_engine_selector: pass + test_search_suggestions: pass + test_search_term_persists: pass + test_server_not_found_error: pass + test_suggestion_engine_selection: pass +audio_video: + test_allow_audio_video_functionality: + win: unstable + mac: pass + linux: unstable + test_block_audio_video_functionality: pass + test_users_actions_saved_on_reload: pass +bookmarks_and_history: + test_add_new_other_bookmark: pass + test_bookmark_via_bookmark_menu: pass + test_bookmark_website_via_star_button: pass + test_clear_all_history: pass + test_clear_recent_history_displayed: pass + test_delete_bookmarks_from_toolbar: pass + test_delete_other_bookmarks: pass + test_deleted_page_not_remembered: pass + test_open_bookmark_in_new_window_via_toolbar_context_menu: pass + test_open_bookmark_in_private_window_via_toolbar_context_menu: pass + test_open_bookmarks_from_toolbar: pass + test_open_websites_from_history: pass + test_private_window_website_not_in_history: pass + test_toggle_bookmarks_toolbar: pass + test_user_can_forget_history: pass +downloads: + test_add_zip_type: pass + test_download_pdf: pass + test_download_pdf_from_context_menu: unstable +drag_and_drop: + test_copy_entire_row_column: pass +find_toolbar: + test_find_in_pdf: pass + test_find_toolbar_nav: pass + test_find_toolbar_search: pass +form_autofill: + test_address_autofill_attribute: pass + test_autofill_cc_cvv: pass + test_autofill_credit_card: pass + test_autofill_credit_card_doorhanger: pass + test_autofill_credit_card_enable: pass + test_autofill_credit_card_four_fields: pass + test_cc_clear_form: pass + test_clear_form: pass + test_create_new_cc_profile: pass + test_create_profile_autofill: pass + test_delete_cc_profile: pass + test_enable_disable_autofill: pass + test_form_autofill_suggestions: pass + test_name_autofill_attribute: pass + test_private_mode_info_not_saved: pass + test_telephone_autofill_attribute: pass + test_updating_address: pass +geolocation: + test_geolocation_shared_via_html5: + win: unstable + mac: pass + linux: pass + test_geolocation_shared_via_w3c_api: + win: unstable + mac: pass + linux: pass +language_packs: + test_language_pack_install_addons: pass + test_language_pack_install_preferences: pass +menus: + test_frequently_used_context_menu: + win: unstable + mac: pass + linux: pass + test_hyperlink_context_menu: pass + test_tab_context_menu_actions: pass + test_tab_context_menu_close: pass +networking: + test_default_dns_protection: pass + test_http_site: pass +notifications: + test_audio_video_permissions_notification: + win: pass + mac: unstable + linux: pass + test_camera_permissions_notification: + win: pass + mac: unstable + linux: pass + test_deny_geolocation: pass + test_deny_screen_capture: pass + test_geolocation_prompt_presence: pass + test_microphone_permissions_notification: + win: pass + mac: unstable + linux: pass + test_notifications_displayed: pass + test_screen_share_permission_prompt: pass + test_webextension_completed_installation_successfully_displayed: pass +password_manager: + test_add_password_non_ascii_chars: pass + test_add_password_save_valid_data: pass + test_auto_saved_generated_password_context_menu: pass + test_can_view_password_when_PP_enabled: pass + test_changes_made_in_edit_mode_are_saved: pass + test_delete_login: pass + test_multiple_saved_logins: pass + test_never_save_login_via_doorhanger: pass + test_primary_password_triggered_on_about_logins_access: pass + test_save_login_via_doorhanger: pass + test_update_login_via_doorhanger: pass +pdf_viewer: + test_download_pdf_with_form_fields: pass + test_download_triggered_on_content_disposition_attachment: pass + test_open_pdf_in_FF: pass + test_pdf_navigation: pass + test_zoom_pdf_viewer: pass +preferences: + test_check_for_updates: pass + test_clear_cookie_data: + win: unstable + mac: pass + linux: pass + test_firefox_home_new_tabs: pass + test_firefox_home_on_launch: pass + test_lang_pack_changed_from_about_prefs: pass + test_never_remember_history: pass + test_notifications_change_set: pass +printing_ui: + test_print_preview: pass +profile: + test_set_default_profile: pass +reader_view: + test_improved_type_control_panel: pass + test_reader_view_close_sidebar: pass + test_reader_view_location_bar: pass +scrolling_panning_zooming: + test_default_zoom_persists: pass + test_mouse_wheel_zoom: + win: pass + mac: unstable + linux: pass + test_zoom_text_only: pass +security_and_privacy: + test_blocking_cryptominers: pass + test_blocking_fingerprinters: pass + test_blocking_social_media_trackers: unstable + test_cookies_not_saved_private_browsing: pass + test_cross_site_tracking_cookies_blocked: pass + test_cryptominers_blocked_and_shown_in_info_panel: pass + test_detected_blocked_trackers_found: pass + test_downloads_from_private_not_leaked: pass + test_https_enabled_private_browsing: pass + test_never_remember_browsing_history: pass + test_no_trackers_detected: pass + test_open_link_in_private_window: pass + test_phishing_and_malware_warnings: pass + test_private_browser_password_doorhanger: pass + test_private_session_awesome_bar_exclusion: pass + test_private_session_history_exclusion: pass + test_private_window_from_panelui: pass + test_third_party_content_blocked_private_browsing: pass + test_tls_v1_2_protocol: pass + test_trackers_crypto_fingerprint_blocked: pass + test_tracking_content_custom_mode: pass + test_undo_close_tab_private_browsing: pass +tabs: + test_active_tab: pass + test_navigation_multiple_tabs: pass + test_open_bookmark_in_new_tab: pass + test_open_new_tab: pass + test_open_new_tab_keys: pass + test_open_new_tab_via_hyperlink: pass + test_pin_tab: pass + test_reopen_tab_through_context_menu: pass + test_reopen_tab_through_history_menu: pass + test_reopen_tabs_through_keys: pass +theme_and_toolbar: + test_customize_themes_and_redirect: pass From 6c4dd00a94d9480832082ccbf26b563f82e79746 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Wed, 12 Nov 2025 11:52:01 -0800 Subject: [PATCH 06/29] typo --- .github/workflows/smoke.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 2ba824bcb..5d16580c4 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -169,7 +169,7 @@ jobs: if: ${{ inputs.mac_installer_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; - echo "STARFOX_MANIFEST='manifest/incident.yaml'" >> "$GITHUB_ENV" + echo "STARFOX_MANIFEST='manifests/incident.yaml'" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; - name: Install dependencies run: | @@ -233,7 +233,7 @@ jobs: MANUAL_DOWNLOAD_LINK: ${{ inputs.linux_tarball_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; - echo "STARFOX_MANIFEST='manifest/incident.yaml'" >> "$GITHUB_ENV" + echo "STARFOX_MANIFEST='manifests/incident.yaml'" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; sudo apt install gnome-screenshot uname -m; From 65af523d8be02e146779be379b98dcae0d4b1170 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Thu, 13 Nov 2025 12:34:30 -0800 Subject: [PATCH 07/29] attempt to correctly choose tests for win / linux --- .github/workflows/smoke.yml | 4 ++-- manifests/incident.yaml | 10 ++++++++-- modules/browser_object_autofill_popup.py | 3 ++- modules/page_object_autofill.py | 4 ++-- .../test_auto_saved_generated_password_context_menu.py | 4 +--- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 5d16580c4..1d343215c 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -169,7 +169,7 @@ jobs: if: ${{ inputs.mac_installer_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; - echo "STARFOX_MANIFEST='manifests/incident.yaml'" >> "$GITHUB_ENV" + echo "STARFOX_MANIFEST=manifests/incident.yaml" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; - name: Install dependencies run: | @@ -233,7 +233,7 @@ jobs: MANUAL_DOWNLOAD_LINK: ${{ inputs.linux_tarball_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; - echo "STARFOX_MANIFEST='manifests/incident.yaml'" >> "$GITHUB_ENV" + echo "STARFOX_MANIFEST=manifests/incident.yaml" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; sudo apt install gnome-screenshot uname -m; diff --git a/manifests/incident.yaml b/manifests/incident.yaml index 7a0291a6c..8115e5891 100644 --- a/manifests/incident.yaml +++ b/manifests/incident.yaml @@ -1,7 +1,10 @@ --- address_bar_and_search: test_add_engine_address_bar: pass - test_addon_suggestion: pass + test_addon_suggestion: + win: unstable + linux: pass + mac: unstable test_default_search_provider_change_awesome_bar: pass test_google_search_counts_us: win: pass @@ -15,7 +18,10 @@ address_bar_and_search: test_search_code_google_non_us: pass test_search_code_google_us: pass test_search_engine_selector: pass - test_search_suggestions: pass + test_search_suggestions: + win: pass + linux: pass + mac: unstable test_search_term_persists: pass test_server_not_found_error: pass test_suggestion_engine_selection: pass diff --git a/modules/browser_object_autofill_popup.py b/modules/browser_object_autofill_popup.py index b0bbe5380..a4c79d9e4 100644 --- a/modules/browser_object_autofill_popup.py +++ b/modules/browser_object_autofill_popup.py @@ -141,5 +141,6 @@ def verify_update_password_doorhanger(self, nav, expected_text): # Verify the doorhanger text self.expect( - lambda _: expected_text in self.get_element("password-update-doorhanger").text + lambda _: expected_text + in self.get_element("password-update-doorhanger").text ) diff --git a/modules/page_object_autofill.py b/modules/page_object_autofill.py index 3c0b88f37..3d85c152f 100644 --- a/modules/page_object_autofill.py +++ b/modules/page_object_autofill.py @@ -860,8 +860,8 @@ def generate_secure_password(self, context_menu): # Wait until the password field is actually filled self.parent.expect( lambda _: ( - (elem := self.parent.get_element("password-login-field")) - and elem.get_attribute("value") not in ("", None) + (elem := self.parent.get_element("password-login-field")) + and elem.get_attribute("value") not in ("", None) ) ) diff --git a/tests/password_manager/test_auto_saved_generated_password_context_menu.py b/tests/password_manager/test_auto_saved_generated_password_context_menu.py index 78bfecdc8..f1a6a17b7 100644 --- a/tests/password_manager/test_auto_saved_generated_password_context_menu.py +++ b/tests/password_manager/test_auto_saved_generated_password_context_menu.py @@ -36,9 +36,7 @@ def test_auto_saved_generated_password_context_menu(driver: Firefox): login_autofill.LoginForm(login_autofill).generate_secure_password(context_menu) # Verify the update doorhanger is displayed - autofill_popup_panel.verify_update_password_doorhanger( - nav, UPDATE_DOORHANGER_TEXT - ) + autofill_popup_panel.verify_update_password_doorhanger(nav, UPDATE_DOORHANGER_TEXT) # Navigate to about:logins page tabs.switch_to_new_tab() From cb83b4e65680b25f7ce18e48a876de1ef76b5feb Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Thu, 13 Nov 2025 13:13:12 -0800 Subject: [PATCH 08/29] switch conftest version gathering from fixture --- conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 18b45e369..22d4bee7c 100644 --- a/conftest.py +++ b/conftest.py @@ -200,6 +200,12 @@ def _screenshot_whole_screen(filename: str, driver: Firefox, opt_ci: bool): return fullpath +def _get_version(driver: Firefox): + driver.get("chrome://browser/content/aboutDialog.xhtml") + version_el = driver.find_element(By.ID, "version") + return version_el.text + + @pytest.fixture() def opt_headless(request): return request.config.getoption("--run-headless") @@ -496,7 +502,7 @@ def driver( WebDriverWait(driver, timeout=40).until( EC.presence_of_element_located((By.TAG_NAME, "body")) ) - json_metadata["fx_version"] = version + json_metadata["fx_version"] = _get_version(driver) json_metadata["machine_config"] = machine_config json_metadata["suite_id"] = suite_id json_metadata["test_case"] = test_case From 65eda4940ef09f4edcd4574d576734116997f040 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 14 Nov 2025 10:32:30 -0800 Subject: [PATCH 09/29] debug manifests not respected in mac/lin ci --- choose_ci_set.py | 8 +++++--- conftest.py | 2 +- manifests/incident.yaml | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/choose_ci_set.py b/choose_ci_set.py index 7776b2aed..d3897d369 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -114,6 +114,8 @@ def convert_manifest_to_list(manifest_loc): manifest = yaml.safe_load(open(manifest_loc)) toplevel = [".", "tests"] tests = [] + if manifest: + print(f"Reading {manifest_loc}") for suite in manifest: print(suite) if suite == "add_manifest": @@ -122,20 +124,20 @@ def convert_manifest_to_list(manifest_loc): convert_manifest_to_list(SLASH.join(["manifests", linked_manifest])) ) for test in manifest[suite]: - print(f" {test}") + # print(f" {test}") addtest = False test_name = f"{test}.py" if manifest[suite][test] == "pass": addtest = True elif isinstance(manifest[suite][test], dict): - print(f"==={manifest[suite][test].get(sysname())}===") + # print(f"==={manifest[suite][test].get(sysname())}===") if manifest[suite][test].get(sysname()) == "pass": addtest = True else: for subtest in manifest[suite][test]: if subtest in ["mac", "win", "linux"]: continue - print(f" {subtest}") + # print(f" {subtest}") test_name = f"{test}.py::{subtest}" if isinstance(manifest[suite][test][subtest], dict): if manifest[suite][test][subtest].get(sysname()) == "pass": diff --git a/conftest.py b/conftest.py index 22d4bee7c..5e7127a36 100644 --- a/conftest.py +++ b/conftest.py @@ -512,7 +512,7 @@ def driver( finally: if hard_quit: return - if "driver" in locals() or "driver" in globals(): + if ("driver" in locals() or "driver" in globals()) and driver: driver.quit() diff --git a/manifests/incident.yaml b/manifests/incident.yaml index 8115e5891..d2f912d19 100644 --- a/manifests/incident.yaml +++ b/manifests/incident.yaml @@ -5,7 +5,10 @@ address_bar_and_search: win: unstable linux: pass mac: unstable - test_default_search_provider_change_awesome_bar: pass + test_default_search_provider_change_awesome_bar: + win: unstable + linux: pass + mac: pass test_google_search_counts_us: win: pass linux: pass From ba22fae533861b67bc5ec4c25e785cf60add6418 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 14 Nov 2025 11:04:34 -0800 Subject: [PATCH 10/29] debug manifests not respected in mac/lin ci --- choose_ci_set.py | 1 + 1 file changed, 1 insertion(+) diff --git a/choose_ci_set.py b/choose_ci_set.py index d3897d369..c4b08b9b9 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -155,6 +155,7 @@ def convert_manifest_to_list(manifest_loc): if __name__ == "__main__": + print("Selecting test set...") if os.path.exists(".env"): with open(".env") as fh: contents = fh.read() From 0f4bcdc9f9616f66fd642b038fe7ec2cde128290 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 14 Nov 2025 11:10:30 -0800 Subject: [PATCH 11/29] debug manifests not respected in mac/lin ci --- choose_ci_set.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/choose_ci_set.py b/choose_ci_set.py index c4b08b9b9..a5b490083 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -164,8 +164,8 @@ def convert_manifest_to_list(manifest_loc): if "RUN_ALL='true'" in contents: os.environ["MANUAL"] = "true" - if os.environ.get("TESTRAIL_REPORT") or os.environ.get("MANUAL"): - # Run all tests if this is a scheduled beta or a manual run + if os.environ.get("TESTRAIL_REPORT"): + # Run all tests if this is a scheduled run with open(OUTPUT_FILE, "w") as fh: fh.write("tests") sys.exit(0) From f9708ac817d300c79f73432e5085df37b683d840 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 14 Nov 2025 12:13:19 -0800 Subject: [PATCH 12/29] debug manifests not respected in mac/lin ci --- conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 5e7127a36..48c02472f 100644 --- a/conftest.py +++ b/conftest.py @@ -203,7 +203,9 @@ def _screenshot_whole_screen(filename: str, driver: Firefox, opt_ci: bool): def _get_version(driver: Firefox): driver.get("chrome://browser/content/aboutDialog.xhtml") version_el = driver.find_element(By.ID, "version") - return version_el.text + version = version_el.text + driver.get("about:blank") + return version @pytest.fixture() From 007b09a99691248cb6413bf068345f862a580566 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 14 Nov 2025 12:38:17 -0800 Subject: [PATCH 13/29] mark certain incident tests unstable --- manifests/incident.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/manifests/incident.yaml b/manifests/incident.yaml index d2f912d19..6b17ddd66 100644 --- a/manifests/incident.yaml +++ b/manifests/incident.yaml @@ -5,10 +5,7 @@ address_bar_and_search: win: unstable linux: pass mac: unstable - test_default_search_provider_change_awesome_bar: - win: unstable - linux: pass - mac: pass + test_default_search_provider_change_awesome_bar: unstable test_google_search_counts_us: win: pass linux: pass From f19dde2f49a39e4180364e19710babc1c907c5a3 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 14 Nov 2025 13:48:11 -0800 Subject: [PATCH 14/29] incident manifest --- choose_ci_set.py | 1 - manifests/incident.yaml | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/choose_ci_set.py b/choose_ci_set.py index a5b490083..932f0e1c8 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -7,7 +7,6 @@ import yaml CI_MANIFEST = "manifests/ci.yaml" -CI_MARK = "@pytest.mark.ci" HEADED_MARK = "@pytest.mark.headed" MIN_RUN_SIZE = 7 OUTPUT_FILE = "selected_tests" diff --git a/manifests/incident.yaml b/manifests/incident.yaml index 6b17ddd66..53623cc7f 100644 --- a/manifests/incident.yaml +++ b/manifests/incident.yaml @@ -18,10 +18,7 @@ address_bar_and_search: test_search_code_google_non_us: pass test_search_code_google_us: pass test_search_engine_selector: pass - test_search_suggestions: - win: pass - linux: pass - mac: unstable + test_search_suggestions: unstable test_search_term_persists: pass test_server_not_found_error: pass test_suggestion_engine_selection: pass @@ -49,7 +46,10 @@ bookmarks_and_history: test_toggle_bookmarks_toolbar: pass test_user_can_forget_history: pass downloads: - test_add_zip_type: pass + test_add_zip_type: + win: pass + mac: pass + linux: unstable test_download_pdf: pass test_download_pdf_from_context_menu: unstable drag_and_drop: From a7f33f5c2f4deb57e2fdfeca7640e4be714a2b55 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Mon, 17 Nov 2025 11:33:31 -0800 Subject: [PATCH 15/29] remove unnecessary incident test --- manifests/incident.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/manifests/incident.yaml b/manifests/incident.yaml index 53623cc7f..6b8231ace 100644 --- a/manifests/incident.yaml +++ b/manifests/incident.yaml @@ -153,7 +153,6 @@ profile: test_set_default_profile: pass reader_view: test_improved_type_control_panel: pass - test_reader_view_close_sidebar: pass test_reader_view_location_bar: pass scrolling_panning_zooming: test_default_zoom_persists: pass From 193a1fcae164591e7956a50570dca2f79ba8928c Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Tue, 18 Nov 2025 09:25:35 -0800 Subject: [PATCH 16/29] add manifest maker, change chooser to manifests / git logic only --- choose_ci_set.py | 8 +- choose_test_channel.py | 6 +- make_manifest.py | 36 +++ manifests/all.yaml | 495 +++++++++++++++++++++++++++++++++++++++++ manifests/smoke.yaml | 495 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 1037 insertions(+), 3 deletions(-) create mode 100644 make_manifest.py create mode 100644 manifests/all.yaml create mode 100644 manifests/smoke.yaml diff --git a/choose_ci_set.py b/choose_ci_set.py index 932f0e1c8..bd7464319 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -165,8 +165,10 @@ def convert_manifest_to_list(manifest_loc): if os.environ.get("TESTRAIL_REPORT"): # Run all tests if this is a scheduled run + run_list = convert_manifest_to_list("manifests/smoke.yaml") + run_list = dedupe(run_list) with open(OUTPUT_FILE, "w") as fh: - fh.write("tests") + fh.write("\n".join(run_list)) sys.exit(0) if os.environ.get("STARFOX_MANIFEST"): @@ -203,8 +205,10 @@ def convert_manifest_to_list(manifest_loc): if main_conftest in committed_files or base_page in committed_files: # Run all the tests (no files as arguments) if main conftest or basepage changed + run_list = convert_manifest_to_list("manifests/all.yaml") + run_list = dedupe(run_list) with open(OUTPUT_FILE, "w") as fh: - fh.write("tests") + fh.write("\n".join(run_list)) sys.exit(0) all_tests = [] diff --git a/choose_test_channel.py b/choose_test_channel.py index 06198725d..705a5859f 100644 --- a/choose_test_channel.py +++ b/choose_test_channel.py @@ -3,7 +3,7 @@ import sys from subprocess import check_output -ALL_CHANNELS = ["smoke", "l10n"] +ALL_CHANNELS = ["smoke", "l10n", "functional"] SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) SLASH = "/" if "/" in SCRIPT_DIR else "\\" @@ -29,6 +29,10 @@ ] ) +if len(sys.argv) > 1: + print(sys.argv[1:]) + sys.exit(0) + check_output(["git", "fetch", "--quiet", "--depth=1", "origin", "main"]) committed_files = ( diff --git a/make_manifest.py b/make_manifest.py new file mode 100644 index 000000000..2495c8f21 --- /dev/null +++ b/make_manifest.py @@ -0,0 +1,36 @@ +import os +import re + +import yaml + +TESTFILE_RE = re.compile(r"test_.*\.py") +TEST_RE = re.compile(r"def (test.*)\(") + +if __name__ == "__main__": + testdict = {} + for root, _, files in os.walk("tests"): + for f in files: + if not TESTFILE_RE.match(f): + continue + pointer = testdict + for level in os.path.split(root)[1:]: + if level not in testdict: + pointer[level] = {} + pointer = pointer[level] + status = "pass" + filename = os.path.join(root, f) + with open(filename) as fh: + metatest = f.rsplit(".", 1)[0] + for line in fh.readlines(): + m = TEST_RE.search(line) + if m and m[1] != "test_case": + if metatest not in pointer: + pointer[metatest] = {m[1]: status} + else: + pointer[metatest][m[1]] = status + print(yaml.safe_dump(testdict), "\n\n\n\n===\n\n\n\n") + status = "pass" + if "mark.unstable" in line: + status = "unstable" + with open("new_manifest.yaml", "w") as fh: + yaml.safe_dump(testdict, fh) diff --git a/manifests/all.yaml b/manifests/all.yaml new file mode 100644 index 000000000..b8b24774c --- /dev/null +++ b/manifests/all.yaml @@ -0,0 +1,495 @@ +address_bar_and_search: + test_adaptive_history_autofill: + test_add_adaptive_history_autofill: pass + test_add_engine_address_bar: + test_add_search_engine_from_address_bar: pass + test_added_open_search_engine_default: + test_added_open_search_engine_default: pass + test_addon_suggestion: + test_addon_suggestion_based_on_search_input: pass + test_addressbar_search_engine_keywords: + test_addressbar_search_engine_keywords: pass + test_copied_url_contains_https: + test_copied_url_contains_https: pass + test_ctrl_enter_fixes_url: + test_ctrl_enter_fixes_url: pass + test_default_search_provider_change_awesome_bar: + test_default_search_provider_change_awesome_bar: pass + test_default_search_provider_change_legacy_search_bar: + test_default_search_provider_change_legacy_search_bar: pass + test_dont_show_search_suggestions_in_private_window: + test_no_search_engine_suggestions_in_private_window: pass + test_google_search_counts_us: + test_google_search_counts_us: pass + test_google_withads_url_bar_us: + test_google_withads_url_bar_us: pass + test_insertion_point_no_search_terms_display: + test_insertion_point_no_search_terms_display: pass + test_no_suggestions_for_empty_query: + test_suggestions_for_empty_query_not_shown_in_search_mode: pass + test_non_sponsored_topsite_context_menu_option: + test_non_sponsored_topsite_context_menu_option: pass + test_refresh_firefox_dialog: + test_refresh_firefox_dialog: pass + test_sap_google_adclick: + test_sap_google_adclick: unstable + test_seach_suggestions_can_be_disabled: + test_search_suggestions_pref_affects_urlbar_and_searchbar: pass + test_search_code_google_non_us: + test_search_code_google_non_us: pass + test_search_code_google_us: + test_search_code_google_us: pass + test_search_engine_selector: + test_search_engine_selector_and_validator: pass + test_search_mode_appears_after_input: + test_private_mode_repeat_after_enabling_pref: pass + test_search_mode_appears_and_suggestions_update: pass + test_search_mode_persists_mixed_with_bing: + test_search_mode_persists_mixed_with_bing: pass + test_search_modes_for_sites: + test_search_modes_for_sites: pass + test_search_string_displayed_when_addressbar_unfocused: + test_address_bar_and_search_string_displayed_when_addressbar_unfocused: pass + test_search_suggestions: + test_search_suggests_enabled: pass + test_search_term_persists: + test_search_term_persists: unstable + test_searchbar_display_alpenglow_theme: + test_searchbar_display_alpenglow_theme: pass + test_searchbar_on_engines_remove_restore: + test_searchbar_engine_remove_restore_affects_both_windows: pass + test_searchbar_results_shown_in_a_new_tab: + test_search_bar_results_shown_in_a_new_tab: pass + test_searchmode_change_tab: + test_searchmode_change_tab: pass + test_server_not_found_error: + test_server_not_found_error: pass + test_server_not_found_error_pb: + test_server_not_found_error_on_private_window: pass + test_suggestion_engine_selection: + test_search_suggestion_for_engine_selection: pass + test_tile_menu_options: + test_default_tile_hover_states: pass + test_tile_context_menu_options: pass +audio_video: + test_allow_audio_video_functionality: + test_allow_audio_video_functionality: pass + test_block_audio_video_functionality: + test_block_audio_video_functionality: pass + test_users_actions_saved_on_reload: + test_users_actions_saved_on_reload: pass +bookmarks_and_history: + test_add_new_other_bookmark: + test_add_new_other_bookmark: pass + test_bookmark_via_bookmark_menu: + test_bookmark_via_hamburger_menu: pass + test_bookmark_website_via_star_button: + test_bookmark_website_via_star: pass + test_clear_all_history: + test_clear_all_history: pass + test_clear_recent_history_displayed: + test_clear_recent_history_displayed: pass + test_delete_bookmarks_from_toolbar: + test_delete_bookmarks_from_toolbar: pass + test_delete_other_bookmarks: + test_delete_other_bookmarks: pass + test_deleted_page_not_remembered: + test_deleted_page_not_remembered: pass + test_edit_bookmark_from_bookmark_menu: + test_edit_bookmark_from_bookmark_menu: pass + test_edit_bookmark_via_star_button: + test_edit_bookmark_via_star_button: pass + test_history_menu_from_different_places: + test_history_menu_in_different_places: pass + test_import_bookmarks_chrome: + test_chrome_bookmarks_imported: pass + test_import_bookmarks_edge: + test_edge_bookmarks_imported: pass + test_open_all_bookmarks_from_bookmarks_toolbar: + test_open_all_bookmarks_from_bookmarks_toolbar: pass + test_open_bookmark_in_new_window_via_toolbar_context_menu: + test_open_bookmark_in_new_window_via_toolbar_context_menu: pass + test_open_bookmark_in_private_window_via_toolbar_context_menu: + test_open_bookmark_in_new_private_window_via_toolbar_context_menu: pass + test_open_bookmarks_from_toolbar: + test_open_bookmarks_from_toolbar: pass + test_open_websites_from_history: + test_open_websites_from_history: pass + test_opened_website_in_new_tab_present_in_hamburger_history_menu: + test_the_website_opened_in_new_tab_is_present_in_history_menu: pass + test_opened_website_in_new_window_present_in_hamburger_history_menu: + test_the_website_opened_in_new_window_is_present_in_history_menu: pass + test_opened_website_present_in_hamburger_history_menu: + test_the_most_recent_website_is_present_in_history_menu: pass + test_private_window_website_not_in_history: + test_opened_website_in_private_window_not_captured_in_history_list: pass + test_toggle_bookmarks_toolbar: + test_toggle_bookmark_toolbar: pass + test_user_can_forget_history: + test_user_can_forget_history: pass +downloads: + test_add_mime_type_doc: + test_mime_type_doc: pass + test_add_zip_type: + test_add_zip_type: pass + test_download_pdf: + test_download_pdf: pass + test_download_pdf_from_context_menu: + test_download_pdf_from_context_menu: pass + test_mixed_content_download_via_https: + test_mixed_content_download_via_https: unstable + test_set_always_ask_file_type: + test_set_always_ask_file_type: pass +drag_and_drop: + test_copy_entire_row_column: + test_copy_entire_row_column: pass + test_copy_from_an_editor_paste_in_another: + test_copy_from_an_editor_paste_in_another: pass + test_copy_hyperlink_table: + test_copy_table_with_hyperlink: pass + test_copy_table_header: + test_copy_table_header: pass + test_paste_image_text: + test_paste_image_text: pass +find_toolbar: + test_find_in_pdf: + test_find_in_pdf_using_key_combos: pass + test_find_toolbar_nav: + test_find_toolbar_navigation: pass + test_find_toolbar_search: + test_find_toolbar_search: pass +form_autofill: + test_address_autofill_attribute: + test_address_attribute_selection: pass + test_autofill_cc_cvv: + test_autofill_cc_cvv: pass + test_autofill_credit_card: + test_autofill_credit_card: pass + test_autofill_credit_card_doorhanger: + test_autofill_credit_card_door_hanger: pass + test_autofill_credit_card_enable: + test_enable_disable_form_autofill_cc: pass + test_autofill_credit_card_four_fields: + test_autofill_four_fields: pass + test_cc_clear_form: + test_clear_form_credit_card: pass + test_clear_form: + test_clear_form: pass + test_create_new_cc_profile: + test_create_new_cc_profile: pass + test_create_profile_autofill: + test_create_address_profile: pass + test_delete_cc_profile: + test_delete_cc_profile: pass + test_edit_credit_card: + test_edit_credit_card_profile: pass + test_enable_disable_autofill: + test_enable_disable_autofill: pass + test_form_autofill_suggestions: + test_form_autofill_suggestions: pass + test_name_autofill_attribute: + test_name_attribute_selection: pass + test_private_mode_info_not_saved: + test_private_mode_info_not_saved: pass + test_telephone_autofill_attribute: + test_telephone_attribute_autofill: pass + test_updating_address: + test_update_address: pass + test_updating_credit_card: + test_update_cc_no_dupe_name: pass +geolocation: + test_geolocation_shared_via_html5: + test_allow_permission_on_geolocation_via_html5: pass + test_block_permission_on_geolocation_via_w3c_api: pass + test_geolocation_shared_via_w3c_api: + test_allow_permission_on_geolocation_via_w3c_api: pass + test_block_permission_on_geolocation_via_w3c_api: pass +language_packs: + test_language_pack_install_addons: + test_language_pack_install_from_addons: pass + test_language_pack_install_preferences: + test_language_pack_install_about_preferences: pass +menus: + test_copy_paste_actions: + test_login_form_copy_paste: pass + test_search_field_copy_paste: pass + test_text_area_copy_paste: pass + test_frequently_used_context_menu: + test_inspect: pass + test_save_page_as: pass + test_take_screenshot: pass + test_hyperlink_context_menu: + test_open_link_in_new_window: pass + test_image_context_menu_actions: + test_copy_image_link: pass + test_open_image_in_new_tab: pass + test_save_image_as: pass + test_tab_context_menu_actions: + test_close_multiple_tabs_to_right: pass + test_duplicate_tab: pass + test_tab_context_menu_close: + test_close_multiple_tabs_other_tabs: pass + test_close_multiple_tabs_to_left: pass + test_copy_link: pass +meta: + test_selectors: + test_a_selector: pass + test_version: + test_version: pass +networking: + test_default_dns_protection: + test_doh_enforces_secure_dns_resolution: pass + test_http_site: + test_http_site: pass +notifications: + test_audio_video_permissions_notification: + test_camera_and_microphone_permissions_notification: pass + test_camera_permissions_notification: + test_camera_permissions_notification: pass + test_deny_geolocation: + test_deny_geolocation: pass + test_deny_screen_capture: + test_deny_screen_capture: pass + test_geolocation_prompt_presence: + test_geolocation_prompt_is_triggered_on_request_location_on_a_website: pass + test_microphone_permissions_notification: + test_microphone_permissions_notification: pass + test_notifications_displayed: + test_notifications_displayed: pass + test_screen_share_permission_prompt: + test_screen_share_permission_prompt: pass + test_webextension_completed_installation_successfully_displayed: + test_webextension_completed_installation_successfully_displayed: pass +password_manager: + test_about_logins_navigation_from_context_menu: + test_about_logins_navigation_from_login_form_context_menu: pass + test_about_logins_navigation_from_hamburger_menu: + test_about_logins_navigation_from_password_hamburger_menu: pass + test_about_logins_search_username: + test_about_logins_search_username: pass + test_about_logins_search_website: + test_about_logins_search_website: pass + test_add_password_non_ascii_chars: + test_add_password_non_ascii_chars: pass + test_add_password_save_valid_data: + test_add_password_save_valid_data: pass + test_add_primary_password: + test_add_primary_password: pass + test_auto_saved_generated_password_context_menu: + test_auto_saved_generated_password_context_menu: pass + test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load: + test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load: pass + test_can_view_password_when_PP_enabled: + test_password_can_be_shown: pass + test_changes_made_in_edit_mode_are_saved: + test_changes_made_in_edit_mode_are_saved: pass + test_delete_login: + test_delete_login: pass + test_multiple_saved_logins: + test_multiple_saved_logins: pass + test_never_save_login_via_doorhanger: + test_never_save_login_via_doorhanger: pass + test_password_csv_correctness: + test_password_csv_correctness: pass + test_password_csv_export: + test_password_csv_export: pass + test_primary_password_triggered_on_about_logins_access: + test_primary_password_triggered_on_about_logins_access_via_hamburger_menu: pass + test_save_login_via_doorhanger: + test_save_login_via_doorhanger: pass + test_saved_hyperlink_redirects_to_corresponding_page: + test_saved_hyperlink_redirects_to_corresponding_page: pass + test_update_login_via_doorhanger: + test_update_login_via_doorhanger: pass +pdf_viewer: + test_add_image_pdf: + test_add_image_pdf: pass + test_download_pdf_with_form_fields: + test_download_pdf_with_form_fields: pass + test_download_triggered_on_content_disposition_attachment: + test_download_panel_triggered_on_content_disposition_attachment: pass + test_open_pdf_in_FF: + test_open_pdf_in_fx: pass + test_pdf_data_can_be_cleared: + test_pdf_data_can_be_cleared: pass + test_pdf_download: + test_pdf_download: pass + test_pdf_input_numbers: + test_pdf_input_numbers: pass + test_pdf_navigation: + test_navigation_keys: pass + test_navigation_next_prev: pass + test_navigation_page_numbers: pass + test_toolbar_options_cursor: pass + test_toolbar_options_rotate_ccw: pass + test_toolbar_options_rotate_cw: pass + test_zoom_pdf_viewer: + test_zoom_pdf_viewer_keys: pass + test_zoom_pdf_viewer_toolbar: pass +pocket: + test_basic_de: + test_localized_pocket_layout_DE: pass + test_basic_fr: + test_localized_pocket_layout_FR: pass + test_basic_gb: + test_localized_pocket_layout_GB: pass + test_basic_us: + test_localized_pocket_layout_US: pass + test_new_tab_about_blank: pass +preferences: + test_check_for_updates: + test_check_for_updates: pass + test_clear_cookie_data: + test_clear_cookie_data: pass + test_firefox_home_new_tabs: + test_firefox_home_new_tab: pass + test_firefox_home_on_launch: + test_firefox_home_on_launch: pass + test_lang_pack_changed_from_about_prefs: + test_lang_pack_changed_from_about_prefs: pass + test_manage_cookie_data: + test_manage_cookie_data: pass + test_never_remember_history: + test_never_remember_history: pass + test_notifications_change_set: + test_notifications_allow: pass +printing_ui: + test_print_preview: + test_print_preview_keys: pass + test_print_preview_menu: pass + test_print_to_pdf: + test_print_to_pdf: pass +profile: + test_set_default_profile: + test_set_default_profile: pass +reader_view: + test_improved_type_control_panel: + test_type_control_panel_character_spacing: pass + test_type_control_panel_content_width: pass + test_type_control_panel_font: pass + test_type_control_panel_line_spacing: pass + test_type_control_panel_size: pass + test_type_control_panel_text_alignment: pass + test_type_control_panel_word_spacing: pass + test_reader_view_close_sidebar: + test_reader_view_close_from_sidebar: pass + test_reader_view_location_bar: + test_reader_view_open_close_using_keys: pass + test_reader_view_open_close_using_searchbar: pass +scrolling_panning_zooming: + test_default_zoom_persists: + test_default_zoom_across_tabs: pass + test_mouse_wheel_zoom: + test_mouse_wheel_zoom: pass + test_zoom_from_menu_bar: + test_zoom_from_menu_bar: pass + test_zoom_menu_correlation: + test_zoom_level_div_position: pass + test_zoom_text_only: + test_zoom_text_only_after_restart: pass + test_zoom_text_only_from_settings: pass +security_and_privacy: + test_blocking_cryptominers: + test_blocking_cryptominers: pass + test_blocking_fingerprinters: + test_blocking_fingerprinter: pass + test_blocking_social_media_trackers: + test_blocking_social_media_trackers: pass + test_cookies_not_saved_private_browsing: + test_cookies_not_saved_private_browsing: pass + test_cross_site_tracking_cookies_blocked: + test_cross_site_tracking_cookies_blocked: pass + test_cryptominers_blocked_and_shown_in_info_panel: + test_cryptominers_blocked_and_shown_in_info_panel: pass + test_detected_blocked_trackers_found: + test_detected_blocked_trackers_found: pass + test_downloads_from_private_not_leaked: + test_downloads_from_private_not_leaked: pass + test_https_enabled_private_browsing: + test_https_first_mode_in_private_browsing: pass + test_never_remember_browsing_history: + test_never_remember_browsing_history_from_panel: pass + test_never_remember_browsing_history_settings: pass + test_no_trackers_detected: + test_no_trackers_detected: pass + test_open_link_in_private_window: + test_open_link_in_private_window: pass + test_phishing_and_malware_warnings: + test_phishing_and_malware_warning_errors: pass + test_private_browser_password_doorhanger: + test_no_password_doorhanger_private_browsing: pass + test_private_session_awesome_bar_exclusion: + test_websites_visited_in_private_browser_not_displayed_in_awesome_bar: pass + test_private_session_history_exclusion: + test_websites_visited_in_private_browser_not_displayed_in_history: pass + test_private_window_from_panelui: + test_private_window_from_panelui: pass + test_third_party_content_blocked_private_browsing: + test_third_party_content_blocked_private_browsing_allowed_tracking: pass + test_third_party_content_blocked_private_browsing_cross_site: pass + test_third_party_content_private_browsing_tracking_statuses: pass + test_tls_v1_2_protocol: + test_tls_v1_2_protocol: pass + test_trackers_crypto_fingerprint_blocked: + test_cross_site_trackrs_crypto_fingerprinter_blocked: pass + test_tracking_content_custom_mode: + test_allowed_tracking_content: pass + test_blocked_tracking_content: pass + test_undo_close_tab_private_browsing: + test_undo_close_tab_private_browsing: pass +sync_and_fxa: + test_existing_fxa: + test_sync_existing_fxa: pass + test_new_fxa: + test_sync_new_fxa: pass +tabs: + test_active_tab: + test_active_tab: pass + test_change_position_of_pinned_tabs: + test_change_position_of_pinned_tabs: pass + test_close_pinned_tab_via_mouse: + test_close_pinned_tab_via_middle_click: pass + test_close_tab_through_middle_mouse_click: + test_close_tab_through_middle_mouse_click: pass + test_display_customize_button: + test_customize_button_displayed_in_tab_bar: pass + test_list_all_tabs: + test_list_all_tabs: pass + test_mute_tabs: + test_mute_unmute_tab: pass + test_navigation_multiple_tabs: + test_navigation_multiple_tabs: pass + test_open_bookmark_in_new_tab: + test_open_bookmark_in_new_tab: pass + test_open_new_bg_tab_via_mouse_and_keyboard: + test_open_new_bg_tab_via_mouse_and_keyboard: pass + test_open_new_tab: + test_open_new_tab_plus: pass + test_open_new_tab_keys: + test_open_new_tab_via_keyboard: pass + test_open_new_tab_via_hyperlink: + test_open_new_via_hyperlink: pass + test_pin_tab: + test_pin_tab: pass + test_pin_unpin_selected_tabs: + test_pin_unpin_selected_tabs: pass + test_play_mute_unmute_tabs_via_toggle: + test_play_mute_unmute_tabs_via_toggle: pass + test_reload_overiding_cache_keys: + test_reload_overiding_cache_keys: pass + test_reload_tab_via_keyboard: + test_reload_tab_via_keyboard: pass + test_reopen_tab_through_context_menu: + test_reopen_tab_through_context_menu: pass + test_reopen_tab_through_history_menu: + test_reopen_tab_through_history_menu: pass + test_reopen_tabs_through_keys: + test_reopen_tabs_through_keys: pass +theme_and_toolbar: + test_customize_themes_and_redirect: + test_activate_theme_background_matches_expected: pass + test_alpenglow_theme: pass + test_redirect_to_addons: pass + test_installed_theme_enabled: + test_find_more_themes: pass + test_installed_theme_enabled: pass diff --git a/manifests/smoke.yaml b/manifests/smoke.yaml new file mode 100644 index 000000000..b8b24774c --- /dev/null +++ b/manifests/smoke.yaml @@ -0,0 +1,495 @@ +address_bar_and_search: + test_adaptive_history_autofill: + test_add_adaptive_history_autofill: pass + test_add_engine_address_bar: + test_add_search_engine_from_address_bar: pass + test_added_open_search_engine_default: + test_added_open_search_engine_default: pass + test_addon_suggestion: + test_addon_suggestion_based_on_search_input: pass + test_addressbar_search_engine_keywords: + test_addressbar_search_engine_keywords: pass + test_copied_url_contains_https: + test_copied_url_contains_https: pass + test_ctrl_enter_fixes_url: + test_ctrl_enter_fixes_url: pass + test_default_search_provider_change_awesome_bar: + test_default_search_provider_change_awesome_bar: pass + test_default_search_provider_change_legacy_search_bar: + test_default_search_provider_change_legacy_search_bar: pass + test_dont_show_search_suggestions_in_private_window: + test_no_search_engine_suggestions_in_private_window: pass + test_google_search_counts_us: + test_google_search_counts_us: pass + test_google_withads_url_bar_us: + test_google_withads_url_bar_us: pass + test_insertion_point_no_search_terms_display: + test_insertion_point_no_search_terms_display: pass + test_no_suggestions_for_empty_query: + test_suggestions_for_empty_query_not_shown_in_search_mode: pass + test_non_sponsored_topsite_context_menu_option: + test_non_sponsored_topsite_context_menu_option: pass + test_refresh_firefox_dialog: + test_refresh_firefox_dialog: pass + test_sap_google_adclick: + test_sap_google_adclick: unstable + test_seach_suggestions_can_be_disabled: + test_search_suggestions_pref_affects_urlbar_and_searchbar: pass + test_search_code_google_non_us: + test_search_code_google_non_us: pass + test_search_code_google_us: + test_search_code_google_us: pass + test_search_engine_selector: + test_search_engine_selector_and_validator: pass + test_search_mode_appears_after_input: + test_private_mode_repeat_after_enabling_pref: pass + test_search_mode_appears_and_suggestions_update: pass + test_search_mode_persists_mixed_with_bing: + test_search_mode_persists_mixed_with_bing: pass + test_search_modes_for_sites: + test_search_modes_for_sites: pass + test_search_string_displayed_when_addressbar_unfocused: + test_address_bar_and_search_string_displayed_when_addressbar_unfocused: pass + test_search_suggestions: + test_search_suggests_enabled: pass + test_search_term_persists: + test_search_term_persists: unstable + test_searchbar_display_alpenglow_theme: + test_searchbar_display_alpenglow_theme: pass + test_searchbar_on_engines_remove_restore: + test_searchbar_engine_remove_restore_affects_both_windows: pass + test_searchbar_results_shown_in_a_new_tab: + test_search_bar_results_shown_in_a_new_tab: pass + test_searchmode_change_tab: + test_searchmode_change_tab: pass + test_server_not_found_error: + test_server_not_found_error: pass + test_server_not_found_error_pb: + test_server_not_found_error_on_private_window: pass + test_suggestion_engine_selection: + test_search_suggestion_for_engine_selection: pass + test_tile_menu_options: + test_default_tile_hover_states: pass + test_tile_context_menu_options: pass +audio_video: + test_allow_audio_video_functionality: + test_allow_audio_video_functionality: pass + test_block_audio_video_functionality: + test_block_audio_video_functionality: pass + test_users_actions_saved_on_reload: + test_users_actions_saved_on_reload: pass +bookmarks_and_history: + test_add_new_other_bookmark: + test_add_new_other_bookmark: pass + test_bookmark_via_bookmark_menu: + test_bookmark_via_hamburger_menu: pass + test_bookmark_website_via_star_button: + test_bookmark_website_via_star: pass + test_clear_all_history: + test_clear_all_history: pass + test_clear_recent_history_displayed: + test_clear_recent_history_displayed: pass + test_delete_bookmarks_from_toolbar: + test_delete_bookmarks_from_toolbar: pass + test_delete_other_bookmarks: + test_delete_other_bookmarks: pass + test_deleted_page_not_remembered: + test_deleted_page_not_remembered: pass + test_edit_bookmark_from_bookmark_menu: + test_edit_bookmark_from_bookmark_menu: pass + test_edit_bookmark_via_star_button: + test_edit_bookmark_via_star_button: pass + test_history_menu_from_different_places: + test_history_menu_in_different_places: pass + test_import_bookmarks_chrome: + test_chrome_bookmarks_imported: pass + test_import_bookmarks_edge: + test_edge_bookmarks_imported: pass + test_open_all_bookmarks_from_bookmarks_toolbar: + test_open_all_bookmarks_from_bookmarks_toolbar: pass + test_open_bookmark_in_new_window_via_toolbar_context_menu: + test_open_bookmark_in_new_window_via_toolbar_context_menu: pass + test_open_bookmark_in_private_window_via_toolbar_context_menu: + test_open_bookmark_in_new_private_window_via_toolbar_context_menu: pass + test_open_bookmarks_from_toolbar: + test_open_bookmarks_from_toolbar: pass + test_open_websites_from_history: + test_open_websites_from_history: pass + test_opened_website_in_new_tab_present_in_hamburger_history_menu: + test_the_website_opened_in_new_tab_is_present_in_history_menu: pass + test_opened_website_in_new_window_present_in_hamburger_history_menu: + test_the_website_opened_in_new_window_is_present_in_history_menu: pass + test_opened_website_present_in_hamburger_history_menu: + test_the_most_recent_website_is_present_in_history_menu: pass + test_private_window_website_not_in_history: + test_opened_website_in_private_window_not_captured_in_history_list: pass + test_toggle_bookmarks_toolbar: + test_toggle_bookmark_toolbar: pass + test_user_can_forget_history: + test_user_can_forget_history: pass +downloads: + test_add_mime_type_doc: + test_mime_type_doc: pass + test_add_zip_type: + test_add_zip_type: pass + test_download_pdf: + test_download_pdf: pass + test_download_pdf_from_context_menu: + test_download_pdf_from_context_menu: pass + test_mixed_content_download_via_https: + test_mixed_content_download_via_https: unstable + test_set_always_ask_file_type: + test_set_always_ask_file_type: pass +drag_and_drop: + test_copy_entire_row_column: + test_copy_entire_row_column: pass + test_copy_from_an_editor_paste_in_another: + test_copy_from_an_editor_paste_in_another: pass + test_copy_hyperlink_table: + test_copy_table_with_hyperlink: pass + test_copy_table_header: + test_copy_table_header: pass + test_paste_image_text: + test_paste_image_text: pass +find_toolbar: + test_find_in_pdf: + test_find_in_pdf_using_key_combos: pass + test_find_toolbar_nav: + test_find_toolbar_navigation: pass + test_find_toolbar_search: + test_find_toolbar_search: pass +form_autofill: + test_address_autofill_attribute: + test_address_attribute_selection: pass + test_autofill_cc_cvv: + test_autofill_cc_cvv: pass + test_autofill_credit_card: + test_autofill_credit_card: pass + test_autofill_credit_card_doorhanger: + test_autofill_credit_card_door_hanger: pass + test_autofill_credit_card_enable: + test_enable_disable_form_autofill_cc: pass + test_autofill_credit_card_four_fields: + test_autofill_four_fields: pass + test_cc_clear_form: + test_clear_form_credit_card: pass + test_clear_form: + test_clear_form: pass + test_create_new_cc_profile: + test_create_new_cc_profile: pass + test_create_profile_autofill: + test_create_address_profile: pass + test_delete_cc_profile: + test_delete_cc_profile: pass + test_edit_credit_card: + test_edit_credit_card_profile: pass + test_enable_disable_autofill: + test_enable_disable_autofill: pass + test_form_autofill_suggestions: + test_form_autofill_suggestions: pass + test_name_autofill_attribute: + test_name_attribute_selection: pass + test_private_mode_info_not_saved: + test_private_mode_info_not_saved: pass + test_telephone_autofill_attribute: + test_telephone_attribute_autofill: pass + test_updating_address: + test_update_address: pass + test_updating_credit_card: + test_update_cc_no_dupe_name: pass +geolocation: + test_geolocation_shared_via_html5: + test_allow_permission_on_geolocation_via_html5: pass + test_block_permission_on_geolocation_via_w3c_api: pass + test_geolocation_shared_via_w3c_api: + test_allow_permission_on_geolocation_via_w3c_api: pass + test_block_permission_on_geolocation_via_w3c_api: pass +language_packs: + test_language_pack_install_addons: + test_language_pack_install_from_addons: pass + test_language_pack_install_preferences: + test_language_pack_install_about_preferences: pass +menus: + test_copy_paste_actions: + test_login_form_copy_paste: pass + test_search_field_copy_paste: pass + test_text_area_copy_paste: pass + test_frequently_used_context_menu: + test_inspect: pass + test_save_page_as: pass + test_take_screenshot: pass + test_hyperlink_context_menu: + test_open_link_in_new_window: pass + test_image_context_menu_actions: + test_copy_image_link: pass + test_open_image_in_new_tab: pass + test_save_image_as: pass + test_tab_context_menu_actions: + test_close_multiple_tabs_to_right: pass + test_duplicate_tab: pass + test_tab_context_menu_close: + test_close_multiple_tabs_other_tabs: pass + test_close_multiple_tabs_to_left: pass + test_copy_link: pass +meta: + test_selectors: + test_a_selector: pass + test_version: + test_version: pass +networking: + test_default_dns_protection: + test_doh_enforces_secure_dns_resolution: pass + test_http_site: + test_http_site: pass +notifications: + test_audio_video_permissions_notification: + test_camera_and_microphone_permissions_notification: pass + test_camera_permissions_notification: + test_camera_permissions_notification: pass + test_deny_geolocation: + test_deny_geolocation: pass + test_deny_screen_capture: + test_deny_screen_capture: pass + test_geolocation_prompt_presence: + test_geolocation_prompt_is_triggered_on_request_location_on_a_website: pass + test_microphone_permissions_notification: + test_microphone_permissions_notification: pass + test_notifications_displayed: + test_notifications_displayed: pass + test_screen_share_permission_prompt: + test_screen_share_permission_prompt: pass + test_webextension_completed_installation_successfully_displayed: + test_webextension_completed_installation_successfully_displayed: pass +password_manager: + test_about_logins_navigation_from_context_menu: + test_about_logins_navigation_from_login_form_context_menu: pass + test_about_logins_navigation_from_hamburger_menu: + test_about_logins_navigation_from_password_hamburger_menu: pass + test_about_logins_search_username: + test_about_logins_search_username: pass + test_about_logins_search_website: + test_about_logins_search_website: pass + test_add_password_non_ascii_chars: + test_add_password_non_ascii_chars: pass + test_add_password_save_valid_data: + test_add_password_save_valid_data: pass + test_add_primary_password: + test_add_primary_password: pass + test_auto_saved_generated_password_context_menu: + test_auto_saved_generated_password_context_menu: pass + test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load: + test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load: pass + test_can_view_password_when_PP_enabled: + test_password_can_be_shown: pass + test_changes_made_in_edit_mode_are_saved: + test_changes_made_in_edit_mode_are_saved: pass + test_delete_login: + test_delete_login: pass + test_multiple_saved_logins: + test_multiple_saved_logins: pass + test_never_save_login_via_doorhanger: + test_never_save_login_via_doorhanger: pass + test_password_csv_correctness: + test_password_csv_correctness: pass + test_password_csv_export: + test_password_csv_export: pass + test_primary_password_triggered_on_about_logins_access: + test_primary_password_triggered_on_about_logins_access_via_hamburger_menu: pass + test_save_login_via_doorhanger: + test_save_login_via_doorhanger: pass + test_saved_hyperlink_redirects_to_corresponding_page: + test_saved_hyperlink_redirects_to_corresponding_page: pass + test_update_login_via_doorhanger: + test_update_login_via_doorhanger: pass +pdf_viewer: + test_add_image_pdf: + test_add_image_pdf: pass + test_download_pdf_with_form_fields: + test_download_pdf_with_form_fields: pass + test_download_triggered_on_content_disposition_attachment: + test_download_panel_triggered_on_content_disposition_attachment: pass + test_open_pdf_in_FF: + test_open_pdf_in_fx: pass + test_pdf_data_can_be_cleared: + test_pdf_data_can_be_cleared: pass + test_pdf_download: + test_pdf_download: pass + test_pdf_input_numbers: + test_pdf_input_numbers: pass + test_pdf_navigation: + test_navigation_keys: pass + test_navigation_next_prev: pass + test_navigation_page_numbers: pass + test_toolbar_options_cursor: pass + test_toolbar_options_rotate_ccw: pass + test_toolbar_options_rotate_cw: pass + test_zoom_pdf_viewer: + test_zoom_pdf_viewer_keys: pass + test_zoom_pdf_viewer_toolbar: pass +pocket: + test_basic_de: + test_localized_pocket_layout_DE: pass + test_basic_fr: + test_localized_pocket_layout_FR: pass + test_basic_gb: + test_localized_pocket_layout_GB: pass + test_basic_us: + test_localized_pocket_layout_US: pass + test_new_tab_about_blank: pass +preferences: + test_check_for_updates: + test_check_for_updates: pass + test_clear_cookie_data: + test_clear_cookie_data: pass + test_firefox_home_new_tabs: + test_firefox_home_new_tab: pass + test_firefox_home_on_launch: + test_firefox_home_on_launch: pass + test_lang_pack_changed_from_about_prefs: + test_lang_pack_changed_from_about_prefs: pass + test_manage_cookie_data: + test_manage_cookie_data: pass + test_never_remember_history: + test_never_remember_history: pass + test_notifications_change_set: + test_notifications_allow: pass +printing_ui: + test_print_preview: + test_print_preview_keys: pass + test_print_preview_menu: pass + test_print_to_pdf: + test_print_to_pdf: pass +profile: + test_set_default_profile: + test_set_default_profile: pass +reader_view: + test_improved_type_control_panel: + test_type_control_panel_character_spacing: pass + test_type_control_panel_content_width: pass + test_type_control_panel_font: pass + test_type_control_panel_line_spacing: pass + test_type_control_panel_size: pass + test_type_control_panel_text_alignment: pass + test_type_control_panel_word_spacing: pass + test_reader_view_close_sidebar: + test_reader_view_close_from_sidebar: pass + test_reader_view_location_bar: + test_reader_view_open_close_using_keys: pass + test_reader_view_open_close_using_searchbar: pass +scrolling_panning_zooming: + test_default_zoom_persists: + test_default_zoom_across_tabs: pass + test_mouse_wheel_zoom: + test_mouse_wheel_zoom: pass + test_zoom_from_menu_bar: + test_zoom_from_menu_bar: pass + test_zoom_menu_correlation: + test_zoom_level_div_position: pass + test_zoom_text_only: + test_zoom_text_only_after_restart: pass + test_zoom_text_only_from_settings: pass +security_and_privacy: + test_blocking_cryptominers: + test_blocking_cryptominers: pass + test_blocking_fingerprinters: + test_blocking_fingerprinter: pass + test_blocking_social_media_trackers: + test_blocking_social_media_trackers: pass + test_cookies_not_saved_private_browsing: + test_cookies_not_saved_private_browsing: pass + test_cross_site_tracking_cookies_blocked: + test_cross_site_tracking_cookies_blocked: pass + test_cryptominers_blocked_and_shown_in_info_panel: + test_cryptominers_blocked_and_shown_in_info_panel: pass + test_detected_blocked_trackers_found: + test_detected_blocked_trackers_found: pass + test_downloads_from_private_not_leaked: + test_downloads_from_private_not_leaked: pass + test_https_enabled_private_browsing: + test_https_first_mode_in_private_browsing: pass + test_never_remember_browsing_history: + test_never_remember_browsing_history_from_panel: pass + test_never_remember_browsing_history_settings: pass + test_no_trackers_detected: + test_no_trackers_detected: pass + test_open_link_in_private_window: + test_open_link_in_private_window: pass + test_phishing_and_malware_warnings: + test_phishing_and_malware_warning_errors: pass + test_private_browser_password_doorhanger: + test_no_password_doorhanger_private_browsing: pass + test_private_session_awesome_bar_exclusion: + test_websites_visited_in_private_browser_not_displayed_in_awesome_bar: pass + test_private_session_history_exclusion: + test_websites_visited_in_private_browser_not_displayed_in_history: pass + test_private_window_from_panelui: + test_private_window_from_panelui: pass + test_third_party_content_blocked_private_browsing: + test_third_party_content_blocked_private_browsing_allowed_tracking: pass + test_third_party_content_blocked_private_browsing_cross_site: pass + test_third_party_content_private_browsing_tracking_statuses: pass + test_tls_v1_2_protocol: + test_tls_v1_2_protocol: pass + test_trackers_crypto_fingerprint_blocked: + test_cross_site_trackrs_crypto_fingerprinter_blocked: pass + test_tracking_content_custom_mode: + test_allowed_tracking_content: pass + test_blocked_tracking_content: pass + test_undo_close_tab_private_browsing: + test_undo_close_tab_private_browsing: pass +sync_and_fxa: + test_existing_fxa: + test_sync_existing_fxa: pass + test_new_fxa: + test_sync_new_fxa: pass +tabs: + test_active_tab: + test_active_tab: pass + test_change_position_of_pinned_tabs: + test_change_position_of_pinned_tabs: pass + test_close_pinned_tab_via_mouse: + test_close_pinned_tab_via_middle_click: pass + test_close_tab_through_middle_mouse_click: + test_close_tab_through_middle_mouse_click: pass + test_display_customize_button: + test_customize_button_displayed_in_tab_bar: pass + test_list_all_tabs: + test_list_all_tabs: pass + test_mute_tabs: + test_mute_unmute_tab: pass + test_navigation_multiple_tabs: + test_navigation_multiple_tabs: pass + test_open_bookmark_in_new_tab: + test_open_bookmark_in_new_tab: pass + test_open_new_bg_tab_via_mouse_and_keyboard: + test_open_new_bg_tab_via_mouse_and_keyboard: pass + test_open_new_tab: + test_open_new_tab_plus: pass + test_open_new_tab_keys: + test_open_new_tab_via_keyboard: pass + test_open_new_tab_via_hyperlink: + test_open_new_via_hyperlink: pass + test_pin_tab: + test_pin_tab: pass + test_pin_unpin_selected_tabs: + test_pin_unpin_selected_tabs: pass + test_play_mute_unmute_tabs_via_toggle: + test_play_mute_unmute_tabs_via_toggle: pass + test_reload_overiding_cache_keys: + test_reload_overiding_cache_keys: pass + test_reload_tab_via_keyboard: + test_reload_tab_via_keyboard: pass + test_reopen_tab_through_context_menu: + test_reopen_tab_through_context_menu: pass + test_reopen_tab_through_history_menu: + test_reopen_tab_through_history_menu: pass + test_reopen_tabs_through_keys: + test_reopen_tabs_through_keys: pass +theme_and_toolbar: + test_customize_themes_and_redirect: + test_activate_theme_background_matches_expected: pass + test_alpenglow_theme: pass + test_redirect_to_addons: pass + test_installed_theme_enabled: + test_find_more_themes: pass + test_installed_theme_enabled: pass From 79d69bb38e57eefb5700bb287c882e59eb5eebc7 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Tue, 18 Nov 2025 10:40:04 -0800 Subject: [PATCH 17/29] remove unstable mark and rely on manifests --- ci_pyproject.toml | 5 +---- ci_pyproject_headed.toml | 5 +---- ci_xvfb_pyproject.toml | 5 +---- ci_xvfb_pyproject_headed.toml | 5 +---- .../test_google_search_counts_us.py | 12 +++++------- .../test_google_withads_url_bar_us.py | 9 ++++----- .../test_sap_google_adclick.py | 2 +- .../test_search_modes_for_sites.py | 3 ++- .../test_search_term_persists.py | 2 +- .../test_allow_audio_video_functionality.py | 8 +------- .../test_import_bookmarks_edge.py | 11 +---------- tests/downloads/test_add_mime_type_doc.py | 8 +------- .../test_mixed_content_download_via_https.py | 5 +++-- .../test_geolocation_shared_via_html5.py | 15 ++++++--------- .../test_geolocation_shared_via_w3c_api.py | 15 ++++++--------- .../test_frequently_used_context_menu.py | 6 +----- ...st_audio_video_permissions_notification.py | 8 +++----- .../test_camera_permissions_notification.py | 8 +++----- ...est_microphone_permissions_notification.py | 8 +++----- .../test_password_csv_correctness.py | 8 +++++--- .../test_password_csv_export.py | 5 +++-- tests/pdf_viewer/test_add_image_pdf.py | 3 +-- tests/printing_ui/test_print_to_pdf.py | 6 +----- .../test_mouse_wheel_zoom.py | 19 +++++++++---------- tests/tabs/test_mute_tabs.py | 6 +----- .../test_play_mute_unmute_tabs_via_toggle.py | 8 +++++--- .../test_reopen_tab_through_history_menu.py | 5 +---- .../test_installed_theme_enabled.py | 11 ++--------- 28 files changed, 73 insertions(+), 138 deletions(-) diff --git a/ci_pyproject.toml b/ci_pyproject.toml index 8548738f5..c178ae8e1 100644 --- a/ci_pyproject.toml +++ b/ci_pyproject.toml @@ -5,17 +5,14 @@ log_cli_level = "warn" markers = [ "audio: test is reliant on audio", "headed: test must run in headed mode (e.g. uses pynput)", - "incident: incident smoke tests", - "unstable: temporary mark for unstable tests", "slow: test is clocked at more than 30s on modern machines", - "ci: basic tests to run in ci", "locale_de: tests run in DE locale versions", "locale_fr: tests run in FR locale versions", "locale_gb: tests run in GB locale versions", "noxvfb: tests that should not run in xvfb sessions" ] -addopts = "-vs --ci --run-headless --json-report --json-report-file artifacts/report.json --reruns 3 -n auto --reruns-delay 3 -m 'not incident and not unstable and not headed' --html=artifacts/report.html" +addopts = "-vs --ci --run-headless --json-report --json-report-file artifacts/report.json --reruns 3 -n auto --reruns-delay 3 -m 'not headed' --html=artifacts/report.html" [tool.ruff] target-version = "py310" diff --git a/ci_pyproject_headed.toml b/ci_pyproject_headed.toml index 5916a6a44..281f8ceb2 100644 --- a/ci_pyproject_headed.toml +++ b/ci_pyproject_headed.toml @@ -5,16 +5,13 @@ log_cli_level = "warn" markers = [ "audio: test is reliant on audio", "headed: test must run in headed mode (e.g. uses pynput)", - "incident: incident smoke tests", - "unstable: temporary mark for unstable tests", "slow: test is clocked at more than 30s on modern machines", - "ci: basic tests to run in ci", "locale_de: tests run in DE locale versions", "locale_fr: tests run in FR locale versions", "locale_gb: tests run in GB locale versions", "noxvfb: tests that should not run in xvfb sessions" ] -addopts = "-vs --ci --json-report --json-report-file artifacts/report_headed.json --reruns 3 --reruns-delay 2 -m 'not incident and not unstable and headed' --html=artifacts/report_headed.html" +addopts = "-vs --ci --json-report --json-report-file artifacts/report_headed.json --reruns 3 --reruns-delay 2 -m 'headed' --html=artifacts/report_headed.html" [tool.ruff] target-version = "py310" diff --git a/ci_xvfb_pyproject.toml b/ci_xvfb_pyproject.toml index 11f51f8f1..1f8dd217c 100644 --- a/ci_xvfb_pyproject.toml +++ b/ci_xvfb_pyproject.toml @@ -5,16 +5,13 @@ log_cli_level = "warn" markers = [ "audio: test is reliant on audio", "headed: test must run in headed mode (e.g. uses pynput)", - "incident: incident smoke tests", - "unstable: temporary mark for unstable tests", "slow: test is clocked at more than 30s on modern machines", - "ci: basic tests to run in ci", "locale_de: tests run in DE locale versions", "locale_fr: tests run in FR locale versions", "locale_gb: tests run in GB locale versions", "noxvfb: tests that should not run in xvfb sessions" ] -addopts = "-vs --ci --run-headless --json-report --json-report-file artifacts/report.json -n auto --reruns 3 --reruns-delay 3 -m 'not unstable and not headed and not noxvfb' --html=artifacts/report.html" +addopts = "-vs --ci --run-headless --json-report --json-report-file artifacts/report.json -n auto --reruns 3 --reruns-delay 3 -m 'not headed and not noxvfb' --html=artifacts/report.html" [tool.ruff] target-version = "py310" diff --git a/ci_xvfb_pyproject_headed.toml b/ci_xvfb_pyproject_headed.toml index 7b5483c05..660e6f1a9 100644 --- a/ci_xvfb_pyproject_headed.toml +++ b/ci_xvfb_pyproject_headed.toml @@ -5,16 +5,13 @@ log_cli_level = "warn" markers = [ "audio: test is reliant on audio", "headed: test must run in headed mode (e.g. uses pynput)", - "incident: incident smoke tests", - "unstable: temporary mark for unstable tests", "slow: test is clocked at more than 30s on modern machines", - "ci: basic tests to run in ci", "locale_de: tests run in DE locale versions", "locale_fr: tests run in FR locale versions", "locale_gb: tests run in GB locale versions", "noxvfb: tests that should not run in xvfb sessions" ] -addopts = "-vs --ci --json-report --json-report-file artifacts/report_headed.json -n auto --reruns 3 --reruns-delay 2 -m 'not unstable and not noxvfb and headed' --html=artifacts/report_headed.html" +addopts = "-vs --ci --json-report --json-report-file artifacts/report_headed.json -n auto --reruns 3 --reruns-delay 2 -m 'not noxvfb and headed' --html=artifacts/report_headed.html" [tool.ruff] target-version = "py310" diff --git a/tests/address_bar_and_search/test_google_search_counts_us.py b/tests/address_bar_and_search/test_google_search_counts_us.py index 893a4f06a..a7edd9637 100644 --- a/tests/address_bar_and_search/test_google_search_counts_us.py +++ b/tests/address_bar_and_search/test_google_search_counts_us.py @@ -1,5 +1,3 @@ -import sys -from os import environ from time import sleep import pytest @@ -16,15 +14,13 @@ WAIT_AFTER_SEARCH = 5 WAIT_TELEMETRY_LOAD = 2 -MAC_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("darwin") - @pytest.fixture() def test_case(): return "3029528" -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in macOS GitHub Actions") +# This test is unstable in MacOS GHA for now def test_google_search_counts_us(driver: Firefox): """ C1365026 - Verify Google search counts in telemetry from the URL bar (US region). @@ -53,9 +49,11 @@ def test_google_search_counts_us(driver: Firefox): sleep(2) continue else: - pytest.fail("CAPTCHA triggered repeatedly. Giving up after 5 attempts.") + pytest.fail( + "CAPTCHA triggered repeatedly. Giving up after 5 attempts.") - provider_ok = utils.assert_json_value(json_data, SEARCH_PROVIDER_PATH, 1) + provider_ok = utils.assert_json_value( + json_data, SEARCH_PROVIDER_PATH, 1) tag_ok = utils.assert_json_value(json_data, SEARCH_TAG_PATH, 1) if provider_ok and tag_ok: diff --git a/tests/address_bar_and_search/test_google_withads_url_bar_us.py b/tests/address_bar_and_search/test_google_withads_url_bar_us.py index dd06b786a..3933906f0 100644 --- a/tests/address_bar_and_search/test_google_withads_url_bar_us.py +++ b/tests/address_bar_and_search/test_google_withads_url_bar_us.py @@ -8,8 +8,6 @@ from modules.page_object import AboutTelemetry from modules.util import Utilities -MAC_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("darwin") - @pytest.fixture() def test_case(): @@ -21,8 +19,8 @@ def add_to_prefs_list(): return [("cookiebanners.service.mode", 1)] -@pytest.mark.unstable(reason="Google re-captcha") -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Google re-captcha makes test unstable for now +# Mac GHA makes test unstable for now def test_google_withads_url_bar_us(driver): """ C1365070 - Retry up to 5 times if Google CAPTCHA blocks telemetry path. @@ -44,7 +42,8 @@ def test_google_withads_url_bar_us(driver): sleep(2) continue else: - pytest.fail("CAPTCHA triggered repeatedly. Giving up after 5 attempts.") + pytest.fail( + "CAPTCHA triggered repeatedly. Giving up after 5 attempts.") about_telemetry = AboutTelemetry(driver).open() sleep(5) diff --git a/tests/address_bar_and_search/test_sap_google_adclick.py b/tests/address_bar_and_search/test_sap_google_adclick.py index 0c6caf75e..395b643db 100644 --- a/tests/address_bar_and_search/test_sap_google_adclick.py +++ b/tests/address_bar_and_search/test_sap_google_adclick.py @@ -17,7 +17,7 @@ def test_case(): return "3029662" -@pytest.mark.unstable(reason="Google re-captcha") +# Google recaptcha makes this test unstable for now def test_sap_google_adclick(driver: Firefox): """ C1365108 - Verify Google ad click from URL bar is recorded in telemetry (US region). diff --git a/tests/address_bar_and_search/test_search_modes_for_sites.py b/tests/address_bar_and_search/test_search_modes_for_sites.py index 56fed538e..2a67223ae 100644 --- a/tests/address_bar_and_search/test_search_modes_for_sites.py +++ b/tests/address_bar_and_search/test_search_modes_for_sites.py @@ -17,7 +17,8 @@ def test_case(): return "3028754" -@pytest.mark.unstable(reason="Google re-captcha and manual tagged this test as removed") +# Google recaptcha makes this test unstable for now +# Test has been removed from main TestRail suites @pytest.mark.parametrize("search_engine, prefix, url", SEARCH_MODES) def test_search_modes_for_sites( driver: Firefox, search_engine: str, prefix: str, url: str diff --git a/tests/address_bar_and_search/test_search_term_persists.py b/tests/address_bar_and_search/test_search_term_persists.py index 37e332d9d..f409fc361 100644 --- a/tests/address_bar_and_search/test_search_term_persists.py +++ b/tests/address_bar_and_search/test_search_term_persists.py @@ -24,7 +24,7 @@ def add_to_prefs_list(): SECOND_SEARCH = "lion" -@pytest.mark.unstable +# Google re-captcha makes this test unstable for now def test_search_term_persists(driver: Firefox): """ C2153943 - Persist search term basic functionality diff --git a/tests/audio_video/test_allow_audio_video_functionality.py b/tests/audio_video/test_allow_audio_video_functionality.py index 9826a0dc8..23e8178d8 100644 --- a/tests/audio_video/test_allow_audio_video_functionality.py +++ b/tests/audio_video/test_allow_audio_video_functionality.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox @@ -14,13 +11,10 @@ def test_case(): return "330155" -WIN_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") TEST_URL = "https://www.w3schools.com/html/mov_bbb.mp4" -@pytest.mark.skipif( - WIN_GHA, reason="Audio playback not supported in Windows CI environment" -) +# Test is unstable in Windows GHA because audio playback is not allowed @pytest.mark.audio @pytest.mark.noxvfb def test_allow_audio_video_functionality(driver: Firefox): diff --git a/tests/bookmarks_and_history/test_import_bookmarks_edge.py b/tests/bookmarks_and_history/test_import_bookmarks_edge.py index 779d52896..6e34a58ed 100644 --- a/tests/bookmarks_and_history/test_import_bookmarks_edge.py +++ b/tests/bookmarks_and_history/test_import_bookmarks_edge.py @@ -1,5 +1,4 @@ import os -import sys from shutil import copyfile import pytest @@ -46,15 +45,7 @@ def edge_bookmarks(sys_platform, home_folder): return target -@pytest.mark.skipif( - sys.platform.lower().startswith("linux"), - reason="Only testing Edge on Win and MacOS", -) -@pytest.mark.skipif( - os.environ.get("GITHUB_ACTIONS") == "true" - and not sys.platform.lower().startswith("win"), - reason="No Edge on GHA Mac", -) +# Test should only be run on Windows machines. def test_edge_bookmarks_imported(driver: Firefox, edge_bookmarks, sys_platform): about_prefs = AboutPrefs(driver, category="General") about_prefs.open() diff --git a/tests/downloads/test_add_mime_type_doc.py b/tests/downloads/test_add_mime_type_doc.py index 96492fd46..1102ceb16 100644 --- a/tests/downloads/test_add_mime_type_doc.py +++ b/tests/downloads/test_add_mime_type_doc.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox @@ -14,7 +11,6 @@ def test_case(): DOC_LINK = "https://sapphire-hendrika-5.tiiny.site/" -WIN_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") @pytest.fixture() @@ -28,9 +24,7 @@ def expected_app_name(sys_platform: str, opt_ci: bool) -> str: return "LibreOffice Writer" -@pytest.mark.skipif( - WIN_GHA, reason="Most runners don’t have a .doc handler registered on Windows" -) +# Test is unstable on Windows GHA because the image does not have a .doc handler @pytest.mark.noxvfb def test_mime_type_doc(driver: Firefox, sys_platform: str, opt_ci: bool, delete_files): """ diff --git a/tests/downloads/test_mixed_content_download_via_https.py b/tests/downloads/test_mixed_content_download_via_https.py index e3ceb24ee..3a919bb81 100644 --- a/tests/downloads/test_mixed_content_download_via_https.py +++ b/tests/downloads/test_mixed_content_download_via_https.py @@ -21,7 +21,7 @@ def delete_files_regex_string(): MAX_CHECKS = 30 -@pytest.mark.unstable(reason="Unstable in CI environment") +# This test has been found to be unstable in CI def test_mixed_content_download_via_https(driver: Firefox, delete_files): """ C1756722: Verify that the user can download mixed content via HTTPS @@ -32,7 +32,8 @@ def test_mixed_content_download_via_https(driver: Firefox, delete_files): # Wait for the test website to wake up and download the content web_page.open() - web_page.wait.until(lambda _: nav.element_visible("download-target-element")) + web_page.wait.until(lambda _: nav.element_visible( + "download-target-element")) # Verify download name matches expected pattern nav.verify_download_name(r"file-sample_100kB(\(\d+\))?.odt$") diff --git a/tests/geolocation/test_geolocation_shared_via_html5.py b/tests/geolocation/test_geolocation_shared_via_html5.py index 971b48337..b25f12b9b 100644 --- a/tests/geolocation/test_geolocation_shared_via_html5.py +++ b/tests/geolocation/test_geolocation_shared_via_html5.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox from selenium.webdriver.common.by import By @@ -28,24 +25,24 @@ def add_to_prefs_list(): TEST_URL = "https://browserleaks.com/geo" -WIN_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") - def wait_for_geolocation_data(web_page, timeout=20): """Wait until both latitude and longitude data are available.""" web_page.custom_wait(timeout=timeout).until( lambda _: all( [ - web_page.find_element(By.ID, "latitude").get_attribute("data-raw") + web_page.find_element( + By.ID, "latitude").get_attribute("data-raw") is not None, - web_page.find_element(By.ID, "longitude").get_attribute("data-raw") + web_page.find_element( + By.ID, "longitude").get_attribute("data-raw") is not None, ] ) ) -@pytest.mark.skipif(WIN_GHA, reason="Recent permission changes at their side") +# Test is unstable on Windows GHA because of permission changes on the CI image def test_allow_permission_on_geolocation_via_html5(driver: Firefox): """ C15189 - Verify that geolocation is successfully shared when the user allows permission via the HTML5 Geolocation API @@ -77,7 +74,7 @@ def test_allow_permission_on_geolocation_via_html5(driver: Firefox): assert permission_icon.is_displayed() -@pytest.mark.skipif(WIN_GHA, reason="Recent permission changes at their side") +# Test is unstable on Windows GHA because of permission changes on the CI image def test_block_permission_on_geolocation_via_w3c_api(driver: Firefox): """ C15189 - Verify that geolocation is not shared when the user blocks permission via the HTML5 Geolocation API diff --git a/tests/geolocation/test_geolocation_shared_via_w3c_api.py b/tests/geolocation/test_geolocation_shared_via_w3c_api.py index 4b76e4638..c73c2074c 100644 --- a/tests/geolocation/test_geolocation_shared_via_w3c_api.py +++ b/tests/geolocation/test_geolocation_shared_via_w3c_api.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox @@ -27,8 +24,6 @@ def add_to_prefs_list(): TEST_URL = "https://www.w3schools.com/html/html5_geolocation.asp" -WIN_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") - @pytest.fixture() def temp_selectors(): @@ -51,7 +46,7 @@ def temp_selectors(): } -@pytest.mark.skipif(WIN_GHA, reason="Recent permission changes at their side") +# Test is unstable on Windows GHA because of permission changes on the CI image def test_allow_permission_on_geolocation_via_w3c_api(driver: Firefox, temp_selectors): """ C15186 - Verify that geolocation is successfully shared when the user allows permission via the W3C Geolocation API @@ -83,7 +78,8 @@ def test_allow_permission_on_geolocation_via_w3c_api(driver: Firefox, temp_selec # Click the 'Try It' button and Allow the location sharing while choose the option Remember this decision web_page.click_on("geolocation-button-selector") - nav.handle_geolocation_prompt(button_type="primary", remember_this_decision=True) + nav.handle_geolocation_prompt( + button_type="primary", remember_this_decision=True) # Check that the location marker is displayed # if map is displayed, style attribute will be available @@ -97,7 +93,7 @@ def test_allow_permission_on_geolocation_via_w3c_api(driver: Firefox, temp_selec assert permission_icon.is_displayed() -@pytest.mark.skipif(WIN_GHA, reason="Recent permission changes at their side") +# Test is unstable on Windows GHA because of permission changes on the CI image def test_block_permission_on_geolocation_via_w3c_api(driver: Firefox, temp_selectors): """ C15186 - Verify that geolocation is not shared when the user blocks permission via the W3C Geolocation API @@ -126,7 +122,8 @@ def test_block_permission_on_geolocation_via_w3c_api(driver: Firefox, temp_selec # Click the 'Try It' button and Block the location sharing while choose the option Remember this decision tabs.open_single_page_in_new_tab(web_page, num_tabs=2) web_page.click_on("geolocation-button-selector") - nav.handle_geolocation_prompt(button_type="secondary", remember_this_decision=True) + nav.handle_geolocation_prompt( + button_type="secondary", remember_this_decision=True) # Check that the location marker is displayed # if map is not displayed, style attribute will not be available diff --git a/tests/menus/test_frequently_used_context_menu.py b/tests/menus/test_frequently_used_context_menu.py index dfe0ed72d..1cdb94a67 100644 --- a/tests/menus/test_frequently_used_context_menu.py +++ b/tests/menus/test_frequently_used_context_menu.py @@ -1,6 +1,5 @@ import os import platform -import sys from time import sleep import pytest @@ -16,10 +15,7 @@ def test_case(): return "2637623" -WIN_GHA = os.environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") - - -@pytest.mark.skipif(WIN_GHA, reason="Test unstable in Windows Github Actions") +# Test is unstable in Windows GHA for now @pytest.mark.headed def test_save_page_as(driver: Firefox): """ diff --git a/tests/notifications/test_audio_video_permissions_notification.py b/tests/notifications/test_audio_video_permissions_notification.py index da53480aa..9c3414ffe 100644 --- a/tests/notifications/test_audio_video_permissions_notification.py +++ b/tests/notifications/test_audio_video_permissions_notification.py @@ -1,5 +1,3 @@ -import sys -from os import environ from time import sleep import pytest @@ -32,10 +30,9 @@ def add_to_prefs_list(): TEST_URL = "https://mozilla.github.io/webrtc-landing/gum_test.html" -MAC_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("darwin") -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Test is unstable in MacOS GHA for now def test_camera_and_microphone_permissions_notification( driver: Firefox, temp_selectors ): @@ -52,7 +49,8 @@ def test_camera_and_microphone_permissions_notification( # Verify that the notification is displayed nav.element_visible("popup-notification") - nav.expect_element_attribute_contains("popup-notification", "label", "Allow ") + nav.expect_element_attribute_contains( + "popup-notification", "label", "Allow ") nav.expect_element_attribute_contains( "popup-notification", "name", "mozilla.github.io" ) diff --git a/tests/notifications/test_camera_permissions_notification.py b/tests/notifications/test_camera_permissions_notification.py index aacba7e48..4c4e9fa9d 100644 --- a/tests/notifications/test_camera_permissions_notification.py +++ b/tests/notifications/test_camera_permissions_notification.py @@ -1,5 +1,3 @@ -import sys -from os import environ from time import sleep import pytest @@ -32,10 +30,9 @@ def add_to_prefs_list(): TEST_URL = "https://mozilla.github.io/webrtc-landing/gum_test.html" -MAC_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("darwin") -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Test is unstable in MacOS GHA for now def test_camera_permissions_notification(driver: Firefox, temp_selectors): """ C122536 - Verify that Camera only permission prompt is successfully displayed when the website asks for camera permissions @@ -49,7 +46,8 @@ def test_camera_permissions_notification(driver: Firefox, temp_selectors): web_page.click_on("camera-only") # Verify that the notification is displayed - nav.expect_element_attribute_contains("popup-notification", "label", "Allow ") + nav.expect_element_attribute_contains( + "popup-notification", "label", "Allow ") nav.expect_element_attribute_contains( "popup-notification", "name", "mozilla.github.io" ) diff --git a/tests/notifications/test_microphone_permissions_notification.py b/tests/notifications/test_microphone_permissions_notification.py index 423e807e1..a3073d1b8 100644 --- a/tests/notifications/test_microphone_permissions_notification.py +++ b/tests/notifications/test_microphone_permissions_notification.py @@ -1,5 +1,3 @@ -import sys -from os import environ from time import sleep import pytest @@ -32,10 +30,9 @@ def add_to_prefs_list(): TEST_URL = "https://mozilla.github.io/webrtc-landing/gum_test.html" -MAC_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("darwin") -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Test is unstable in MacOS GHA for now def test_microphone_permissions_notification(driver: Firefox, temp_selectors): """ C122539 - Verify that Microphone only permission prompt is successfully displayed when the website asks for microphone permissions @@ -50,7 +47,8 @@ def test_microphone_permissions_notification(driver: Firefox, temp_selectors): # Verify that the notification is displayed nav.element_visible("popup-notification") - nav.expect_element_attribute_contains("popup-notification", "label", "Allow ") + nav.expect_element_attribute_contains( + "popup-notification", "label", "Allow ") nav.expect_element_attribute_contains( "popup-notification", "name", "mozilla.github.io" ) diff --git a/tests/password_manager/test_password_csv_correctness.py b/tests/password_manager/test_password_csv_correctness.py index 10cf7ee5c..ba279b223 100644 --- a/tests/password_manager/test_password_csv_correctness.py +++ b/tests/password_manager/test_password_csv_correctness.py @@ -14,7 +14,7 @@ def test_case(): return "2241522" -@pytest.mark.unstable(reason="Bug 1996004") +# This test is unstable for now: Bug 1996004 @pytest.mark.headed @pytest.mark.noxvfb def test_password_csv_correctness( @@ -34,7 +34,8 @@ def test_password_csv_correctness( about_logins.export_passwords_csv(downloads_folder, "passwords.csv") # Verify the exported csv file is present in the target folder - csv_file = about_logins.verify_csv_export(downloads_folder, "passwords.csv") + csv_file = about_logins.verify_csv_export( + downloads_folder, "passwords.csv") assert os.path.exists(csv_file) # Verify the contents of the exported csv file @@ -46,7 +47,8 @@ def test_password_csv_correctness( reader = csv.DictReader(pw) actual_logins = {} for row in reader: - actual_logins[row["username"] + "@" + row["url"][8:]] = row["password"] + actual_logins[row["username"] + "@" + + row["url"][8:]] = row["password"] assert re.match(guid_pattern, row["guid"]) assert re.match(time_pattern, row["timeCreated"]) assert re.match(time_pattern, row["timeLastUsed"]) diff --git a/tests/password_manager/test_password_csv_export.py b/tests/password_manager/test_password_csv_export.py index 576b1e9a1..7c4b61847 100644 --- a/tests/password_manager/test_password_csv_export.py +++ b/tests/password_manager/test_password_csv_export.py @@ -13,7 +13,7 @@ def test_case(): return "2241521" -@pytest.mark.unstable(reason="Bug 1996005") +# This test is unstable for now: Bug 1996005 @pytest.mark.headed @pytest.mark.noxvfb def test_password_csv_export( @@ -36,7 +36,8 @@ def test_password_csv_export( keyboard.tap(Key.enter) # Verify the exported csv file is present in the target folder - csv_file = about_logins.verify_csv_export(downloads_folder, "passwords.csv") + csv_file = about_logins.verify_csv_export( + downloads_folder, "passwords.csv") assert os.path.exists(csv_file) # Delete the password.csv created diff --git a/tests/pdf_viewer/test_add_image_pdf.py b/tests/pdf_viewer/test_add_image_pdf.py index 4174697bb..a5df7e415 100644 --- a/tests/pdf_viewer/test_add_image_pdf.py +++ b/tests/pdf_viewer/test_add_image_pdf.py @@ -1,5 +1,4 @@ from pathlib import Path -from platform import system from time import sleep import pytest @@ -33,7 +32,7 @@ def add_to_prefs_list(): ] -@pytest.mark.skipif(system().lower().startswith("linux"), reason="Bug 1983852") +# This test is unstable in Linux (all CI) for now: Bug 1983852 @pytest.mark.headed @pytest.mark.noxvfb def test_add_image_pdf(driver: Firefox, sys_platform, pdf_viewer: GenericPdf): diff --git a/tests/printing_ui/test_print_to_pdf.py b/tests/printing_ui/test_print_to_pdf.py index 09df9b887..69218a8f7 100644 --- a/tests/printing_ui/test_print_to_pdf.py +++ b/tests/printing_ui/test_print_to_pdf.py @@ -1,6 +1,5 @@ import logging import os -from platform import system import pytest from selenium.webdriver import Firefox @@ -45,10 +44,7 @@ def file_is_somewhere(): return False -@pytest.mark.skipif( - system().lower().startswith("win") or system().lower().startswith("linux"), - reason="Bug 1974011", -) +# Test is unstable in Windows GHA and Linux Taskcluster for now: Bug 1974011 @pytest.mark.headed def test_print_to_pdf( driver: Firefox, diff --git a/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py b/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py index ef5c0bdf6..e3adacb7e 100644 --- a/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py +++ b/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py @@ -1,5 +1,4 @@ import logging -import platform import time import pytest @@ -19,11 +18,7 @@ def test_case(): TEST_PAGE = "https://www.example.com" -# Skip this test if running on macOS -@pytest.mark.skipif( - platform.system() == "Darwin", - reason="Test skipped on macOS due to incompatible zoom controls", -) +# This test is not compatible with MacOS wheel controls def test_mouse_wheel_zoom(driver: Firefox): """ This test verifies that the X-coordinate of a
element's position @@ -38,7 +33,8 @@ def test_mouse_wheel_zoom(driver: Firefox): # Locate the main
element on the page div = driver.find_element(By.TAG_NAME, "div") - initial_position = div.location["x"] # Get the initial X position of the div + # Get the initial X position of the div + initial_position = div.location["x"] logging.info(f"Initial X position of div: {initial_position}") # Initialize ActionChains for zooming with Ctrl + Mouse Wheel @@ -55,7 +51,8 @@ def test_mouse_wheel_zoom(driver: Firefox): # Switch to chrome context to check zoom level in the toolbar with driver.context(driver.CONTEXT_CHROME): zoom_button = nav.get_element("toolbar-zoom-level") - zoom_level = nav.get_element("toolbar-zoom-level").get_attribute("label") + zoom_level = nav.get_element( + "toolbar-zoom-level").get_attribute("label") logging.info(f"Zoom level after zoom-in: {zoom_level}") # Assert that the zoom level label is "110%" after zooming in @@ -71,7 +68,8 @@ def test_mouse_wheel_zoom(driver: Firefox): # **Step 2**: Reset zoom to 100% using the keyboard shortcut (Ctrl + 0) with driver.context(driver.CONTEXT_CHROME): - actions.key_down(Keys.CONTROL).send_keys("0").key_up(Keys.CONTROL).perform() + actions.key_down(Keys.CONTROL).send_keys( + "0").key_up(Keys.CONTROL).perform() time.sleep(1) # Allow time for reset effect to take place reset_position = driver.find_element(By.TAG_NAME, "div").location["x"] logging.info(f"X position of div after zoom-reset: {reset_position}") @@ -111,5 +109,6 @@ def test_mouse_wheel_zoom(driver: Firefox): # Reset the zoom level back to 100% with driver.context(driver.CONTEXT_CHROME): - actions.key_down(Keys.CONTROL).send_keys("0").key_up(Keys.CONTROL).perform() + actions.key_down(Keys.CONTROL).send_keys( + "0").key_up(Keys.CONTROL).perform() time.sleep(1) # Allow time for reset effect to take place diff --git a/tests/tabs/test_mute_tabs.py b/tests/tabs/test_mute_tabs.py index 80aa03f9c..0fac924db 100644 --- a/tests/tabs/test_mute_tabs.py +++ b/tests/tabs/test_mute_tabs.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox from selenium.webdriver.support import expected_conditions as EC @@ -11,7 +8,6 @@ COOKIE_CONSENT_SELECTOR = ( "button[aria-label^='Accept all'], button[aria-label^='Accept the use']" ) -WIN_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") @pytest.fixture() @@ -24,7 +20,7 @@ def add_to_prefs_list(): return [("network.cookie.cookieBehavior", "2")] -@pytest.mark.skipif(WIN_GHA, reason="Test unstable on Windows in Github Actions") +# This test is unstable in Windows GHA for now @pytest.mark.audio def test_mute_unmute_tab(screenshot, driver: Firefox, video_url: str): """C134719, test that tabs can be muted and unmuted""" diff --git a/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py b/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py index b4762e811..d6df4bf4c 100644 --- a/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py +++ b/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py @@ -26,9 +26,9 @@ def add_to_prefs_list(): return [("network.cookie.cookieBehavior", "2")] +# This test is unstable in Windows GHA for now @pytest.mark.audio @pytest.mark.headed -@pytest.mark.skipif(GHA, reason="Test unstable in Github Actions") def test_play_mute_unmute_tabs_via_toggle(driver: Firefox, sys_platform: str): """ C246981 - Verify that play/mute/unmute tabs via toggle audio works @@ -48,7 +48,8 @@ def test_play_mute_unmute_tabs_via_toggle(driver: Firefox, sys_platform: str): # Locate and open 2 latest videos in new tabs video_selector = "ytd-rich-item-renderer a#video-title-link" video_links = wait.until( - EC.visibility_of_all_elements_located((By.CSS_SELECTOR, video_selector)) + EC.visibility_of_all_elements_located( + (By.CSS_SELECTOR, video_selector)) ) for i in range(2): @@ -125,7 +126,8 @@ def click_multi_tab_audio_button(): for i in [2, 3]: tabs.expect_tab_sound_status(i, tabs.MEDIA_STATUS.MUTED) tab = tabs.get_tab(i) - assert tab.get_attribute("muted") is not None, f"Tab {i} should be muted" + assert tab.get_attribute( + "muted") is not None, f"Tab {i} should be muted" # Click Unmute button click_multi_tab_audio_button() diff --git a/tests/tabs/test_reopen_tab_through_history_menu.py b/tests/tabs/test_reopen_tab_through_history_menu.py index 53b8f37ab..09e8fe6d6 100644 --- a/tests/tabs/test_reopen_tab_through_history_menu.py +++ b/tests/tabs/test_reopen_tab_through_history_menu.py @@ -1,5 +1,4 @@ import logging -from platform import system import pytest from selenium.webdriver import Firefox @@ -26,9 +25,7 @@ def test_case(): NUM_TABS = 6 -@pytest.mark.skipif( - system().lower().startswith("linux"), reason="Currently unstable in linux" -) +# This test is unstable in Linux Taskcluster for now def test_reopen_tab_through_history_menu(driver: Firefox): """C134650 - Verify that the recently closed tab can be reopened from the history menu""" diff --git a/tests/theme_and_toolbar/test_installed_theme_enabled.py b/tests/theme_and_toolbar/test_installed_theme_enabled.py index ec39cc802..42cbb006f 100644 --- a/tests/theme_and_toolbar/test_installed_theme_enabled.py +++ b/tests/theme_and_toolbar/test_installed_theme_enabled.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox @@ -12,15 +9,11 @@ def test_case(): return "118174" -MAC_GHA: bool = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith( - "darwin" -) - AMO_HOST: str = "addons.mozilla.org" AMO_THEMES_PATH: str = "firefox/themes" -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Test is unstable in MacOS GHA for now def test_find_more_themes(driver: Firefox) -> None: """ C118174 (part 1): From about:addons > Themes, 'Find more themes' opens AMO in a new tab. @@ -37,7 +30,7 @@ def test_find_more_themes(driver: Firefox) -> None: base.url_contains(AMO_THEMES_PATH) -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Test is unstable in MacOS GHA for now def test_installed_theme_enabled(driver: Firefox) -> None: """ C118174 (part 2): Install a recommended theme from AMO and ensure it becomes enabled immediately. From 9c7a0fd6f911c1ab3adceed09c93334b10511387 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Wed, 19 Nov 2025 16:08:54 -0800 Subject: [PATCH 18/29] single source of pass/fail truth --- choose_ci_set.py | 49 +- manifests/all.yaml | 495 ------------- manifests/ci.yaml | 55 +- manifests/incident.yaml | 314 ++++---- manifests/key.yaml | 218 ++++++ manifests/smoke.yaml | 684 ++++++------------ .../test_google_search_counts_us.py | 6 +- .../test_google_withads_url_bar_us.py | 3 +- .../test_mixed_content_download_via_https.py | 3 +- .../test_geolocation_shared_via_html5.py | 6 +- .../test_geolocation_shared_via_w3c_api.py | 6 +- ...st_audio_video_permissions_notification.py | 3 +- .../test_camera_permissions_notification.py | 3 +- ...est_microphone_permissions_notification.py | 3 +- .../test_password_csv_correctness.py | 6 +- .../test_password_csv_export.py | 3 +- .../test_mouse_wheel_zoom.py | 9 +- .../test_play_mute_unmute_tabs_via_toggle.py | 6 +- 18 files changed, 631 insertions(+), 1241 deletions(-) delete mode 100644 manifests/all.yaml create mode 100644 manifests/key.yaml diff --git a/choose_ci_set.py b/choose_ci_set.py index bd7464319..2ee7d0f2a 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -7,6 +7,8 @@ import yaml CI_MANIFEST = "manifests/ci.yaml" +MANIFEST_KEY = "manifests/key.yaml" +SUPPORTED_OSES = ["mac", "win", "linux"] HEADED_MARK = "@pytest.mark.headed" MIN_RUN_SIZE = 7 OUTPUT_FILE = "selected_tests" @@ -111,38 +113,41 @@ def sysname(): def convert_manifest_to_list(manifest_loc): manifest = yaml.safe_load(open(manifest_loc)) + mkey = yaml.safe_load(open(MANIFEST_KEY)) toplevel = [".", "tests"] tests = [] if manifest: print(f"Reading {manifest_loc}") for suite in manifest: print(suite) - if suite == "add_manifest": - for linked_manifest in manifest[suite]: - tests.extend( - convert_manifest_to_list(SLASH.join(["manifests", linked_manifest])) - ) - for test in manifest[suite]: - # print(f" {test}") + if suite not in mkey: + print(f"{suite} not in {MANIFEST_KEY}") + continue + + for testfile in manifest[suite]: + print(f" {testfile}") addtest = False - test_name = f"{test}.py" - if manifest[suite][test] == "pass": + test_name = f"{testfile}.py" + if testfile not in mkey[suite]: + print(f"{suite}/{testfile} not in {MANIFEST_KEY}::{suite}") + continue + if mkey[suite][testfile] == "pass": addtest = True - elif isinstance(manifest[suite][test], dict): - # print(f"==={manifest[suite][test].get(sysname())}===") - if manifest[suite][test].get(sysname()) == "pass": - addtest = True + elif isinstance(mkey[suite][testfile], dict): + if any([x in mkey[suite][testfile] for x in SUPPORTED_OSES]): + if mkey[suite][testfile][sysname()] == "pass": + addtest = True else: - for subtest in manifest[suite][test]: - if subtest in ["mac", "win", "linux"]: - continue - # print(f" {subtest}") - test_name = f"{test}.py::{subtest}" - if isinstance(manifest[suite][test][subtest], dict): - if manifest[suite][test][subtest].get(sysname()) == "pass": - addtest = True - elif manifest[suite][test][subtest] == "pass": + for subtest in mkey[suite][testfile]: + if mkey[suite][testfile][subtest] == "pass": + test_name = f"{test_name}::{subtest}" addtest = True + elif isinstance(mkey[suite][testfile][subtest], dict): + if sysname() in mkey[suite][testfile][subtest]: + if mkey[suite][testfile][subtest][sysname()] == "pass": + test_name = f"{test_name}::{subtest}" + addtest = True + if addtest: test_to_add = SLASH.join(toplevel + [suite, test_name]) assert os.path.exists(test_to_add.split("::")[0]), ( diff --git a/manifests/all.yaml b/manifests/all.yaml deleted file mode 100644 index b8b24774c..000000000 --- a/manifests/all.yaml +++ /dev/null @@ -1,495 +0,0 @@ -address_bar_and_search: - test_adaptive_history_autofill: - test_add_adaptive_history_autofill: pass - test_add_engine_address_bar: - test_add_search_engine_from_address_bar: pass - test_added_open_search_engine_default: - test_added_open_search_engine_default: pass - test_addon_suggestion: - test_addon_suggestion_based_on_search_input: pass - test_addressbar_search_engine_keywords: - test_addressbar_search_engine_keywords: pass - test_copied_url_contains_https: - test_copied_url_contains_https: pass - test_ctrl_enter_fixes_url: - test_ctrl_enter_fixes_url: pass - test_default_search_provider_change_awesome_bar: - test_default_search_provider_change_awesome_bar: pass - test_default_search_provider_change_legacy_search_bar: - test_default_search_provider_change_legacy_search_bar: pass - test_dont_show_search_suggestions_in_private_window: - test_no_search_engine_suggestions_in_private_window: pass - test_google_search_counts_us: - test_google_search_counts_us: pass - test_google_withads_url_bar_us: - test_google_withads_url_bar_us: pass - test_insertion_point_no_search_terms_display: - test_insertion_point_no_search_terms_display: pass - test_no_suggestions_for_empty_query: - test_suggestions_for_empty_query_not_shown_in_search_mode: pass - test_non_sponsored_topsite_context_menu_option: - test_non_sponsored_topsite_context_menu_option: pass - test_refresh_firefox_dialog: - test_refresh_firefox_dialog: pass - test_sap_google_adclick: - test_sap_google_adclick: unstable - test_seach_suggestions_can_be_disabled: - test_search_suggestions_pref_affects_urlbar_and_searchbar: pass - test_search_code_google_non_us: - test_search_code_google_non_us: pass - test_search_code_google_us: - test_search_code_google_us: pass - test_search_engine_selector: - test_search_engine_selector_and_validator: pass - test_search_mode_appears_after_input: - test_private_mode_repeat_after_enabling_pref: pass - test_search_mode_appears_and_suggestions_update: pass - test_search_mode_persists_mixed_with_bing: - test_search_mode_persists_mixed_with_bing: pass - test_search_modes_for_sites: - test_search_modes_for_sites: pass - test_search_string_displayed_when_addressbar_unfocused: - test_address_bar_and_search_string_displayed_when_addressbar_unfocused: pass - test_search_suggestions: - test_search_suggests_enabled: pass - test_search_term_persists: - test_search_term_persists: unstable - test_searchbar_display_alpenglow_theme: - test_searchbar_display_alpenglow_theme: pass - test_searchbar_on_engines_remove_restore: - test_searchbar_engine_remove_restore_affects_both_windows: pass - test_searchbar_results_shown_in_a_new_tab: - test_search_bar_results_shown_in_a_new_tab: pass - test_searchmode_change_tab: - test_searchmode_change_tab: pass - test_server_not_found_error: - test_server_not_found_error: pass - test_server_not_found_error_pb: - test_server_not_found_error_on_private_window: pass - test_suggestion_engine_selection: - test_search_suggestion_for_engine_selection: pass - test_tile_menu_options: - test_default_tile_hover_states: pass - test_tile_context_menu_options: pass -audio_video: - test_allow_audio_video_functionality: - test_allow_audio_video_functionality: pass - test_block_audio_video_functionality: - test_block_audio_video_functionality: pass - test_users_actions_saved_on_reload: - test_users_actions_saved_on_reload: pass -bookmarks_and_history: - test_add_new_other_bookmark: - test_add_new_other_bookmark: pass - test_bookmark_via_bookmark_menu: - test_bookmark_via_hamburger_menu: pass - test_bookmark_website_via_star_button: - test_bookmark_website_via_star: pass - test_clear_all_history: - test_clear_all_history: pass - test_clear_recent_history_displayed: - test_clear_recent_history_displayed: pass - test_delete_bookmarks_from_toolbar: - test_delete_bookmarks_from_toolbar: pass - test_delete_other_bookmarks: - test_delete_other_bookmarks: pass - test_deleted_page_not_remembered: - test_deleted_page_not_remembered: pass - test_edit_bookmark_from_bookmark_menu: - test_edit_bookmark_from_bookmark_menu: pass - test_edit_bookmark_via_star_button: - test_edit_bookmark_via_star_button: pass - test_history_menu_from_different_places: - test_history_menu_in_different_places: pass - test_import_bookmarks_chrome: - test_chrome_bookmarks_imported: pass - test_import_bookmarks_edge: - test_edge_bookmarks_imported: pass - test_open_all_bookmarks_from_bookmarks_toolbar: - test_open_all_bookmarks_from_bookmarks_toolbar: pass - test_open_bookmark_in_new_window_via_toolbar_context_menu: - test_open_bookmark_in_new_window_via_toolbar_context_menu: pass - test_open_bookmark_in_private_window_via_toolbar_context_menu: - test_open_bookmark_in_new_private_window_via_toolbar_context_menu: pass - test_open_bookmarks_from_toolbar: - test_open_bookmarks_from_toolbar: pass - test_open_websites_from_history: - test_open_websites_from_history: pass - test_opened_website_in_new_tab_present_in_hamburger_history_menu: - test_the_website_opened_in_new_tab_is_present_in_history_menu: pass - test_opened_website_in_new_window_present_in_hamburger_history_menu: - test_the_website_opened_in_new_window_is_present_in_history_menu: pass - test_opened_website_present_in_hamburger_history_menu: - test_the_most_recent_website_is_present_in_history_menu: pass - test_private_window_website_not_in_history: - test_opened_website_in_private_window_not_captured_in_history_list: pass - test_toggle_bookmarks_toolbar: - test_toggle_bookmark_toolbar: pass - test_user_can_forget_history: - test_user_can_forget_history: pass -downloads: - test_add_mime_type_doc: - test_mime_type_doc: pass - test_add_zip_type: - test_add_zip_type: pass - test_download_pdf: - test_download_pdf: pass - test_download_pdf_from_context_menu: - test_download_pdf_from_context_menu: pass - test_mixed_content_download_via_https: - test_mixed_content_download_via_https: unstable - test_set_always_ask_file_type: - test_set_always_ask_file_type: pass -drag_and_drop: - test_copy_entire_row_column: - test_copy_entire_row_column: pass - test_copy_from_an_editor_paste_in_another: - test_copy_from_an_editor_paste_in_another: pass - test_copy_hyperlink_table: - test_copy_table_with_hyperlink: pass - test_copy_table_header: - test_copy_table_header: pass - test_paste_image_text: - test_paste_image_text: pass -find_toolbar: - test_find_in_pdf: - test_find_in_pdf_using_key_combos: pass - test_find_toolbar_nav: - test_find_toolbar_navigation: pass - test_find_toolbar_search: - test_find_toolbar_search: pass -form_autofill: - test_address_autofill_attribute: - test_address_attribute_selection: pass - test_autofill_cc_cvv: - test_autofill_cc_cvv: pass - test_autofill_credit_card: - test_autofill_credit_card: pass - test_autofill_credit_card_doorhanger: - test_autofill_credit_card_door_hanger: pass - test_autofill_credit_card_enable: - test_enable_disable_form_autofill_cc: pass - test_autofill_credit_card_four_fields: - test_autofill_four_fields: pass - test_cc_clear_form: - test_clear_form_credit_card: pass - test_clear_form: - test_clear_form: pass - test_create_new_cc_profile: - test_create_new_cc_profile: pass - test_create_profile_autofill: - test_create_address_profile: pass - test_delete_cc_profile: - test_delete_cc_profile: pass - test_edit_credit_card: - test_edit_credit_card_profile: pass - test_enable_disable_autofill: - test_enable_disable_autofill: pass - test_form_autofill_suggestions: - test_form_autofill_suggestions: pass - test_name_autofill_attribute: - test_name_attribute_selection: pass - test_private_mode_info_not_saved: - test_private_mode_info_not_saved: pass - test_telephone_autofill_attribute: - test_telephone_attribute_autofill: pass - test_updating_address: - test_update_address: pass - test_updating_credit_card: - test_update_cc_no_dupe_name: pass -geolocation: - test_geolocation_shared_via_html5: - test_allow_permission_on_geolocation_via_html5: pass - test_block_permission_on_geolocation_via_w3c_api: pass - test_geolocation_shared_via_w3c_api: - test_allow_permission_on_geolocation_via_w3c_api: pass - test_block_permission_on_geolocation_via_w3c_api: pass -language_packs: - test_language_pack_install_addons: - test_language_pack_install_from_addons: pass - test_language_pack_install_preferences: - test_language_pack_install_about_preferences: pass -menus: - test_copy_paste_actions: - test_login_form_copy_paste: pass - test_search_field_copy_paste: pass - test_text_area_copy_paste: pass - test_frequently_used_context_menu: - test_inspect: pass - test_save_page_as: pass - test_take_screenshot: pass - test_hyperlink_context_menu: - test_open_link_in_new_window: pass - test_image_context_menu_actions: - test_copy_image_link: pass - test_open_image_in_new_tab: pass - test_save_image_as: pass - test_tab_context_menu_actions: - test_close_multiple_tabs_to_right: pass - test_duplicate_tab: pass - test_tab_context_menu_close: - test_close_multiple_tabs_other_tabs: pass - test_close_multiple_tabs_to_left: pass - test_copy_link: pass -meta: - test_selectors: - test_a_selector: pass - test_version: - test_version: pass -networking: - test_default_dns_protection: - test_doh_enforces_secure_dns_resolution: pass - test_http_site: - test_http_site: pass -notifications: - test_audio_video_permissions_notification: - test_camera_and_microphone_permissions_notification: pass - test_camera_permissions_notification: - test_camera_permissions_notification: pass - test_deny_geolocation: - test_deny_geolocation: pass - test_deny_screen_capture: - test_deny_screen_capture: pass - test_geolocation_prompt_presence: - test_geolocation_prompt_is_triggered_on_request_location_on_a_website: pass - test_microphone_permissions_notification: - test_microphone_permissions_notification: pass - test_notifications_displayed: - test_notifications_displayed: pass - test_screen_share_permission_prompt: - test_screen_share_permission_prompt: pass - test_webextension_completed_installation_successfully_displayed: - test_webextension_completed_installation_successfully_displayed: pass -password_manager: - test_about_logins_navigation_from_context_menu: - test_about_logins_navigation_from_login_form_context_menu: pass - test_about_logins_navigation_from_hamburger_menu: - test_about_logins_navigation_from_password_hamburger_menu: pass - test_about_logins_search_username: - test_about_logins_search_username: pass - test_about_logins_search_website: - test_about_logins_search_website: pass - test_add_password_non_ascii_chars: - test_add_password_non_ascii_chars: pass - test_add_password_save_valid_data: - test_add_password_save_valid_data: pass - test_add_primary_password: - test_add_primary_password: pass - test_auto_saved_generated_password_context_menu: - test_auto_saved_generated_password_context_menu: pass - test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load: - test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load: pass - test_can_view_password_when_PP_enabled: - test_password_can_be_shown: pass - test_changes_made_in_edit_mode_are_saved: - test_changes_made_in_edit_mode_are_saved: pass - test_delete_login: - test_delete_login: pass - test_multiple_saved_logins: - test_multiple_saved_logins: pass - test_never_save_login_via_doorhanger: - test_never_save_login_via_doorhanger: pass - test_password_csv_correctness: - test_password_csv_correctness: pass - test_password_csv_export: - test_password_csv_export: pass - test_primary_password_triggered_on_about_logins_access: - test_primary_password_triggered_on_about_logins_access_via_hamburger_menu: pass - test_save_login_via_doorhanger: - test_save_login_via_doorhanger: pass - test_saved_hyperlink_redirects_to_corresponding_page: - test_saved_hyperlink_redirects_to_corresponding_page: pass - test_update_login_via_doorhanger: - test_update_login_via_doorhanger: pass -pdf_viewer: - test_add_image_pdf: - test_add_image_pdf: pass - test_download_pdf_with_form_fields: - test_download_pdf_with_form_fields: pass - test_download_triggered_on_content_disposition_attachment: - test_download_panel_triggered_on_content_disposition_attachment: pass - test_open_pdf_in_FF: - test_open_pdf_in_fx: pass - test_pdf_data_can_be_cleared: - test_pdf_data_can_be_cleared: pass - test_pdf_download: - test_pdf_download: pass - test_pdf_input_numbers: - test_pdf_input_numbers: pass - test_pdf_navigation: - test_navigation_keys: pass - test_navigation_next_prev: pass - test_navigation_page_numbers: pass - test_toolbar_options_cursor: pass - test_toolbar_options_rotate_ccw: pass - test_toolbar_options_rotate_cw: pass - test_zoom_pdf_viewer: - test_zoom_pdf_viewer_keys: pass - test_zoom_pdf_viewer_toolbar: pass -pocket: - test_basic_de: - test_localized_pocket_layout_DE: pass - test_basic_fr: - test_localized_pocket_layout_FR: pass - test_basic_gb: - test_localized_pocket_layout_GB: pass - test_basic_us: - test_localized_pocket_layout_US: pass - test_new_tab_about_blank: pass -preferences: - test_check_for_updates: - test_check_for_updates: pass - test_clear_cookie_data: - test_clear_cookie_data: pass - test_firefox_home_new_tabs: - test_firefox_home_new_tab: pass - test_firefox_home_on_launch: - test_firefox_home_on_launch: pass - test_lang_pack_changed_from_about_prefs: - test_lang_pack_changed_from_about_prefs: pass - test_manage_cookie_data: - test_manage_cookie_data: pass - test_never_remember_history: - test_never_remember_history: pass - test_notifications_change_set: - test_notifications_allow: pass -printing_ui: - test_print_preview: - test_print_preview_keys: pass - test_print_preview_menu: pass - test_print_to_pdf: - test_print_to_pdf: pass -profile: - test_set_default_profile: - test_set_default_profile: pass -reader_view: - test_improved_type_control_panel: - test_type_control_panel_character_spacing: pass - test_type_control_panel_content_width: pass - test_type_control_panel_font: pass - test_type_control_panel_line_spacing: pass - test_type_control_panel_size: pass - test_type_control_panel_text_alignment: pass - test_type_control_panel_word_spacing: pass - test_reader_view_close_sidebar: - test_reader_view_close_from_sidebar: pass - test_reader_view_location_bar: - test_reader_view_open_close_using_keys: pass - test_reader_view_open_close_using_searchbar: pass -scrolling_panning_zooming: - test_default_zoom_persists: - test_default_zoom_across_tabs: pass - test_mouse_wheel_zoom: - test_mouse_wheel_zoom: pass - test_zoom_from_menu_bar: - test_zoom_from_menu_bar: pass - test_zoom_menu_correlation: - test_zoom_level_div_position: pass - test_zoom_text_only: - test_zoom_text_only_after_restart: pass - test_zoom_text_only_from_settings: pass -security_and_privacy: - test_blocking_cryptominers: - test_blocking_cryptominers: pass - test_blocking_fingerprinters: - test_blocking_fingerprinter: pass - test_blocking_social_media_trackers: - test_blocking_social_media_trackers: pass - test_cookies_not_saved_private_browsing: - test_cookies_not_saved_private_browsing: pass - test_cross_site_tracking_cookies_blocked: - test_cross_site_tracking_cookies_blocked: pass - test_cryptominers_blocked_and_shown_in_info_panel: - test_cryptominers_blocked_and_shown_in_info_panel: pass - test_detected_blocked_trackers_found: - test_detected_blocked_trackers_found: pass - test_downloads_from_private_not_leaked: - test_downloads_from_private_not_leaked: pass - test_https_enabled_private_browsing: - test_https_first_mode_in_private_browsing: pass - test_never_remember_browsing_history: - test_never_remember_browsing_history_from_panel: pass - test_never_remember_browsing_history_settings: pass - test_no_trackers_detected: - test_no_trackers_detected: pass - test_open_link_in_private_window: - test_open_link_in_private_window: pass - test_phishing_and_malware_warnings: - test_phishing_and_malware_warning_errors: pass - test_private_browser_password_doorhanger: - test_no_password_doorhanger_private_browsing: pass - test_private_session_awesome_bar_exclusion: - test_websites_visited_in_private_browser_not_displayed_in_awesome_bar: pass - test_private_session_history_exclusion: - test_websites_visited_in_private_browser_not_displayed_in_history: pass - test_private_window_from_panelui: - test_private_window_from_panelui: pass - test_third_party_content_blocked_private_browsing: - test_third_party_content_blocked_private_browsing_allowed_tracking: pass - test_third_party_content_blocked_private_browsing_cross_site: pass - test_third_party_content_private_browsing_tracking_statuses: pass - test_tls_v1_2_protocol: - test_tls_v1_2_protocol: pass - test_trackers_crypto_fingerprint_blocked: - test_cross_site_trackrs_crypto_fingerprinter_blocked: pass - test_tracking_content_custom_mode: - test_allowed_tracking_content: pass - test_blocked_tracking_content: pass - test_undo_close_tab_private_browsing: - test_undo_close_tab_private_browsing: pass -sync_and_fxa: - test_existing_fxa: - test_sync_existing_fxa: pass - test_new_fxa: - test_sync_new_fxa: pass -tabs: - test_active_tab: - test_active_tab: pass - test_change_position_of_pinned_tabs: - test_change_position_of_pinned_tabs: pass - test_close_pinned_tab_via_mouse: - test_close_pinned_tab_via_middle_click: pass - test_close_tab_through_middle_mouse_click: - test_close_tab_through_middle_mouse_click: pass - test_display_customize_button: - test_customize_button_displayed_in_tab_bar: pass - test_list_all_tabs: - test_list_all_tabs: pass - test_mute_tabs: - test_mute_unmute_tab: pass - test_navigation_multiple_tabs: - test_navigation_multiple_tabs: pass - test_open_bookmark_in_new_tab: - test_open_bookmark_in_new_tab: pass - test_open_new_bg_tab_via_mouse_and_keyboard: - test_open_new_bg_tab_via_mouse_and_keyboard: pass - test_open_new_tab: - test_open_new_tab_plus: pass - test_open_new_tab_keys: - test_open_new_tab_via_keyboard: pass - test_open_new_tab_via_hyperlink: - test_open_new_via_hyperlink: pass - test_pin_tab: - test_pin_tab: pass - test_pin_unpin_selected_tabs: - test_pin_unpin_selected_tabs: pass - test_play_mute_unmute_tabs_via_toggle: - test_play_mute_unmute_tabs_via_toggle: pass - test_reload_overiding_cache_keys: - test_reload_overiding_cache_keys: pass - test_reload_tab_via_keyboard: - test_reload_tab_via_keyboard: pass - test_reopen_tab_through_context_menu: - test_reopen_tab_through_context_menu: pass - test_reopen_tab_through_history_menu: - test_reopen_tab_through_history_menu: pass - test_reopen_tabs_through_keys: - test_reopen_tabs_through_keys: pass -theme_and_toolbar: - test_customize_themes_and_redirect: - test_activate_theme_background_matches_expected: pass - test_alpenglow_theme: pass - test_redirect_to_addons: pass - test_installed_theme_enabled: - test_find_more_themes: pass - test_installed_theme_enabled: pass diff --git a/manifests/ci.yaml b/manifests/ci.yaml index 920179845..9594d6eea 100644 --- a/manifests/ci.yaml +++ b/manifests/ci.yaml @@ -1,54 +1,33 @@ address_bar_and_search: - test_default_search_provider_change_awesome_bar: - test_default_search_provider_change_awesome_bar: pass + - test_default_search_provider_change_awesome_bar bookmarks_and_history: - test_bookmark_website_via_star_button: - test_bookmark_website_via_star: pass + - test_bookmark_website_via_star_button downloads: - test_set_always_ask_file_type: - test_set_always_ask_file_type: pass + - test_set_always_ask_file_type find_toolbar: - test_find_toolbar_search: - test_find_toolbar_search: pass + - test_find_toolbar_search form_autofill: - test_telephone_autofill_attribute: - test_telephone_attribute_autofill: pass + - test_telephone_autofill_attribute menus: - test_copy_paste_actions: - test_login_form_copy_paste: pass - test_image_context_menu_actions: - test_save_image_as: pass + - test_copy_paste_actions + - test_image_context_menu_actions networking: - test_http_site: - test_http_site: pass + - test_http_site notifications: - test_notifications_displayed: - test_notifications_displayed: pass + - test_notifications_displayed password_manager: - test_about_logins_navigation_from_context_menu: - test_about_logins_navigation_from_login_form_context_menu: pass + - test_about_logins_navigation_from_context_menu pdf_viewer: - test_pdf_data_can_be_cleared: - test_pdf_data_can_be_cleared: pass + - test_pdf_data_can_be_cleared printing_ui: - test_print_preview: - test_print_preview_keys: pass + - test_print_preview reader_view: - test_improved_type_control_panel: - test_type_control_panel_font[sans-serif]: pass - test_improved_type_control_panel: - test_type_control_panel_font[serif]: pass - test_improved_type_control_panel: - test_type_control_panel_font[monospace]: pass + - test_improved_type_control_panel scrolling_panning_zooming: - test_zoom_text_only: - test_zoom_text_only_from_settings: pass + - test_zoom_text_only security_and_privacy: - test_cross_site_tracking_cookies_blocked: - test_cross_site_tracking_cookies_blocked: pass + - test_cross_site_tracking_cookies_blocked tabs: - test_active_tab: - test_active_tab: pass + - test_active_tab theme_and_toolbar: - test_customize_themes_and_redirect: - test_redirect_to_addons: pass + - test_customize_themes_and_redirect diff --git a/manifests/incident.yaml b/manifests/incident.yaml index 6b8231ace..774f2c3d4 100644 --- a/manifests/incident.yaml +++ b/manifests/incident.yaml @@ -1,199 +1,159 @@ ---- address_bar_and_search: - test_add_engine_address_bar: pass - test_addon_suggestion: - win: unstable - linux: pass - mac: unstable - test_default_search_provider_change_awesome_bar: unstable - test_google_search_counts_us: - win: pass - linux: pass - mac: unstable - test_google_withads_url_bar_us: - win: pass - linux: pass - mac: unstable - test_sap_google_adclick: unstable - test_search_code_google_non_us: pass - test_search_code_google_us: pass - test_search_engine_selector: pass - test_search_suggestions: unstable - test_search_term_persists: pass - test_server_not_found_error: pass - test_suggestion_engine_selection: pass +- test_add_engine_address_bar +- test_addon_suggestion +- test_default_search_provider_change_awesome_bar +- test_google_search_counts_us +- test_google_withads_url_bar_us +- test_sap_google_adclick +- test_search_code_google_non_us +- test_search_code_google_us +- test_search_engine_selector +- test_search_suggestions +- test_search_term_persists +- test_server_not_found_error +- test_suggestion_engine_selection audio_video: - test_allow_audio_video_functionality: - win: unstable - mac: pass - linux: unstable - test_block_audio_video_functionality: pass - test_users_actions_saved_on_reload: pass +- test_allow_audio_video_functionality +- test_block_audio_video_functionality +- test_users_actions_saved_on_reload bookmarks_and_history: - test_add_new_other_bookmark: pass - test_bookmark_via_bookmark_menu: pass - test_bookmark_website_via_star_button: pass - test_clear_all_history: pass - test_clear_recent_history_displayed: pass - test_delete_bookmarks_from_toolbar: pass - test_delete_other_bookmarks: pass - test_deleted_page_not_remembered: pass - test_open_bookmark_in_new_window_via_toolbar_context_menu: pass - test_open_bookmark_in_private_window_via_toolbar_context_menu: pass - test_open_bookmarks_from_toolbar: pass - test_open_websites_from_history: pass - test_private_window_website_not_in_history: pass - test_toggle_bookmarks_toolbar: pass - test_user_can_forget_history: pass +- test_add_new_other_bookmark +- test_bookmark_via_bookmark_menu +- test_bookmark_website_via_star_button +- test_clear_all_history +- test_clear_recent_history_displayed +- test_delete_bookmarks_from_toolbar +- test_delete_other_bookmarks +- test_deleted_page_not_remembered +- test_open_bookmark_in_new_window_via_toolbar_context_menu +- test_open_bookmark_in_private_window_via_toolbar_context_menu +- test_open_bookmarks_from_toolbar +- test_open_websites_from_history +- test_private_window_website_not_in_history +- test_toggle_bookmarks_toolbar +- test_user_can_forget_history downloads: - test_add_zip_type: - win: pass - mac: pass - linux: unstable - test_download_pdf: pass - test_download_pdf_from_context_menu: unstable +- test_add_zip_type +- test_download_pdf +- test_download_pdf_from_context_menu drag_and_drop: - test_copy_entire_row_column: pass +- test_copy_entire_row_column find_toolbar: - test_find_in_pdf: pass - test_find_toolbar_nav: pass - test_find_toolbar_search: pass +- test_find_in_pdf +- test_find_toolbar_nav +- test_find_toolbar_search form_autofill: - test_address_autofill_attribute: pass - test_autofill_cc_cvv: pass - test_autofill_credit_card: pass - test_autofill_credit_card_doorhanger: pass - test_autofill_credit_card_enable: pass - test_autofill_credit_card_four_fields: pass - test_cc_clear_form: pass - test_clear_form: pass - test_create_new_cc_profile: pass - test_create_profile_autofill: pass - test_delete_cc_profile: pass - test_enable_disable_autofill: pass - test_form_autofill_suggestions: pass - test_name_autofill_attribute: pass - test_private_mode_info_not_saved: pass - test_telephone_autofill_attribute: pass - test_updating_address: pass +- test_address_autofill_attribute +- test_autofill_cc_cvv +- test_autofill_credit_card +- test_autofill_credit_card_doorhanger +- test_autofill_credit_card_enable +- test_autofill_credit_card_four_fields +- test_cc_clear_form +- test_clear_form +- test_create_new_cc_profile +- test_create_profile_autofill +- test_delete_cc_profile +- test_enable_disable_autofill +- test_form_autofill_suggestions +- test_name_autofill_attribute +- test_private_mode_info_not_saved +- test_telephone_autofill_attribute +- test_updating_address geolocation: - test_geolocation_shared_via_html5: - win: unstable - mac: pass - linux: pass - test_geolocation_shared_via_w3c_api: - win: unstable - mac: pass - linux: pass +- test_geolocation_shared_via_html5 +- test_geolocation_shared_via_w3c_api language_packs: - test_language_pack_install_addons: pass - test_language_pack_install_preferences: pass +- test_language_pack_install_addons +- test_language_pack_install_preferences menus: - test_frequently_used_context_menu: - win: unstable - mac: pass - linux: pass - test_hyperlink_context_menu: pass - test_tab_context_menu_actions: pass - test_tab_context_menu_close: pass +- test_frequently_used_context_menu +- test_hyperlink_context_menu +- test_tab_context_menu_actions +- test_tab_context_menu_close networking: - test_default_dns_protection: pass - test_http_site: pass +- test_default_dns_protection +- test_http_site notifications: - test_audio_video_permissions_notification: - win: pass - mac: unstable - linux: pass - test_camera_permissions_notification: - win: pass - mac: unstable - linux: pass - test_deny_geolocation: pass - test_deny_screen_capture: pass - test_geolocation_prompt_presence: pass - test_microphone_permissions_notification: - win: pass - mac: unstable - linux: pass - test_notifications_displayed: pass - test_screen_share_permission_prompt: pass - test_webextension_completed_installation_successfully_displayed: pass +- test_audio_video_permissions_notification +- test_camera_permissions_notification +- test_deny_geolocation +- test_deny_screen_capture +- test_geolocation_prompt_presence +- test_microphone_permissions_notification +- test_notifications_displayed +- test_screen_share_permission_prompt +- test_webextension_completed_installation_successfully_displayed password_manager: - test_add_password_non_ascii_chars: pass - test_add_password_save_valid_data: pass - test_auto_saved_generated_password_context_menu: pass - test_can_view_password_when_PP_enabled: pass - test_changes_made_in_edit_mode_are_saved: pass - test_delete_login: pass - test_multiple_saved_logins: pass - test_never_save_login_via_doorhanger: pass - test_primary_password_triggered_on_about_logins_access: pass - test_save_login_via_doorhanger: pass - test_update_login_via_doorhanger: pass +- test_add_password_non_ascii_chars +- test_add_password_save_valid_data +- test_auto_saved_generated_password_context_menu +- test_can_view_password_when_PP_enabled +- test_changes_made_in_edit_mode_are_saved +- test_delete_login +- test_multiple_saved_logins +- test_never_save_login_via_doorhanger +- test_primary_password_triggered_on_about_logins_access +- test_save_login_via_doorhanger +- test_update_login_via_doorhanger pdf_viewer: - test_download_pdf_with_form_fields: pass - test_download_triggered_on_content_disposition_attachment: pass - test_open_pdf_in_FF: pass - test_pdf_navigation: pass - test_zoom_pdf_viewer: pass +- test_download_pdf_with_form_fields +- test_download_triggered_on_content_disposition_attachment +- test_open_pdf_in_FF +- test_pdf_navigation +- test_zoom_pdf_viewer preferences: - test_check_for_updates: pass - test_clear_cookie_data: - win: unstable - mac: pass - linux: pass - test_firefox_home_new_tabs: pass - test_firefox_home_on_launch: pass - test_lang_pack_changed_from_about_prefs: pass - test_never_remember_history: pass - test_notifications_change_set: pass +- test_check_for_updates +- test_clear_cookie_data +- test_firefox_home_new_tabs +- test_firefox_home_on_launch +- test_lang_pack_changed_from_about_prefs +- test_never_remember_history +- test_notifications_change_set printing_ui: - test_print_preview: pass +- test_print_preview profile: - test_set_default_profile: pass +- test_set_default_profile reader_view: - test_improved_type_control_panel: pass - test_reader_view_location_bar: pass +- test_improved_type_control_panel +- test_reader_view_location_bar scrolling_panning_zooming: - test_default_zoom_persists: pass - test_mouse_wheel_zoom: - win: pass - mac: unstable - linux: pass - test_zoom_text_only: pass +- test_default_zoom_persists +- test_mouse_wheel_zoom +- test_zoom_text_only security_and_privacy: - test_blocking_cryptominers: pass - test_blocking_fingerprinters: pass - test_blocking_social_media_trackers: unstable - test_cookies_not_saved_private_browsing: pass - test_cross_site_tracking_cookies_blocked: pass - test_cryptominers_blocked_and_shown_in_info_panel: pass - test_detected_blocked_trackers_found: pass - test_downloads_from_private_not_leaked: pass - test_https_enabled_private_browsing: pass - test_never_remember_browsing_history: pass - test_no_trackers_detected: pass - test_open_link_in_private_window: pass - test_phishing_and_malware_warnings: pass - test_private_browser_password_doorhanger: pass - test_private_session_awesome_bar_exclusion: pass - test_private_session_history_exclusion: pass - test_private_window_from_panelui: pass - test_third_party_content_blocked_private_browsing: pass - test_tls_v1_2_protocol: pass - test_trackers_crypto_fingerprint_blocked: pass - test_tracking_content_custom_mode: pass - test_undo_close_tab_private_browsing: pass +- test_blocking_cryptominers +- test_blocking_fingerprinters +- test_blocking_social_media_trackers +- test_cookies_not_saved_private_browsing +- test_cross_site_tracking_cookies_blocked +- test_cryptominers_blocked_and_shown_in_info_panel +- test_detected_blocked_trackers_found +- test_downloads_from_private_not_leaked +- test_https_enabled_private_browsing +- test_never_remember_browsing_history +- test_no_trackers_detected +- test_open_link_in_private_window +- test_phishing_and_malware_warnings +- test_private_browser_password_doorhanger +- test_private_session_awesome_bar_exclusion +- test_private_session_history_exclusion +- test_private_window_from_panelui +- test_third_party_content_blocked_private_browsing +- test_tls_v1_2_protocol +- test_trackers_crypto_fingerprint_blocked +- test_tracking_content_custom_mode +- test_undo_close_tab_private_browsing tabs: - test_active_tab: pass - test_navigation_multiple_tabs: pass - test_open_bookmark_in_new_tab: pass - test_open_new_tab: pass - test_open_new_tab_keys: pass - test_open_new_tab_via_hyperlink: pass - test_pin_tab: pass - test_reopen_tab_through_context_menu: pass - test_reopen_tab_through_history_menu: pass - test_reopen_tabs_through_keys: pass +- test_active_tab +- test_navigation_multiple_tabs +- test_open_bookmark_in_new_tab +- test_open_new_tab +- test_open_new_tab_keys +- test_open_new_tab_via_hyperlink +- test_pin_tab +- test_reopen_tab_through_context_menu +- test_reopen_tab_through_history_menu +- test_reopen_tabs_through_keys theme_and_toolbar: - test_customize_themes_and_redirect: pass +- test_customize_themes_and_redirect diff --git a/manifests/key.yaml b/manifests/key.yaml new file mode 100644 index 000000000..dd14478a3 --- /dev/null +++ b/manifests/key.yaml @@ -0,0 +1,218 @@ +address_bar_and_search: + test_add_engine_address_bar: pass + test_addon_suggestion: + linux: pass + mac: unstable + win: unstable + test_default_search_provider_change_awesome_bar: unstable + test_google_search_counts_us: + linux: pass + mac: unstable + win: pass + test_google_withads_url_bar_us: + linux: pass + mac: unstable + win: pass + test_sap_google_adclick: unstable + test_search_code_google_non_us: pass + test_search_code_google_us: pass + test_search_engine_selector: pass + test_search_suggestions: unstable + test_search_term_persists: pass + test_server_not_found_error: pass + test_suggestion_engine_selection: pass +audio_video: + test_allow_audio_video_functionality: + linux: unstable + mac: pass + win: unstable + test_block_audio_video_functionality: pass + test_users_actions_saved_on_reload: pass +bookmarks_and_history: + test_add_new_other_bookmark: pass + test_bookmark_via_bookmark_menu: pass + test_bookmark_website_via_star_button: pass + test_clear_all_history: pass + test_clear_recent_history_displayed: pass + test_delete_bookmarks_from_toolbar: pass + test_delete_other_bookmarks: pass + test_deleted_page_not_remembered: pass + test_open_bookmark_in_new_window_via_toolbar_context_menu: pass + test_open_bookmark_in_private_window_via_toolbar_context_menu: pass + test_open_bookmarks_from_toolbar: pass + test_open_websites_from_history: pass + test_private_window_website_not_in_history: pass + test_toggle_bookmarks_toolbar: pass + test_user_can_forget_history: pass +downloads: + test_add_zip_type: + linux: unstable + mac: pass + win: pass + test_download_pdf: pass + test_download_pdf_from_context_menu: unstable +drag_and_drop: + test_copy_entire_row_column: pass +find_toolbar: + test_find_in_pdf: pass + test_find_toolbar_nav: pass + test_find_toolbar_search: pass +form_autofill: + test_address_autofill_attribute: pass + test_autofill_cc_cvv: pass + test_autofill_credit_card: pass + test_autofill_credit_card_doorhanger: pass + test_autofill_credit_card_enable: pass + test_autofill_credit_card_four_fields: pass + test_cc_clear_form: pass + test_clear_form: pass + test_create_new_cc_profile: pass + test_create_profile_autofill: pass + test_delete_cc_profile: pass + test_enable_disable_autofill: pass + test_form_autofill_suggestions: pass + test_name_autofill_attribute: pass + test_private_mode_info_not_saved: pass + test_telephone_autofill_attribute: pass + test_updating_address: pass +geolocation: + test_geolocation_shared_via_html5: + linux: pass + mac: pass + win: unstable + test_geolocation_shared_via_w3c_api: + linux: pass + mac: pass + win: unstable +language_packs: + test_language_pack_install_addons: pass + test_language_pack_install_preferences: pass +menus: + test_frequently_used_context_menu: + linux: pass + mac: pass + win: unstable + test_hyperlink_context_menu: pass + test_tab_context_menu_actions: pass + test_tab_context_menu_close: pass +meta: + test_selectors: + test_a_selector: pass + test_version: + test_version: pass +networking: + test_default_dns_protection: pass + test_http_site: pass +notifications: + test_audio_video_permissions_notification: + linux: pass + mac: unstable + win: pass + test_camera_permissions_notification: + linux: pass + mac: unstable + win: pass + test_deny_geolocation: pass + test_deny_screen_capture: pass + test_geolocation_prompt_presence: pass + test_microphone_permissions_notification: + linux: pass + mac: unstable + win: pass + test_notifications_displayed: pass + test_screen_share_permission_prompt: pass + test_webextension_completed_installation_successfully_displayed: pass +password_manager: + test_add_password_non_ascii_chars: pass + test_add_password_save_valid_data: pass + test_auto_saved_generated_password_context_menu: pass + test_can_view_password_when_PP_enabled: pass + test_changes_made_in_edit_mode_are_saved: pass + test_delete_login: pass + test_multiple_saved_logins: pass + test_never_save_login_via_doorhanger: pass + test_primary_password_triggered_on_about_logins_access: pass + test_save_login_via_doorhanger: pass + test_update_login_via_doorhanger: pass +pdf_viewer: + test_download_pdf_with_form_fields: pass + test_download_triggered_on_content_disposition_attachment: pass + test_open_pdf_in_FF: pass + test_pdf_navigation: pass + test_zoom_pdf_viewer: pass +pocket: + test_basic_de: + test_localized_pocket_layout_DE: deprecated + test_basic_fr: + test_localized_pocket_layout_FR: deprecated + test_basic_gb: + test_localized_pocket_layout_GB: deprecated + test_basic_us: + test_localized_pocket_layout_US: deprecated + test_new_tab_about_blank: pass +preferences: + test_check_for_updates: pass + test_clear_cookie_data: + linux: pass + mac: pass + win: unstable + test_firefox_home_new_tabs: pass + test_firefox_home_on_launch: pass + test_lang_pack_changed_from_about_prefs: pass + test_never_remember_history: pass + test_notifications_change_set: pass +printing_ui: + test_print_preview: pass +profile: + test_set_default_profile: pass +reader_view: + test_improved_type_control_panel: pass + test_reader_view_location_bar: pass +scrolling_panning_zooming: + test_default_zoom_persists: pass + test_mouse_wheel_zoom: + linux: pass + mac: unstable + win: pass + test_zoom_text_only: pass +security_and_privacy: + test_blocking_cryptominers: pass + test_blocking_fingerprinters: pass + test_blocking_social_media_trackers: unstable + test_cookies_not_saved_private_browsing: pass + test_cross_site_tracking_cookies_blocked: pass + test_cryptominers_blocked_and_shown_in_info_panel: pass + test_detected_blocked_trackers_found: pass + test_downloads_from_private_not_leaked: pass + test_https_enabled_private_browsing: pass + test_never_remember_browsing_history: pass + test_no_trackers_detected: pass + test_open_link_in_private_window: pass + test_phishing_and_malware_warnings: pass + test_private_browser_password_doorhanger: pass + test_private_session_awesome_bar_exclusion: pass + test_private_session_history_exclusion: pass + test_private_window_from_panelui: pass + test_third_party_content_blocked_private_browsing: pass + test_tls_v1_2_protocol: pass + test_trackers_crypto_fingerprint_blocked: pass + test_tracking_content_custom_mode: pass + test_undo_close_tab_private_browsing: pass +sync_and_fxa: + test_existing_fxa: + test_sync_existing_fxa: pass + test_new_fxa: + test_sync_new_fxa: pass +tabs: + test_active_tab: pass + test_navigation_multiple_tabs: pass + test_open_bookmark_in_new_tab: pass + test_open_new_tab: pass + test_open_new_tab_keys: pass + test_open_new_tab_via_hyperlink: pass + test_pin_tab: pass + test_reopen_tab_through_context_menu: pass + test_reopen_tab_through_history_menu: pass + test_reopen_tabs_through_keys: pass +theme_and_toolbar: + test_customize_themes_and_redirect: pass diff --git a/manifests/smoke.yaml b/manifests/smoke.yaml index b8b24774c..ad77aeb39 100644 --- a/manifests/smoke.yaml +++ b/manifests/smoke.yaml @@ -1,495 +1,237 @@ address_bar_and_search: - test_adaptive_history_autofill: - test_add_adaptive_history_autofill: pass - test_add_engine_address_bar: - test_add_search_engine_from_address_bar: pass - test_added_open_search_engine_default: - test_added_open_search_engine_default: pass - test_addon_suggestion: - test_addon_suggestion_based_on_search_input: pass - test_addressbar_search_engine_keywords: - test_addressbar_search_engine_keywords: pass - test_copied_url_contains_https: - test_copied_url_contains_https: pass - test_ctrl_enter_fixes_url: - test_ctrl_enter_fixes_url: pass - test_default_search_provider_change_awesome_bar: - test_default_search_provider_change_awesome_bar: pass - test_default_search_provider_change_legacy_search_bar: - test_default_search_provider_change_legacy_search_bar: pass - test_dont_show_search_suggestions_in_private_window: - test_no_search_engine_suggestions_in_private_window: pass - test_google_search_counts_us: - test_google_search_counts_us: pass - test_google_withads_url_bar_us: - test_google_withads_url_bar_us: pass - test_insertion_point_no_search_terms_display: - test_insertion_point_no_search_terms_display: pass - test_no_suggestions_for_empty_query: - test_suggestions_for_empty_query_not_shown_in_search_mode: pass - test_non_sponsored_topsite_context_menu_option: - test_non_sponsored_topsite_context_menu_option: pass - test_refresh_firefox_dialog: - test_refresh_firefox_dialog: pass - test_sap_google_adclick: - test_sap_google_adclick: unstable - test_seach_suggestions_can_be_disabled: - test_search_suggestions_pref_affects_urlbar_and_searchbar: pass - test_search_code_google_non_us: - test_search_code_google_non_us: pass - test_search_code_google_us: - test_search_code_google_us: pass - test_search_engine_selector: - test_search_engine_selector_and_validator: pass - test_search_mode_appears_after_input: - test_private_mode_repeat_after_enabling_pref: pass - test_search_mode_appears_and_suggestions_update: pass - test_search_mode_persists_mixed_with_bing: - test_search_mode_persists_mixed_with_bing: pass - test_search_modes_for_sites: - test_search_modes_for_sites: pass - test_search_string_displayed_when_addressbar_unfocused: - test_address_bar_and_search_string_displayed_when_addressbar_unfocused: pass - test_search_suggestions: - test_search_suggests_enabled: pass - test_search_term_persists: - test_search_term_persists: unstable - test_searchbar_display_alpenglow_theme: - test_searchbar_display_alpenglow_theme: pass - test_searchbar_on_engines_remove_restore: - test_searchbar_engine_remove_restore_affects_both_windows: pass - test_searchbar_results_shown_in_a_new_tab: - test_search_bar_results_shown_in_a_new_tab: pass - test_searchmode_change_tab: - test_searchmode_change_tab: pass - test_server_not_found_error: - test_server_not_found_error: pass - test_server_not_found_error_pb: - test_server_not_found_error_on_private_window: pass - test_suggestion_engine_selection: - test_search_suggestion_for_engine_selection: pass - test_tile_menu_options: - test_default_tile_hover_states: pass - test_tile_context_menu_options: pass +- test_adaptive_history_autofill +- test_add_engine_address_bar +- test_added_open_search_engine_default +- test_addon_suggestion +- test_addressbar_search_engine_keywords +- test_copied_url_contains_https +- test_ctrl_enter_fixes_url +- test_default_search_provider_change_awesome_bar +- test_default_search_provider_change_legacy_search_bar +- test_dont_show_search_suggestions_in_private_window +- test_google_search_counts_us +- test_google_withads_url_bar_us +- test_insertion_point_no_search_terms_display +- test_no_suggestions_for_empty_query +- test_non_sponsored_topsite_context_menu_option +- test_refresh_firefox_dialog +- test_sap_google_adclick +- test_seach_suggestions_can_be_disabled +- test_search_code_google_non_us +- test_search_code_google_us +- test_search_engine_selector +- test_search_mode_appears_after_input +- test_search_mode_persists_mixed_with_bing +- test_search_modes_for_sites +- test_search_string_displayed_when_addressbar_unfocused +- test_search_suggestions +- test_search_term_persists +- test_searchbar_display_alpenglow_theme +- test_searchbar_on_engines_remove_restore +- test_searchbar_results_shown_in_a_new_tab +- test_searchmode_change_tab +- test_server_not_found_error +- test_server_not_found_error_pb +- test_suggestion_engine_selection +- test_tile_menu_options audio_video: - test_allow_audio_video_functionality: - test_allow_audio_video_functionality: pass - test_block_audio_video_functionality: - test_block_audio_video_functionality: pass - test_users_actions_saved_on_reload: - test_users_actions_saved_on_reload: pass +- test_allow_audio_video_functionality +- test_block_audio_video_functionality +- test_users_actions_saved_on_reload bookmarks_and_history: - test_add_new_other_bookmark: - test_add_new_other_bookmark: pass - test_bookmark_via_bookmark_menu: - test_bookmark_via_hamburger_menu: pass - test_bookmark_website_via_star_button: - test_bookmark_website_via_star: pass - test_clear_all_history: - test_clear_all_history: pass - test_clear_recent_history_displayed: - test_clear_recent_history_displayed: pass - test_delete_bookmarks_from_toolbar: - test_delete_bookmarks_from_toolbar: pass - test_delete_other_bookmarks: - test_delete_other_bookmarks: pass - test_deleted_page_not_remembered: - test_deleted_page_not_remembered: pass - test_edit_bookmark_from_bookmark_menu: - test_edit_bookmark_from_bookmark_menu: pass - test_edit_bookmark_via_star_button: - test_edit_bookmark_via_star_button: pass - test_history_menu_from_different_places: - test_history_menu_in_different_places: pass - test_import_bookmarks_chrome: - test_chrome_bookmarks_imported: pass - test_import_bookmarks_edge: - test_edge_bookmarks_imported: pass - test_open_all_bookmarks_from_bookmarks_toolbar: - test_open_all_bookmarks_from_bookmarks_toolbar: pass - test_open_bookmark_in_new_window_via_toolbar_context_menu: - test_open_bookmark_in_new_window_via_toolbar_context_menu: pass - test_open_bookmark_in_private_window_via_toolbar_context_menu: - test_open_bookmark_in_new_private_window_via_toolbar_context_menu: pass - test_open_bookmarks_from_toolbar: - test_open_bookmarks_from_toolbar: pass - test_open_websites_from_history: - test_open_websites_from_history: pass - test_opened_website_in_new_tab_present_in_hamburger_history_menu: - test_the_website_opened_in_new_tab_is_present_in_history_menu: pass - test_opened_website_in_new_window_present_in_hamburger_history_menu: - test_the_website_opened_in_new_window_is_present_in_history_menu: pass - test_opened_website_present_in_hamburger_history_menu: - test_the_most_recent_website_is_present_in_history_menu: pass - test_private_window_website_not_in_history: - test_opened_website_in_private_window_not_captured_in_history_list: pass - test_toggle_bookmarks_toolbar: - test_toggle_bookmark_toolbar: pass - test_user_can_forget_history: - test_user_can_forget_history: pass +- test_add_new_other_bookmark +- test_bookmark_via_bookmark_menu +- test_bookmark_website_via_star_button +- test_clear_all_history +- test_clear_recent_history_displayed +- test_delete_bookmarks_from_toolbar +- test_delete_other_bookmarks +- test_deleted_page_not_remembered +- test_edit_bookmark_from_bookmark_menu +- test_edit_bookmark_via_star_button +- test_history_menu_from_different_places +- test_import_bookmarks_chrome +- test_import_bookmarks_edge +- test_open_all_bookmarks_from_bookmarks_toolbar +- test_open_bookmark_in_new_window_via_toolbar_context_menu +- test_open_bookmark_in_private_window_via_toolbar_context_menu +- test_open_bookmarks_from_toolbar +- test_open_websites_from_history +- test_opened_website_in_new_tab_present_in_hamburger_history_menu +- test_opened_website_in_new_window_present_in_hamburger_history_menu +- test_opened_website_present_in_hamburger_history_menu +- test_private_window_website_not_in_history +- test_toggle_bookmarks_toolbar +- test_user_can_forget_history downloads: - test_add_mime_type_doc: - test_mime_type_doc: pass - test_add_zip_type: - test_add_zip_type: pass - test_download_pdf: - test_download_pdf: pass - test_download_pdf_from_context_menu: - test_download_pdf_from_context_menu: pass - test_mixed_content_download_via_https: - test_mixed_content_download_via_https: unstable - test_set_always_ask_file_type: - test_set_always_ask_file_type: pass +- test_add_mime_type_doc +- test_add_zip_type +- test_download_pdf +- test_download_pdf_from_context_menu +- test_mixed_content_download_via_https +- test_set_always_ask_file_type drag_and_drop: - test_copy_entire_row_column: - test_copy_entire_row_column: pass - test_copy_from_an_editor_paste_in_another: - test_copy_from_an_editor_paste_in_another: pass - test_copy_hyperlink_table: - test_copy_table_with_hyperlink: pass - test_copy_table_header: - test_copy_table_header: pass - test_paste_image_text: - test_paste_image_text: pass +- test_copy_entire_row_column +- test_copy_from_an_editor_paste_in_another +- test_copy_hyperlink_table +- test_copy_table_header +- test_paste_image_text find_toolbar: - test_find_in_pdf: - test_find_in_pdf_using_key_combos: pass - test_find_toolbar_nav: - test_find_toolbar_navigation: pass - test_find_toolbar_search: - test_find_toolbar_search: pass +- test_find_in_pdf +- test_find_toolbar_nav +- test_find_toolbar_search form_autofill: - test_address_autofill_attribute: - test_address_attribute_selection: pass - test_autofill_cc_cvv: - test_autofill_cc_cvv: pass - test_autofill_credit_card: - test_autofill_credit_card: pass - test_autofill_credit_card_doorhanger: - test_autofill_credit_card_door_hanger: pass - test_autofill_credit_card_enable: - test_enable_disable_form_autofill_cc: pass - test_autofill_credit_card_four_fields: - test_autofill_four_fields: pass - test_cc_clear_form: - test_clear_form_credit_card: pass - test_clear_form: - test_clear_form: pass - test_create_new_cc_profile: - test_create_new_cc_profile: pass - test_create_profile_autofill: - test_create_address_profile: pass - test_delete_cc_profile: - test_delete_cc_profile: pass - test_edit_credit_card: - test_edit_credit_card_profile: pass - test_enable_disable_autofill: - test_enable_disable_autofill: pass - test_form_autofill_suggestions: - test_form_autofill_suggestions: pass - test_name_autofill_attribute: - test_name_attribute_selection: pass - test_private_mode_info_not_saved: - test_private_mode_info_not_saved: pass - test_telephone_autofill_attribute: - test_telephone_attribute_autofill: pass - test_updating_address: - test_update_address: pass - test_updating_credit_card: - test_update_cc_no_dupe_name: pass +- test_address_autofill_attribute +- test_autofill_cc_cvv +- test_autofill_credit_card +- test_autofill_credit_card_doorhanger +- test_autofill_credit_card_enable +- test_autofill_credit_card_four_fields +- test_cc_clear_form +- test_clear_form +- test_create_new_cc_profile +- test_create_profile_autofill +- test_delete_cc_profile +- test_edit_credit_card +- test_enable_disable_autofill +- test_form_autofill_suggestions +- test_name_autofill_attribute +- test_private_mode_info_not_saved +- test_telephone_autofill_attribute +- test_updating_address +- test_updating_credit_card geolocation: - test_geolocation_shared_via_html5: - test_allow_permission_on_geolocation_via_html5: pass - test_block_permission_on_geolocation_via_w3c_api: pass - test_geolocation_shared_via_w3c_api: - test_allow_permission_on_geolocation_via_w3c_api: pass - test_block_permission_on_geolocation_via_w3c_api: pass +- test_geolocation_shared_via_html5 +- test_geolocation_shared_via_w3c_api language_packs: - test_language_pack_install_addons: - test_language_pack_install_from_addons: pass - test_language_pack_install_preferences: - test_language_pack_install_about_preferences: pass +- test_language_pack_install_addons +- test_language_pack_install_preferences menus: - test_copy_paste_actions: - test_login_form_copy_paste: pass - test_search_field_copy_paste: pass - test_text_area_copy_paste: pass - test_frequently_used_context_menu: - test_inspect: pass - test_save_page_as: pass - test_take_screenshot: pass - test_hyperlink_context_menu: - test_open_link_in_new_window: pass - test_image_context_menu_actions: - test_copy_image_link: pass - test_open_image_in_new_tab: pass - test_save_image_as: pass - test_tab_context_menu_actions: - test_close_multiple_tabs_to_right: pass - test_duplicate_tab: pass - test_tab_context_menu_close: - test_close_multiple_tabs_other_tabs: pass - test_close_multiple_tabs_to_left: pass - test_copy_link: pass +- test_copy_paste_actions +- test_frequently_used_context_menu +- test_hyperlink_context_menu +- test_image_context_menu_actions +- test_tab_context_menu_actions +- test_tab_context_menu_close meta: - test_selectors: - test_a_selector: pass - test_version: - test_version: pass +- test_selectors +- test_version networking: - test_default_dns_protection: - test_doh_enforces_secure_dns_resolution: pass - test_http_site: - test_http_site: pass +- test_default_dns_protection +- test_http_site notifications: - test_audio_video_permissions_notification: - test_camera_and_microphone_permissions_notification: pass - test_camera_permissions_notification: - test_camera_permissions_notification: pass - test_deny_geolocation: - test_deny_geolocation: pass - test_deny_screen_capture: - test_deny_screen_capture: pass - test_geolocation_prompt_presence: - test_geolocation_prompt_is_triggered_on_request_location_on_a_website: pass - test_microphone_permissions_notification: - test_microphone_permissions_notification: pass - test_notifications_displayed: - test_notifications_displayed: pass - test_screen_share_permission_prompt: - test_screen_share_permission_prompt: pass - test_webextension_completed_installation_successfully_displayed: - test_webextension_completed_installation_successfully_displayed: pass +- test_audio_video_permissions_notification +- test_camera_permissions_notification +- test_deny_geolocation +- test_deny_screen_capture +- test_geolocation_prompt_presence +- test_microphone_permissions_notification +- test_notifications_displayed +- test_screen_share_permission_prompt +- test_webextension_completed_installation_successfully_displayed password_manager: - test_about_logins_navigation_from_context_menu: - test_about_logins_navigation_from_login_form_context_menu: pass - test_about_logins_navigation_from_hamburger_menu: - test_about_logins_navigation_from_password_hamburger_menu: pass - test_about_logins_search_username: - test_about_logins_search_username: pass - test_about_logins_search_website: - test_about_logins_search_website: pass - test_add_password_non_ascii_chars: - test_add_password_non_ascii_chars: pass - test_add_password_save_valid_data: - test_add_password_save_valid_data: pass - test_add_primary_password: - test_add_primary_password: pass - test_auto_saved_generated_password_context_menu: - test_auto_saved_generated_password_context_menu: pass - test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load: - test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load: pass - test_can_view_password_when_PP_enabled: - test_password_can_be_shown: pass - test_changes_made_in_edit_mode_are_saved: - test_changes_made_in_edit_mode_are_saved: pass - test_delete_login: - test_delete_login: pass - test_multiple_saved_logins: - test_multiple_saved_logins: pass - test_never_save_login_via_doorhanger: - test_never_save_login_via_doorhanger: pass - test_password_csv_correctness: - test_password_csv_correctness: pass - test_password_csv_export: - test_password_csv_export: pass - test_primary_password_triggered_on_about_logins_access: - test_primary_password_triggered_on_about_logins_access_via_hamburger_menu: pass - test_save_login_via_doorhanger: - test_save_login_via_doorhanger: pass - test_saved_hyperlink_redirects_to_corresponding_page: - test_saved_hyperlink_redirects_to_corresponding_page: pass - test_update_login_via_doorhanger: - test_update_login_via_doorhanger: pass +- test_about_logins_navigation_from_context_menu +- test_about_logins_navigation_from_hamburger_menu +- test_about_logins_search_username +- test_about_logins_search_website +- test_add_password_non_ascii_chars +- test_add_password_save_valid_data +- test_add_primary_password +- test_auto_saved_generated_password_context_menu +- test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load +- test_can_view_password_when_PP_enabled +- test_changes_made_in_edit_mode_are_saved +- test_delete_login +- test_multiple_saved_logins +- test_never_save_login_via_doorhanger +- test_password_csv_correctness +- test_password_csv_export +- test_primary_password_triggered_on_about_logins_access +- test_save_login_via_doorhanger +- test_saved_hyperlink_redirects_to_corresponding_page +- test_update_login_via_doorhanger pdf_viewer: - test_add_image_pdf: - test_add_image_pdf: pass - test_download_pdf_with_form_fields: - test_download_pdf_with_form_fields: pass - test_download_triggered_on_content_disposition_attachment: - test_download_panel_triggered_on_content_disposition_attachment: pass - test_open_pdf_in_FF: - test_open_pdf_in_fx: pass - test_pdf_data_can_be_cleared: - test_pdf_data_can_be_cleared: pass - test_pdf_download: - test_pdf_download: pass - test_pdf_input_numbers: - test_pdf_input_numbers: pass - test_pdf_navigation: - test_navigation_keys: pass - test_navigation_next_prev: pass - test_navigation_page_numbers: pass - test_toolbar_options_cursor: pass - test_toolbar_options_rotate_ccw: pass - test_toolbar_options_rotate_cw: pass - test_zoom_pdf_viewer: - test_zoom_pdf_viewer_keys: pass - test_zoom_pdf_viewer_toolbar: pass -pocket: - test_basic_de: - test_localized_pocket_layout_DE: pass - test_basic_fr: - test_localized_pocket_layout_FR: pass - test_basic_gb: - test_localized_pocket_layout_GB: pass - test_basic_us: - test_localized_pocket_layout_US: pass - test_new_tab_about_blank: pass +- test_add_image_pdf +- test_download_pdf_with_form_fields +- test_download_triggered_on_content_disposition_attachment +- test_open_pdf_in_FF +- test_pdf_data_can_be_cleared +- test_pdf_download +- test_pdf_input_numbers +- test_pdf_navigation +- test_zoom_pdf_viewer preferences: - test_check_for_updates: - test_check_for_updates: pass - test_clear_cookie_data: - test_clear_cookie_data: pass - test_firefox_home_new_tabs: - test_firefox_home_new_tab: pass - test_firefox_home_on_launch: - test_firefox_home_on_launch: pass - test_lang_pack_changed_from_about_prefs: - test_lang_pack_changed_from_about_prefs: pass - test_manage_cookie_data: - test_manage_cookie_data: pass - test_never_remember_history: - test_never_remember_history: pass - test_notifications_change_set: - test_notifications_allow: pass +- test_check_for_updates +- test_clear_cookie_data +- test_firefox_home_new_tabs +- test_firefox_home_on_launch +- test_lang_pack_changed_from_about_prefs +- test_manage_cookie_data +- test_never_remember_history +- test_notifications_change_set printing_ui: - test_print_preview: - test_print_preview_keys: pass - test_print_preview_menu: pass - test_print_to_pdf: - test_print_to_pdf: pass +- test_print_preview +- test_print_to_pdf profile: - test_set_default_profile: - test_set_default_profile: pass +- test_set_default_profile reader_view: - test_improved_type_control_panel: - test_type_control_panel_character_spacing: pass - test_type_control_panel_content_width: pass - test_type_control_panel_font: pass - test_type_control_panel_line_spacing: pass - test_type_control_panel_size: pass - test_type_control_panel_text_alignment: pass - test_type_control_panel_word_spacing: pass - test_reader_view_close_sidebar: - test_reader_view_close_from_sidebar: pass - test_reader_view_location_bar: - test_reader_view_open_close_using_keys: pass - test_reader_view_open_close_using_searchbar: pass +- test_improved_type_control_panel +- test_reader_view_close_sidebar +- test_reader_view_location_bar scrolling_panning_zooming: - test_default_zoom_persists: - test_default_zoom_across_tabs: pass - test_mouse_wheel_zoom: - test_mouse_wheel_zoom: pass - test_zoom_from_menu_bar: - test_zoom_from_menu_bar: pass - test_zoom_menu_correlation: - test_zoom_level_div_position: pass - test_zoom_text_only: - test_zoom_text_only_after_restart: pass - test_zoom_text_only_from_settings: pass +- test_default_zoom_persists +- test_mouse_wheel_zoom +- test_zoom_from_menu_bar +- test_zoom_menu_correlation +- test_zoom_text_only security_and_privacy: - test_blocking_cryptominers: - test_blocking_cryptominers: pass - test_blocking_fingerprinters: - test_blocking_fingerprinter: pass - test_blocking_social_media_trackers: - test_blocking_social_media_trackers: pass - test_cookies_not_saved_private_browsing: - test_cookies_not_saved_private_browsing: pass - test_cross_site_tracking_cookies_blocked: - test_cross_site_tracking_cookies_blocked: pass - test_cryptominers_blocked_and_shown_in_info_panel: - test_cryptominers_blocked_and_shown_in_info_panel: pass - test_detected_blocked_trackers_found: - test_detected_blocked_trackers_found: pass - test_downloads_from_private_not_leaked: - test_downloads_from_private_not_leaked: pass - test_https_enabled_private_browsing: - test_https_first_mode_in_private_browsing: pass - test_never_remember_browsing_history: - test_never_remember_browsing_history_from_panel: pass - test_never_remember_browsing_history_settings: pass - test_no_trackers_detected: - test_no_trackers_detected: pass - test_open_link_in_private_window: - test_open_link_in_private_window: pass - test_phishing_and_malware_warnings: - test_phishing_and_malware_warning_errors: pass - test_private_browser_password_doorhanger: - test_no_password_doorhanger_private_browsing: pass - test_private_session_awesome_bar_exclusion: - test_websites_visited_in_private_browser_not_displayed_in_awesome_bar: pass - test_private_session_history_exclusion: - test_websites_visited_in_private_browser_not_displayed_in_history: pass - test_private_window_from_panelui: - test_private_window_from_panelui: pass - test_third_party_content_blocked_private_browsing: - test_third_party_content_blocked_private_browsing_allowed_tracking: pass - test_third_party_content_blocked_private_browsing_cross_site: pass - test_third_party_content_private_browsing_tracking_statuses: pass - test_tls_v1_2_protocol: - test_tls_v1_2_protocol: pass - test_trackers_crypto_fingerprint_blocked: - test_cross_site_trackrs_crypto_fingerprinter_blocked: pass - test_tracking_content_custom_mode: - test_allowed_tracking_content: pass - test_blocked_tracking_content: pass - test_undo_close_tab_private_browsing: - test_undo_close_tab_private_browsing: pass +- test_blocking_cryptominers +- test_blocking_fingerprinters +- test_blocking_social_media_trackers +- test_cookies_not_saved_private_browsing +- test_cross_site_tracking_cookies_blocked +- test_cryptominers_blocked_and_shown_in_info_panel +- test_detected_blocked_trackers_found +- test_downloads_from_private_not_leaked +- test_https_enabled_private_browsing +- test_never_remember_browsing_history +- test_no_trackers_detected +- test_open_link_in_private_window +- test_phishing_and_malware_warnings +- test_private_browser_password_doorhanger +- test_private_session_awesome_bar_exclusion +- test_private_session_history_exclusion +- test_private_window_from_panelui +- test_third_party_content_blocked_private_browsing +- test_tls_v1_2_protocol +- test_trackers_crypto_fingerprint_blocked +- test_tracking_content_custom_mode +- test_undo_close_tab_private_browsing sync_and_fxa: - test_existing_fxa: - test_sync_existing_fxa: pass - test_new_fxa: - test_sync_new_fxa: pass +- test_existing_fxa +- test_new_fxa tabs: - test_active_tab: - test_active_tab: pass - test_change_position_of_pinned_tabs: - test_change_position_of_pinned_tabs: pass - test_close_pinned_tab_via_mouse: - test_close_pinned_tab_via_middle_click: pass - test_close_tab_through_middle_mouse_click: - test_close_tab_through_middle_mouse_click: pass - test_display_customize_button: - test_customize_button_displayed_in_tab_bar: pass - test_list_all_tabs: - test_list_all_tabs: pass - test_mute_tabs: - test_mute_unmute_tab: pass - test_navigation_multiple_tabs: - test_navigation_multiple_tabs: pass - test_open_bookmark_in_new_tab: - test_open_bookmark_in_new_tab: pass - test_open_new_bg_tab_via_mouse_and_keyboard: - test_open_new_bg_tab_via_mouse_and_keyboard: pass - test_open_new_tab: - test_open_new_tab_plus: pass - test_open_new_tab_keys: - test_open_new_tab_via_keyboard: pass - test_open_new_tab_via_hyperlink: - test_open_new_via_hyperlink: pass - test_pin_tab: - test_pin_tab: pass - test_pin_unpin_selected_tabs: - test_pin_unpin_selected_tabs: pass - test_play_mute_unmute_tabs_via_toggle: - test_play_mute_unmute_tabs_via_toggle: pass - test_reload_overiding_cache_keys: - test_reload_overiding_cache_keys: pass - test_reload_tab_via_keyboard: - test_reload_tab_via_keyboard: pass - test_reopen_tab_through_context_menu: - test_reopen_tab_through_context_menu: pass - test_reopen_tab_through_history_menu: - test_reopen_tab_through_history_menu: pass - test_reopen_tabs_through_keys: - test_reopen_tabs_through_keys: pass +- test_active_tab +- test_change_position_of_pinned_tabs +- test_close_pinned_tab_via_mouse +- test_close_tab_through_middle_mouse_click +- test_display_customize_button +- test_list_all_tabs +- test_mute_tabs +- test_navigation_multiple_tabs +- test_open_bookmark_in_new_tab +- test_open_new_bg_tab_via_mouse_and_keyboard +- test_open_new_tab +- test_open_new_tab_keys +- test_open_new_tab_via_hyperlink +- test_pin_tab +- test_pin_unpin_selected_tabs +- test_play_mute_unmute_tabs_via_toggle +- test_reload_overiding_cache_keys +- test_reload_tab_via_keyboard +- test_reopen_tab_through_context_menu +- test_reopen_tab_through_history_menu +- test_reopen_tabs_through_keys theme_and_toolbar: - test_customize_themes_and_redirect: - test_activate_theme_background_matches_expected: pass - test_alpenglow_theme: pass - test_redirect_to_addons: pass - test_installed_theme_enabled: - test_find_more_themes: pass - test_installed_theme_enabled: pass +- test_customize_themes_and_redirect +- test_installed_theme_enabled diff --git a/tests/address_bar_and_search/test_google_search_counts_us.py b/tests/address_bar_and_search/test_google_search_counts_us.py index a7edd9637..c1b4974d0 100644 --- a/tests/address_bar_and_search/test_google_search_counts_us.py +++ b/tests/address_bar_and_search/test_google_search_counts_us.py @@ -49,11 +49,9 @@ def test_google_search_counts_us(driver: Firefox): sleep(2) continue else: - pytest.fail( - "CAPTCHA triggered repeatedly. Giving up after 5 attempts.") + pytest.fail("CAPTCHA triggered repeatedly. Giving up after 5 attempts.") - provider_ok = utils.assert_json_value( - json_data, SEARCH_PROVIDER_PATH, 1) + provider_ok = utils.assert_json_value(json_data, SEARCH_PROVIDER_PATH, 1) tag_ok = utils.assert_json_value(json_data, SEARCH_TAG_PATH, 1) if provider_ok and tag_ok: diff --git a/tests/address_bar_and_search/test_google_withads_url_bar_us.py b/tests/address_bar_and_search/test_google_withads_url_bar_us.py index 3933906f0..501548b30 100644 --- a/tests/address_bar_and_search/test_google_withads_url_bar_us.py +++ b/tests/address_bar_and_search/test_google_withads_url_bar_us.py @@ -42,8 +42,7 @@ def test_google_withads_url_bar_us(driver): sleep(2) continue else: - pytest.fail( - "CAPTCHA triggered repeatedly. Giving up after 5 attempts.") + pytest.fail("CAPTCHA triggered repeatedly. Giving up after 5 attempts.") about_telemetry = AboutTelemetry(driver).open() sleep(5) diff --git a/tests/downloads/test_mixed_content_download_via_https.py b/tests/downloads/test_mixed_content_download_via_https.py index 3a919bb81..b150c32f9 100644 --- a/tests/downloads/test_mixed_content_download_via_https.py +++ b/tests/downloads/test_mixed_content_download_via_https.py @@ -32,8 +32,7 @@ def test_mixed_content_download_via_https(driver: Firefox, delete_files): # Wait for the test website to wake up and download the content web_page.open() - web_page.wait.until(lambda _: nav.element_visible( - "download-target-element")) + web_page.wait.until(lambda _: nav.element_visible("download-target-element")) # Verify download name matches expected pattern nav.verify_download_name(r"file-sample_100kB(\(\d+\))?.odt$") diff --git a/tests/geolocation/test_geolocation_shared_via_html5.py b/tests/geolocation/test_geolocation_shared_via_html5.py index b25f12b9b..b9b754954 100644 --- a/tests/geolocation/test_geolocation_shared_via_html5.py +++ b/tests/geolocation/test_geolocation_shared_via_html5.py @@ -31,11 +31,9 @@ def wait_for_geolocation_data(web_page, timeout=20): web_page.custom_wait(timeout=timeout).until( lambda _: all( [ - web_page.find_element( - By.ID, "latitude").get_attribute("data-raw") + web_page.find_element(By.ID, "latitude").get_attribute("data-raw") is not None, - web_page.find_element( - By.ID, "longitude").get_attribute("data-raw") + web_page.find_element(By.ID, "longitude").get_attribute("data-raw") is not None, ] ) diff --git a/tests/geolocation/test_geolocation_shared_via_w3c_api.py b/tests/geolocation/test_geolocation_shared_via_w3c_api.py index c73c2074c..bd192f570 100644 --- a/tests/geolocation/test_geolocation_shared_via_w3c_api.py +++ b/tests/geolocation/test_geolocation_shared_via_w3c_api.py @@ -78,8 +78,7 @@ def test_allow_permission_on_geolocation_via_w3c_api(driver: Firefox, temp_selec # Click the 'Try It' button and Allow the location sharing while choose the option Remember this decision web_page.click_on("geolocation-button-selector") - nav.handle_geolocation_prompt( - button_type="primary", remember_this_decision=True) + nav.handle_geolocation_prompt(button_type="primary", remember_this_decision=True) # Check that the location marker is displayed # if map is displayed, style attribute will be available @@ -122,8 +121,7 @@ def test_block_permission_on_geolocation_via_w3c_api(driver: Firefox, temp_selec # Click the 'Try It' button and Block the location sharing while choose the option Remember this decision tabs.open_single_page_in_new_tab(web_page, num_tabs=2) web_page.click_on("geolocation-button-selector") - nav.handle_geolocation_prompt( - button_type="secondary", remember_this_decision=True) + nav.handle_geolocation_prompt(button_type="secondary", remember_this_decision=True) # Check that the location marker is displayed # if map is not displayed, style attribute will not be available diff --git a/tests/notifications/test_audio_video_permissions_notification.py b/tests/notifications/test_audio_video_permissions_notification.py index 9c3414ffe..5b3457263 100644 --- a/tests/notifications/test_audio_video_permissions_notification.py +++ b/tests/notifications/test_audio_video_permissions_notification.py @@ -49,8 +49,7 @@ def test_camera_and_microphone_permissions_notification( # Verify that the notification is displayed nav.element_visible("popup-notification") - nav.expect_element_attribute_contains( - "popup-notification", "label", "Allow ") + nav.expect_element_attribute_contains("popup-notification", "label", "Allow ") nav.expect_element_attribute_contains( "popup-notification", "name", "mozilla.github.io" ) diff --git a/tests/notifications/test_camera_permissions_notification.py b/tests/notifications/test_camera_permissions_notification.py index 4c4e9fa9d..a2d3c1d37 100644 --- a/tests/notifications/test_camera_permissions_notification.py +++ b/tests/notifications/test_camera_permissions_notification.py @@ -46,8 +46,7 @@ def test_camera_permissions_notification(driver: Firefox, temp_selectors): web_page.click_on("camera-only") # Verify that the notification is displayed - nav.expect_element_attribute_contains( - "popup-notification", "label", "Allow ") + nav.expect_element_attribute_contains("popup-notification", "label", "Allow ") nav.expect_element_attribute_contains( "popup-notification", "name", "mozilla.github.io" ) diff --git a/tests/notifications/test_microphone_permissions_notification.py b/tests/notifications/test_microphone_permissions_notification.py index a3073d1b8..e28cbd960 100644 --- a/tests/notifications/test_microphone_permissions_notification.py +++ b/tests/notifications/test_microphone_permissions_notification.py @@ -47,8 +47,7 @@ def test_microphone_permissions_notification(driver: Firefox, temp_selectors): # Verify that the notification is displayed nav.element_visible("popup-notification") - nav.expect_element_attribute_contains( - "popup-notification", "label", "Allow ") + nav.expect_element_attribute_contains("popup-notification", "label", "Allow ") nav.expect_element_attribute_contains( "popup-notification", "name", "mozilla.github.io" ) diff --git a/tests/password_manager/test_password_csv_correctness.py b/tests/password_manager/test_password_csv_correctness.py index ba279b223..c84333d32 100644 --- a/tests/password_manager/test_password_csv_correctness.py +++ b/tests/password_manager/test_password_csv_correctness.py @@ -34,8 +34,7 @@ def test_password_csv_correctness( about_logins.export_passwords_csv(downloads_folder, "passwords.csv") # Verify the exported csv file is present in the target folder - csv_file = about_logins.verify_csv_export( - downloads_folder, "passwords.csv") + csv_file = about_logins.verify_csv_export(downloads_folder, "passwords.csv") assert os.path.exists(csv_file) # Verify the contents of the exported csv file @@ -47,8 +46,7 @@ def test_password_csv_correctness( reader = csv.DictReader(pw) actual_logins = {} for row in reader: - actual_logins[row["username"] + "@" + - row["url"][8:]] = row["password"] + actual_logins[row["username"] + "@" + row["url"][8:]] = row["password"] assert re.match(guid_pattern, row["guid"]) assert re.match(time_pattern, row["timeCreated"]) assert re.match(time_pattern, row["timeLastUsed"]) diff --git a/tests/password_manager/test_password_csv_export.py b/tests/password_manager/test_password_csv_export.py index 7c4b61847..4f4d36676 100644 --- a/tests/password_manager/test_password_csv_export.py +++ b/tests/password_manager/test_password_csv_export.py @@ -36,8 +36,7 @@ def test_password_csv_export( keyboard.tap(Key.enter) # Verify the exported csv file is present in the target folder - csv_file = about_logins.verify_csv_export( - downloads_folder, "passwords.csv") + csv_file = about_logins.verify_csv_export(downloads_folder, "passwords.csv") assert os.path.exists(csv_file) # Delete the password.csv created diff --git a/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py b/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py index e3adacb7e..3db3177f2 100644 --- a/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py +++ b/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py @@ -51,8 +51,7 @@ def test_mouse_wheel_zoom(driver: Firefox): # Switch to chrome context to check zoom level in the toolbar with driver.context(driver.CONTEXT_CHROME): zoom_button = nav.get_element("toolbar-zoom-level") - zoom_level = nav.get_element( - "toolbar-zoom-level").get_attribute("label") + zoom_level = nav.get_element("toolbar-zoom-level").get_attribute("label") logging.info(f"Zoom level after zoom-in: {zoom_level}") # Assert that the zoom level label is "110%" after zooming in @@ -68,8 +67,7 @@ def test_mouse_wheel_zoom(driver: Firefox): # **Step 2**: Reset zoom to 100% using the keyboard shortcut (Ctrl + 0) with driver.context(driver.CONTEXT_CHROME): - actions.key_down(Keys.CONTROL).send_keys( - "0").key_up(Keys.CONTROL).perform() + actions.key_down(Keys.CONTROL).send_keys("0").key_up(Keys.CONTROL).perform() time.sleep(1) # Allow time for reset effect to take place reset_position = driver.find_element(By.TAG_NAME, "div").location["x"] logging.info(f"X position of div after zoom-reset: {reset_position}") @@ -109,6 +107,5 @@ def test_mouse_wheel_zoom(driver: Firefox): # Reset the zoom level back to 100% with driver.context(driver.CONTEXT_CHROME): - actions.key_down(Keys.CONTROL).send_keys( - "0").key_up(Keys.CONTROL).perform() + actions.key_down(Keys.CONTROL).send_keys("0").key_up(Keys.CONTROL).perform() time.sleep(1) # Allow time for reset effect to take place diff --git a/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py b/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py index d6df4bf4c..d584177ed 100644 --- a/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py +++ b/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py @@ -48,8 +48,7 @@ def test_play_mute_unmute_tabs_via_toggle(driver: Firefox, sys_platform: str): # Locate and open 2 latest videos in new tabs video_selector = "ytd-rich-item-renderer a#video-title-link" video_links = wait.until( - EC.visibility_of_all_elements_located( - (By.CSS_SELECTOR, video_selector)) + EC.visibility_of_all_elements_located((By.CSS_SELECTOR, video_selector)) ) for i in range(2): @@ -126,8 +125,7 @@ def click_multi_tab_audio_button(): for i in [2, 3]: tabs.expect_tab_sound_status(i, tabs.MEDIA_STATUS.MUTED) tab = tabs.get_tab(i) - assert tab.get_attribute( - "muted") is not None, f"Tab {i} should be muted" + assert tab.get_attribute("muted") is not None, f"Tab {i} should be muted" # Click Unmute button click_multi_tab_audio_button() From 4514cdb6621b412217384ae19ba7a2d904303720 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Thu, 20 Nov 2025 13:53:10 -0800 Subject: [PATCH 19/29] update manifest key; chooser defaults out to a manifest --- choose_ci_set.py | 53 +++++---- manifests/key.yaml | 105 ++++++++++++++++-- ...ault_search_provider_change_awesome_bar.py | 7 +- .../test_bookmark_website_via_star_button.py | 1 - .../test_set_always_ask_file_type.py | 4 +- .../find_toolbar/test_find_toolbar_search.py | 6 +- .../test_telephone_autofill_attribute.py | 11 +- tests/menus/test_copy_paste_actions.py | 4 +- .../menus/test_image_context_menu_actions.py | 7 +- tests/networking/test_http_site.py | 14 +-- .../test_notifications_displayed.py | 7 +- ...out_logins_navigation_from_context_menu.py | 1 - .../test_pdf_data_can_be_cleared.py | 15 +-- tests/printing_ui/test_print_preview.py | 1 - .../test_improved_type_control_panel.py | 72 ++++++------ .../test_zoom_text_only.py | 54 ++++----- ...est_cross_site_tracking_cookies_blocked.py | 13 +-- tests/tabs/test_active_tab.py | 1 - .../test_customize_themes_and_redirect.py | 21 ++-- 19 files changed, 228 insertions(+), 169 deletions(-) diff --git a/choose_ci_set.py b/choose_ci_set.py index 2ee7d0f2a..d80d849a9 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -47,9 +47,8 @@ def localify(path: str) -> str: return path.replace(SCRIPT_DIR, ".") -def get_tests_by_model( - model_name: str, test_paths_and_contents: dict, run_list: list -) -> list: +def get_tests_by_model(model_name: str, test_paths_and_contents: dict, + run_list: list) -> list: """ Given a model name, a dict of paths and their file contents, and the list of existing tests/dirs to check, return matches by local paths @@ -74,11 +73,8 @@ def dedupe(run_list: list) -> list: removes = [] for i, entry in enumerate(run_list): - if ( - not entry.startswith(".") - and not entry.startswith("\\") - and not entry.startswith("/") - ): + if (not entry.startswith(".") and not entry.startswith("\\") + and not entry.startswith("/")): dotslashes.append(i) for dotslash in dotslashes: @@ -119,13 +115,11 @@ def convert_manifest_to_list(manifest_loc): if manifest: print(f"Reading {manifest_loc}") for suite in manifest: - print(suite) if suite not in mkey: print(f"{suite} not in {MANIFEST_KEY}") continue for testfile in manifest[suite]: - print(f" {testfile}") addtest = False test_name = f"{testfile}.py" if testfile not in mkey[suite]: @@ -144,15 +138,15 @@ def convert_manifest_to_list(manifest_loc): addtest = True elif isinstance(mkey[suite][testfile][subtest], dict): if sysname() in mkey[suite][testfile][subtest]: - if mkey[suite][testfile][subtest][sysname()] == "pass": + if mkey[suite][testfile][subtest][ + sysname()] == "pass": test_name = f"{test_name}::{subtest}" addtest = True if addtest: test_to_add = SLASH.join(toplevel + [suite, test_name]) assert os.path.exists(test_to_add.split("::")[0]), ( - f"{test_to_add} could not be found" - ) + f"{test_to_add} could not be found") tests.append(test_to_add) addtest = False return tests @@ -198,12 +192,9 @@ def convert_manifest_to_list(manifest_loc): run_list = [] check_output(["git", "fetch", "--quiet", "--depth=1", "origin", "main"]) - committed_files = ( - check_output(["git", "--no-pager", "diff", "--name-only", "origin/main"]) - .decode() - .replace("/", SLASH) - .splitlines() - ) + committed_files = (check_output( + ["git", "--no-pager", "diff", "--name-only", + "origin/main"]).decode().replace("/", SLASH).splitlines()) main_conftest = "conftest.py" base_page = os.path.join("modules", "page_base.py") @@ -221,7 +212,8 @@ def convert_manifest_to_list(manifest_loc): for root, _, files in os.walk(os.path.join(SCRIPT_DIR, "tests")): for f in files: this_file = os.path.join(root, f) - if re_obj.get("test_re").search(this_file) and "__pycache" not in this_file: + if re_obj.get("test_re").search( + this_file) and "__pycache" not in this_file: all_tests.append(os.path.join(this_file)) with open(this_file, encoding="utf-8") as fh: lines = fh.readlines() @@ -236,7 +228,9 @@ def convert_manifest_to_list(manifest_loc): changed_models = [ f for f in committed_files if re_obj.get("object_model_re").match(f) ] - changed_tests = [f for f in committed_files if re_obj.get("test_re").match(f)] + changed_tests = [ + f for f in committed_files if re_obj.get("test_re").match(f) + ] if changed_suite_conftests: run_list = [ @@ -248,9 +242,9 @@ def convert_manifest_to_list(manifest_loc): for selector_file in changed_selectors: (_, filename) = os.path.split(selector_file) model_name = pascalify(filename.split(".")[0]) - for test_name in get_tests_by_model( - model_name, test_paths_and_contents, run_list - ): + for test_name in get_tests_by_model(model_name, + test_paths_and_contents, + run_list): run_list.append(test_name) if changed_models: @@ -258,9 +252,9 @@ def convert_manifest_to_list(manifest_loc): model_file_contents = "".join([line for line in open(model_file)]) classes = re_obj.get("class_re").findall(model_file_contents) for model_name in classes: - for test_name in get_tests_by_model( - model_name, test_paths_and_contents, run_list - ): + for test_name in get_tests_by_model(model_name, + test_paths_and_contents, + run_list): run_list.append(test_name) if changed_tests: @@ -280,6 +274,7 @@ def convert_manifest_to_list(manifest_loc): if not run_list: ci_paths = convert_manifest_to_list(CI_MANIFEST) + ci_paths = dedupe(ci_paths) with open(OUTPUT_FILE, "w") as fh: fh.write("\n".join(ci_paths)) sys.exit(0) @@ -291,6 +286,8 @@ def convert_manifest_to_list(manifest_loc): if SLASH == "\\": run_list = [entry.replace("/", SLASH) for entry in run_list] run_list = dedupe(run_list) - run_list = [entry for entry in run_list if os.path.exists(entry.split("::")[0])] + run_list = [ + entry for entry in run_list if os.path.exists(entry.split("::")[0]) + ] with open(OUTPUT_FILE, "w") as fh: fh.write("\n".join(run_list)) diff --git a/manifests/key.yaml b/manifests/key.yaml index dd14478a3..f1911b40c 100644 --- a/manifests/key.yaml +++ b/manifests/key.yaml @@ -1,26 +1,48 @@ address_bar_and_search: - test_add_engine_address_bar: pass - test_addon_suggestion: - linux: pass + test_adaptive_history_autofill: + linux: unstable mac: unstable - win: unstable + win: pass + test_add_engine_address_bar: pass + test_added_open_search_engine_default: pass + test_addon_suggestion: unstable + test_addressbar_search_engine_keywords: pass + test_copied_url_contains_https: pass + test_ctrl_enter_fixes_url: pass test_default_search_provider_change_awesome_bar: unstable + test_default_search_provider_change_legacy_search_bar: deprecated + test_dont_show_search_suggestions_in_private_window: pass test_google_search_counts_us: linux: pass mac: unstable win: pass test_google_withads_url_bar_us: - linux: pass + linux: unstable mac: unstable win: pass + test_insertion_point_no_search_terms_display: pass + test_no_suggestions_for_empty_query: pass + test_non_sponsored_topsite_context_menu_option: pass + test_refresh_firefox_dialog: pass test_sap_google_adclick: unstable + test_seach_suggestions_can_be_disabled: pass test_search_code_google_non_us: pass test_search_code_google_us: pass test_search_engine_selector: pass + test_search_mode_appears_after_input: pass + test_search_mode_persists_mixed_with_bing: pass + test_search_modes_for_sites: unstable + test_search_string_displayed_when_addressbar_unfocused: pass test_search_suggestions: unstable test_search_term_persists: pass + test_searchbar_display_alpenglow_theme: pass + test_searchbar_on_engines_remove_restore: pass + test_searchbar_results_shown_in_a_new_tab: pass + test_searchmode_change_tab: pass test_server_not_found_error: pass + test_server_not_found_error_pb: pass test_suggestion_engine_selection: pass + test_tile_menu_options: pass audio_video: test_allow_audio_video_functionality: linux: unstable @@ -37,22 +59,41 @@ bookmarks_and_history: test_delete_bookmarks_from_toolbar: pass test_delete_other_bookmarks: pass test_deleted_page_not_remembered: pass + test_edit_bookmark_from_bookmark_menu: pass + test_edit_bookmark_via_star_button: pass + test_history_menu_from_different_places: pass + test_import_bookmarks_chrome: pass + test_import_bookmarks_edge: + linux: unstable + mac: unstable + win: pass + test_open_all_bookmarks_from_bookmarks_toolbar: pass test_open_bookmark_in_new_window_via_toolbar_context_menu: pass test_open_bookmark_in_private_window_via_toolbar_context_menu: pass test_open_bookmarks_from_toolbar: pass test_open_websites_from_history: pass + test_opened_website_in_new_tab_present_in_hamburger_history_menu: pass + test_opened_website_in_new_window_present_in_hamburger_history_menu: pass + test_opened_website_present_in_hamburger_history_menu: pass test_private_window_website_not_in_history: pass test_toggle_bookmarks_toolbar: pass test_user_can_forget_history: pass downloads: + test_add_mime_type_doc: pass test_add_zip_type: linux: unstable mac: pass win: pass test_download_pdf: pass test_download_pdf_from_context_menu: unstable + test_mixed_content_download_via_https: pass + test_set_always_ask_file_type: pass drag_and_drop: test_copy_entire_row_column: pass + test_copy_from_an_editor_paste_in_another: pass + test_copy_hyperlink_table: pass + test_copy_table_header: pass + test_paste_image_text: pass find_toolbar: test_find_in_pdf: pass test_find_toolbar_nav: pass @@ -69,12 +110,14 @@ form_autofill: test_create_new_cc_profile: pass test_create_profile_autofill: pass test_delete_cc_profile: pass + test_edit_credit_card: pass test_enable_disable_autofill: pass test_form_autofill_suggestions: pass test_name_autofill_attribute: pass test_private_mode_info_not_saved: pass test_telephone_autofill_attribute: pass test_updating_address: pass + test_updating_credit_card: pass geolocation: test_geolocation_shared_via_html5: linux: pass @@ -88,11 +131,13 @@ language_packs: test_language_pack_install_addons: pass test_language_pack_install_preferences: pass menus: + test_copy_paste_actions: pass test_frequently_used_context_menu: linux: pass mac: pass win: unstable test_hyperlink_context_menu: pass + test_image_context_menu_actions: pass test_tab_context_menu_actions: pass test_tab_context_menu_close: pass meta: @@ -123,21 +168,43 @@ notifications: test_screen_share_permission_prompt: pass test_webextension_completed_installation_successfully_displayed: pass password_manager: + test_about_logins_navigation_from_context_menu: pass + test_about_logins_navigation_from_hamburger_menu: pass + test_about_logins_search_username: pass + test_about_logins_search_website: pass test_add_password_non_ascii_chars: pass test_add_password_save_valid_data: pass + test_add_primary_password: pass test_auto_saved_generated_password_context_menu: pass + test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load: pass test_can_view_password_when_PP_enabled: pass test_changes_made_in_edit_mode_are_saved: pass test_delete_login: pass test_multiple_saved_logins: pass test_never_save_login_via_doorhanger: pass + test_password_csv_correctness: + linux: unstable + mac: pass + win: pass + test_password_csv_export: + linux: unstable + mac: pass + win: pass test_primary_password_triggered_on_about_logins_access: pass test_save_login_via_doorhanger: pass + test_saved_hyperlink_redirects_to_corresponding_page: pass test_update_login_via_doorhanger: pass pdf_viewer: + test_add_image_pdf: + linux: unstable + mac: pass + win: pass test_download_pdf_with_form_fields: pass test_download_triggered_on_content_disposition_attachment: pass test_open_pdf_in_FF: pass + test_pdf_data_can_be_cleared: pass + test_pdf_download: pass + test_pdf_input_numbers: pass test_pdf_navigation: pass test_zoom_pdf_viewer: pass pocket: @@ -159,14 +226,20 @@ preferences: test_firefox_home_new_tabs: pass test_firefox_home_on_launch: pass test_lang_pack_changed_from_about_prefs: pass + test_manage_cookie_data: pass test_never_remember_history: pass test_notifications_change_set: pass printing_ui: test_print_preview: pass + test_print_to_pdf: + linux: unstable + mac: pass + win: pass profile: test_set_default_profile: pass reader_view: test_improved_type_control_panel: pass + test_reader_view_close_sidebar: pass test_reader_view_location_bar: pass scrolling_panning_zooming: test_default_zoom_persists: pass @@ -174,6 +247,8 @@ scrolling_panning_zooming: linux: pass mac: unstable win: pass + test_zoom_from_menu_bar: pass + test_zoom_menu_correlation: pass test_zoom_text_only: pass security_and_privacy: test_blocking_cryptominers: pass @@ -200,19 +275,35 @@ security_and_privacy: test_undo_close_tab_private_browsing: pass sync_and_fxa: test_existing_fxa: - test_sync_existing_fxa: pass + linux: unstable + mac: unstable + win: pass test_new_fxa: - test_sync_new_fxa: pass + linux: unstable + mac: unstable + win: pass tabs: test_active_tab: pass + test_change_position_of_pinned_tabs: pass + test_close_pinned_tab_via_mouse: pass + test_close_tab_through_middle_mouse_click: pass + test_display_customize_button: pass + test_list_all_tabs: pass + test_mute_tabs: pass test_navigation_multiple_tabs: pass test_open_bookmark_in_new_tab: pass + test_open_new_bg_tab_via_mouse_and_keyboard: pass test_open_new_tab: pass test_open_new_tab_keys: pass test_open_new_tab_via_hyperlink: pass test_pin_tab: pass + test_pin_unpin_selected_tabs: pass + test_play_mute_unmute_tabs_via_toggle: unstable + test_reload_overiding_cache_keys: pass + test_reload_tab_via_keyboard: pass test_reopen_tab_through_context_menu: pass test_reopen_tab_through_history_menu: pass test_reopen_tabs_through_keys: pass theme_and_toolbar: test_customize_themes_and_redirect: pass + test_installed_theme_enabled: pass diff --git a/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py b/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py index de55b2d51..9659ebb48 100644 --- a/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py +++ b/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py @@ -13,7 +13,6 @@ def test_case(): return "3028795" -@pytest.mark.ci def test_default_search_provider_change_awesome_bar(driver: Firefox): """ C2860208 - Verify that changing the default search provider updates the address bar placeholder. @@ -30,6 +29,6 @@ def test_default_search_provider_change_awesome_bar(driver: Firefox): # Step 3: Re-open new tab and verify placeholder driver.get("about:newtab") - nav.expect_element_attribute_contains( - "awesome-bar", "placeholder", EXPECTED_PLACEHOLDER - ) + nav.expect_element_attribute_contains("awesome-bar", "placeholder", + EXPECTED_PLACEHOLDER) + diff --git a/tests/bookmarks_and_history/test_bookmark_website_via_star_button.py b/tests/bookmarks_and_history/test_bookmark_website_via_star_button.py index 9b71b0cab..d01def77c 100644 --- a/tests/bookmarks_and_history/test_bookmark_website_via_star_button.py +++ b/tests/bookmarks_and_history/test_bookmark_website_via_star_button.py @@ -15,7 +15,6 @@ def test_case(): BOOKMARK_TITLE = "Internet for people" -@pytest.mark.ci def test_bookmark_website_via_star(driver: Firefox): """ C2084539: Verify that the Websites can be bookmarked via star-shaped button diff --git a/tests/downloads/test_set_always_ask_file_type.py b/tests/downloads/test_set_always_ask_file_type.py index b9abd7244..3acb0c3c2 100644 --- a/tests/downloads/test_set_always_ask_file_type.py +++ b/tests/downloads/test_set_always_ask_file_type.py @@ -16,11 +16,9 @@ def delete_files_regex_string(): CONTENT_DISPOSITION_ATTACHMENT_URL = ( - "https://download.novapdf.com/download/samples/pdf-example-bookmarks.pdf" -) + "https://download.novapdf.com/download/samples/pdf-example-bookmarks.pdf") -@pytest.mark.ci def test_set_always_ask_file_type(driver: Firefox, delete_files): """ C1756752 - Ensure that the Always ask option in Firefox Applications settings diff --git a/tests/find_toolbar/test_find_toolbar_search.py b/tests/find_toolbar/test_find_toolbar_search.py index 3c9dabae8..0fd98ddf3 100644 --- a/tests/find_toolbar/test_find_toolbar_search.py +++ b/tests/find_toolbar/test_find_toolbar_search.py @@ -27,10 +27,8 @@ def are_lists_different(a: int, b: int) -> bool: return abs(a - b) > TOLERANCE -@pytest.mark.ci -def test_find_toolbar_search( - driver: Firefox, find_toolbar: FindToolbar, browser_actions: BrowserActions -): +def test_find_toolbar_search(driver: Firefox, find_toolbar: FindToolbar, + browser_actions: BrowserActions): """ C127239: Perform a search (using the Find Toolbar) diff --git a/tests/form_autofill/test_telephone_autofill_attribute.py b/tests/form_autofill/test_telephone_autofill_attribute.py index eeda769c1..a312be24c 100644 --- a/tests/form_autofill/test_telephone_autofill_attribute.py +++ b/tests/form_autofill/test_telephone_autofill_attribute.py @@ -11,7 +11,6 @@ def test_case(): return "122361" -@pytest.mark.ci def test_telephone_attribute_autofill( driver: Firefox, address_autofill: AddressFill, @@ -44,14 +43,16 @@ def test_telephone_attribute_autofill( # Get the first element from the autocomplete dropdown first_item = autofill_popup.get_nth_element(1) - actual_value = autofill_popup.hover(first_item).get_primary_value(first_item) + actual_value = autofill_popup.hover(first_item).get_primary_value( + first_item) # normalize phone number actual_value = util.normalize_phone_number(actual_value) # Get the primary value (telephone) from the first item in the dropdown and assert that the actual value # matches the expected value - expected_telephone = util.normalize_phone_number(autofill_sample_data.telephone) + expected_telephone = util.normalize_phone_number( + autofill_sample_data.telephone) assert expected_telephone == actual_value, ( - f"Expected {expected_telephone}, but got {actual_value}" - ) + f"Expected {expected_telephone}, but got {actual_value}") + diff --git a/tests/menus/test_copy_paste_actions.py b/tests/menus/test_copy_paste_actions.py index 1ccbe090d..dbc7d8347 100644 --- a/tests/menus/test_copy_paste_actions.py +++ b/tests/menus/test_copy_paste_actions.py @@ -15,7 +15,6 @@ def test_case(): return "2264626" -@pytest.mark.ci def test_login_form_copy_paste(driver: Firefox): """C2264626 - Verify that copy and paste actions are displayed in the context menu and work as expected""" # Instantiate objects @@ -25,7 +24,8 @@ def test_login_form_copy_paste(driver: Firefox): random_text = util.generate_random_text("sentence") # Get the field and send text - password_field = login_fill.get_element("input-field", labels=["current-password"]) + password_field = login_fill.get_element("input-field", + labels=["current-password"]) password_field.send_keys(random_text) logging.info(f"Sent the text {random_text} to the textarea.") diff --git a/tests/menus/test_image_context_menu_actions.py b/tests/menus/test_image_context_menu_actions.py index c7fcbe088..7831f7b2b 100644 --- a/tests/menus/test_image_context_menu_actions.py +++ b/tests/menus/test_image_context_menu_actions.py @@ -20,8 +20,7 @@ def delete_files_regex_string(): LINK_IMAGE_URL = ( - "https://en.wikipedia.org/wiki/Firefox#/media/File:Firefox_logo,_2019.svg" -) + "https://en.wikipedia.org/wiki/Firefox#/media/File:Firefox_logo,_2019.svg") LOADED_IMAGE_URL = r"https://upload\.wikimedia\.org/wikipedia/commons/thumb/a/a0/Firefox_logo%2C_2019\.svg/\d+px-Firefox_logo%2C_2019\.svg\.png" SAVED_FILENAME = "Firefox_logo,_2019.svg.png" @@ -43,7 +42,8 @@ def test_open_image_in_new_tab(driver: Firefox): wiki_image_page.context_click(image_logo) # open in a new tab - image_context_menu.click_and_hide_menu("context-menu-open-image-in-new-tab") + image_context_menu.click_and_hide_menu( + "context-menu-open-image-in-new-tab") # switch to the second tab and verify the URL tabs.wait_for_num_tabs(2) @@ -52,7 +52,6 @@ def test_open_image_in_new_tab(driver: Firefox): wiki_image_page.verify_opened_image_url("wikimedia", LOADED_IMAGE_URL) -@pytest.mark.ci @pytest.mark.headed def test_save_image_as(driver: Firefox, sys_platform, delete_files): """ diff --git a/tests/networking/test_http_site.py b/tests/networking/test_http_site.py index 2ffa78536..563f3bdf4 100644 --- a/tests/networking/test_http_site.py +++ b/tests/networking/test_http_site.py @@ -27,7 +27,6 @@ def add_to_prefs_list(): CONNECTION_NOT_SECURE = "Connection is not secure" -@pytest.mark.ci def test_http_site(driver: Firefox): """C2300294 Check that HTTP is allowed when appropriate""" @@ -40,12 +39,12 @@ def test_http_site(driver: Firefox): # Basic functionality prefs.open() - prefs.select_https_only_setting(prefs.HTTPS_ONLY_STATUS.HTTPS_ONLY_DISABLED) + prefs.select_https_only_setting( + prefs.HTTPS_ONLY_STATUS.HTTPS_ONLY_DISABLED) panel_ui.open_and_switch_to_new_window("tab") driver.get(HTTP_SITE) - nav.expect_element_attribute_contains( - "lock-icon", "tooltiptext", CONNECTION_NOT_SECURE - ) + nav.expect_element_attribute_contains("lock-icon", "tooltiptext", + CONNECTION_NOT_SECURE) # Blocking driver.switch_to.window(driver.window_handles[0]) @@ -62,9 +61,8 @@ def test_http_site(driver: Firefox): prefs.select_https_only_setting(prefs.HTTPS_ONLY_STATUS.HTTPS_ONLY_PRIVATE) driver.switch_to.window(driver.window_handles[1]) driver.refresh() - nav.expect_element_attribute_contains( - "lock-icon", "tooltiptext", CONNECTION_NOT_SECURE - ) + nav.expect_element_attribute_contains("lock-icon", "tooltiptext", + CONNECTION_NOT_SECURE) # Private browsing - blocked panel_ui.open_and_switch_to_new_window("private") diff --git a/tests/notifications/test_notifications_displayed.py b/tests/notifications/test_notifications_displayed.py index 4dc293477..58cbd1c5e 100644 --- a/tests/notifications/test_notifications_displayed.py +++ b/tests/notifications/test_notifications_displayed.py @@ -40,11 +40,14 @@ def temp_selectors(): "strategy": "id", "groups": [], }, - "notification-log": {"selectorData": "log", "strategy": "id", "groups": []}, + "notification-log": { + "selectorData": "log", + "strategy": "id", + "groups": [] + }, } -@pytest.mark.ci def test_notifications_displayed(driver: Firefox, temp_page, temp_selectors): """ This test does not (and SHOULD not) test that the OS displays web notifications diff --git a/tests/password_manager/test_about_logins_navigation_from_context_menu.py b/tests/password_manager/test_about_logins_navigation_from_context_menu.py index 5518f9961..107028c9c 100644 --- a/tests/password_manager/test_about_logins_navigation_from_context_menu.py +++ b/tests/password_manager/test_about_logins_navigation_from_context_menu.py @@ -12,7 +12,6 @@ def test_case(): return "2241087" -@pytest.mark.ci def test_about_logins_navigation_from_login_form_context_menu(driver: Firefox): """ C2241087 - Verify that right-clicking the Username field in a login form and then the Manage Passwords option diff --git a/tests/pdf_viewer/test_pdf_data_can_be_cleared.py b/tests/pdf_viewer/test_pdf_data_can_be_cleared.py index 7976bb619..589d1ea80 100644 --- a/tests/pdf_viewer/test_pdf_data_can_be_cleared.py +++ b/tests/pdf_viewer/test_pdf_data_can_be_cleared.py @@ -27,7 +27,6 @@ def file_name(): return "i-9.pdf" -@pytest.mark.ci def test_pdf_data_can_be_cleared( driver: Firefox, pdf_viewer: GenericPdf, @@ -47,7 +46,8 @@ def test_pdf_data_can_be_cleared( # Step 1: Click and type inside the text field for the name section pdf_viewer.fill("first-name-field", TEST_NAME) - pdf_viewer.expect_element_attribute_contains("first-name-field", "value", TEST_NAME) + pdf_viewer.expect_element_attribute_contains("first-name-field", "value", + TEST_NAME) # Step 2: Click over any checkbox and assert the status is updated @@ -57,15 +57,13 @@ def test_pdf_data_can_be_cleared( # Step 3: Select an option from a dropdown and verify the selection dropdown_option = pdf_viewer.select_and_return_dropdown_option( - "state-dropdown-field", By.XPATH, "//option[@value='CA']" - ) + "state-dropdown-field", By.XPATH, "//option[@value='CA']") pdf_viewer.expect(lambda _: dropdown_option.is_displayed()) # Step 4: Delete the text added in step 1 and ensure the input field is empty pdf_viewer.get_element("first-name-field").clear() - pdf_viewer.expect( - lambda _: not pdf_viewer.get_element("first-name-field").get_attribute("value") - ) + pdf_viewer.expect(lambda _: not pdf_viewer.get_element("first-name-field"). + get_attribute("value")) # Step 5: Click the checkbox from step 2 again and ensure it returns to its previous state checkbox.click() @@ -73,7 +71,6 @@ def test_pdf_data_can_be_cleared( # Step 6: Clear the state selection and ensure the field is empty default_option = pdf_viewer.select_and_return_dropdown_option( - "state-dropdown-field", By.XPATH, "//option[@value=' ']" - ) + "state-dropdown-field", By.XPATH, "//option[@value=' ']") default_option.click() pdf_viewer.expect(lambda _: default_option.is_selected()) diff --git a/tests/printing_ui/test_print_preview.py b/tests/printing_ui/test_print_preview.py index f4fe4f112..bbc730ec7 100644 --- a/tests/printing_ui/test_print_preview.py +++ b/tests/printing_ui/test_print_preview.py @@ -19,7 +19,6 @@ def test_print_preview_menu(driver: Firefox): print_preview.open() -@pytest.mark.ci def test_print_preview_keys(driver: Firefox): """C965139 - Check for print preview modal (Key Combo)""" driver.get(TEST_PAGE) diff --git a/tests/reader_view/test_improved_type_control_panel.py b/tests/reader_view/test_improved_type_control_panel.py index b94c4d0d8..acc3c7a7a 100644 --- a/tests/reader_view/test_improved_type_control_panel.py +++ b/tests/reader_view/test_improved_type_control_panel.py @@ -39,7 +39,8 @@ def test_case() -> str: # Helpers -def _open_reader_type_panel(web_page: GenericPage, reader_view: ReaderView) -> None: +def _open_reader_type_panel(web_page: GenericPage, + reader_view: ReaderView) -> None: """ Open the target page, enter Reader View, and open the 'Type' panel. """ @@ -52,14 +53,14 @@ def _css_int(util: Utilities, element, prop: str) -> int: """ Read a CSS property and normalize it to an integer by stripping non-numeric chars. """ - return int(util.remove_all_non_numbers(element.value_of_css_property(prop))) + return int(util.remove_all_non_numbers( + element.value_of_css_property(prop))) -@pytest.mark.ci @pytest.mark.parametrize("font", FONTS) def test_type_control_panel_font( - driver: Firefox, font: Literal["sans-serif", "serif", "monospace"] -) -> None: + driver: Firefox, font: Literal["sans-serif", "serif", + "monospace"]) -> None: """ C130919.1: Ensure the functionality of the type control panels works (font family). """ @@ -72,24 +73,25 @@ def test_type_control_panel_font( # Ensure default is sans-serif first so the next wait has a stable baseline reader_view.wait.until( - lambda _: "sans-serif" in body.value_of_css_property("font-family") - ) + lambda _: "sans-serif" in body.value_of_css_property("font-family")) font_dropdown_root = reader_view.get_element("toolbar-font-selector") - font_dropdown = Dropdown( - page=reader_view, require_shadow=False, root=font_dropdown_root - ) + font_dropdown = Dropdown(page=reader_view, + require_shadow=False, + root=font_dropdown_root) font_dropdown.select_option( f"about-reader-font-type-{font}", option_tag="option", label_name="data-l10n-id", ) - reader_view.wait.until(lambda _: font in body.value_of_css_property("font-family")) + reader_view.wait.until( + lambda _: font in body.value_of_css_property("font-family")) @pytest.mark.parametrize("control", SIZE_CONTROLS) -def test_type_control_panel_size(driver: Firefox, control: SizeControl) -> None: +def test_type_control_panel_size(driver: Firefox, + control: SizeControl) -> None: """ C130919.2: Ensure the functionality of the type control panels works (text size). """ @@ -106,12 +108,10 @@ def test_type_control_panel_size(driver: Firefox, control: SizeControl) -> None: if control == "minus": reader_view.wait.until( - lambda _: _css_int(util, body, "--font-size") < size_before - ) + lambda _: _css_int(util, body, "--font-size") < size_before) else: reader_view.wait.until( - lambda _: _css_int(util, body, "--font-size") > size_before - ) + lambda _: _css_int(util, body, "--font-size") > size_before) @pytest.mark.parametrize("alignment,intended_alignment", ALIGNMENTS) @@ -132,16 +132,13 @@ def test_type_control_panel_text_alignment( reader_view.open_advanced_options() reader_view.get_element(f"toolbar-text-align-{alignment}").click() - reader_view.wait.until( - lambda _: container.value_of_css_property("--text-alignment") - == intended_alignment - ) + reader_view.wait.until(lambda _: container.value_of_css_property( + "--text-alignment") == intended_alignment) @pytest.mark.parametrize("direction", SLIDER_DIRS) -def test_type_control_panel_content_width( - driver: Firefox, direction: SliderDirection -) -> None: +def test_type_control_panel_content_width(driver: Firefox, + direction: SliderDirection) -> None: """ C130919.4: Ensure the functionality of the type control panels works (content width slider). """ @@ -161,18 +158,15 @@ def test_type_control_panel_content_width( if direction == "decrease": reader_view.wait.until( - lambda _: _css_int(util, body, "--content-width") < width_before - ) + lambda _: _css_int(util, body, "--content-width") < width_before) else: reader_view.wait.until( - lambda _: _css_int(util, body, "--content-width") > width_before - ) + lambda _: _css_int(util, body, "--content-width") > width_before) @pytest.mark.parametrize("direction", SLIDER_DIRS) -def test_type_control_panel_line_spacing( - driver: Firefox, direction: SliderDirection -) -> None: +def test_type_control_panel_line_spacing(driver: Firefox, + direction: SliderDirection) -> None: """ C130919.5: Ensure the functionality of the type control panels works (line spacing slider). """ @@ -192,12 +186,10 @@ def test_type_control_panel_line_spacing( if direction == "decrease": reader_view.wait.until( - lambda _: _css_int(util, body, "block-size") < block_before - ) + lambda _: _css_int(util, body, "block-size") < block_before) else: reader_view.wait.until( - lambda _: _css_int(util, body, "block-size") > block_before - ) + lambda _: _css_int(util, body, "block-size") > block_before) def test_type_control_panel_character_spacing(driver: Firefox) -> None: @@ -211,7 +203,8 @@ def test_type_control_panel_character_spacing(driver: Firefox) -> None: _open_reader_type_panel(web_page, reader_view) reader_view.open_advanced_options() - reader_view.change_slider_element_shadow_parent("toolbar-character-spacing") + reader_view.change_slider_element_shadow_parent( + "toolbar-character-spacing") container = web_page.get_element("container-div") letter_before = _css_int(util, container, "--letter-spacing") @@ -219,9 +212,8 @@ def test_type_control_panel_character_spacing(driver: Firefox) -> None: reader_view.change_slider_value(slider, increase=True) - reader_view.wait.until( - lambda _: _css_int(util, container, "--letter-spacing") > letter_before - ) + reader_view.wait.until(lambda _: _css_int( + util, container, "--letter-spacing") > letter_before) def test_type_control_panel_word_spacing(driver: Firefox) -> None: @@ -244,5 +236,5 @@ def test_type_control_panel_word_spacing(driver: Firefox) -> None: reader_view.change_slider_value(slider, increase=True) reader_view.wait.until( - lambda _: _css_int(util, container, "--word-spacing") > word_before - ) + lambda _: _css_int(util, container, "--word-spacing") > word_before) + diff --git a/tests/scrolling_panning_zooming/test_zoom_text_only.py b/tests/scrolling_panning_zooming/test_zoom_text_only.py index c67e32d37..e6ca4d755 100644 --- a/tests/scrolling_panning_zooming/test_zoom_text_only.py +++ b/tests/scrolling_panning_zooming/test_zoom_text_only.py @@ -78,18 +78,17 @@ def reject_consent_page(web_page: GenericPage): try: if web_page.element_clickable("yahoo-consent-page-scroll"): web_page.click_on("yahoo-consent-page-scroll") - web_page.wait.until(lambda _: web_page.element_clickable("yahoo-reject-cookie")) + web_page.wait.until( + lambda _: web_page.element_clickable("yahoo-reject-cookie")) web_page.click_on("yahoo-reject-cookie") except TimeoutException: pass @pytest.mark.skip(reason="Tracked in bug 1991139") -@pytest.mark.ci @pytest.mark.noxvfb -def test_zoom_text_only_from_settings( - driver: Firefox, web_page: GenericPage, reject_consent_page -): +def test_zoom_text_only_from_settings(driver: Firefox, web_page: GenericPage, + reject_consent_page): """ C545733.1: Verify that ticking the zoom text only box would only affect the scale of text. Verify setting the default zoom level applies the chosen zoom level to all websites. @@ -124,9 +123,8 @@ def test_zoom_text_only_from_settings( about_prefs.click_on("zoom-text-only") -def test_zoom_text_only_after_restart( - driver: Firefox, web_page: GenericPage, reject_consent_page -): +def test_zoom_text_only_after_restart(driver: Firefox, web_page: GenericPage, + reject_consent_page): """ C545733.2: Verify that the zoom text only option works after restart @@ -162,16 +160,15 @@ def save_original_data(driver, web_page): Saves the original positions and sizes for comparison. """ driver.switch_to.window(driver.window_handles[0]) - original_website1_image_position = web_page.get_element("yahoo-logo").location["x"] + original_website1_image_position = web_page.get_element( + "yahoo-logo").location["x"] original_website1_text_position = web_page.get_element( - "yahoo-login-button" - ).location["x"] + "yahoo-login-button").location["x"] driver.switch_to.window(driver.window_handles[1]) original_website2_image_size = web_page.get_element("duckduckgo-logo").size original_website2_text_position = web_page.get_element( - "duckduckgo-tagline" - ).location["x"] + "duckduckgo-tagline").location["x"] return ( original_website1_image_position, @@ -195,7 +192,8 @@ def zoom_text_only_functionality_test(driver, nav, web_page, original_data): # Verify Yahoo: image position unchanged, text position changed driver.switch_to.window(driver.window_handles[0]) new_image_position = web_page.get_element("yahoo-logo").location["x"] - new_text_position = web_page.get_element("yahoo-login-button").location["x"] + new_text_position = web_page.get_element( + "yahoo-login-button").location["x"] assert new_image_position == original_website1_image_position assert new_text_position < original_website1_text_position @@ -207,24 +205,20 @@ def zoom_text_only_functionality_test(driver, nav, web_page, original_data): # Verify that zoom level badge is correct with driver.context(driver.CONTEXT_CHROME): - nav.expect_element_attribute_contains("toolbar-zoom-level", "label", "90%") + nav.expect_element_attribute_contains("toolbar-zoom-level", "label", + "90%") # Verify Yahoo at 90%: image position still unchanged, text position changed - assert ( - web_page.get_element("yahoo-logo").location["x"] - == original_website1_image_position - ) - assert ( - web_page.get_element("yahoo-login-button").location["x"] - > original_website1_text_position - ) + assert (web_page.get_element("yahoo-logo").location["x"] == + original_website1_image_position) + assert (web_page.get_element("yahoo-login-button").location["x"] + > original_website1_text_position) # Verify DuckDuckGo: image SIZE unchanged, text position changed driver.switch_to.window(driver.window_handles[1]) - assert ( - web_page.get_element("duckduckgo-logo").size == original_website2_image_size - ), "DuckDuckGo image size should not change (text-only zoom)" - assert ( - web_page.get_element("duckduckgo-tagline").location["x"] - < original_website2_text_position - ) + assert (web_page.get_element( + "duckduckgo-logo").size == original_website2_image_size + ), "DuckDuckGo image size should not change (text-only zoom)" + assert (web_page.get_element("duckduckgo-tagline").location["x"] + < original_website2_text_position) + diff --git a/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py b/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py index b8e48d669..800749676 100644 --- a/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py +++ b/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py @@ -11,13 +11,11 @@ def test_case(): FIRST_TRACKER_WEBSITE = "https://senglehardt.com/test/trackingprotection/test_pages/tracking_protection.html" -ALLOWED_COOKIES = set( - [ - "https://ads-track-digest256.dummytracker.org", - "https://social-track-digest256.dummytracker.org", - "https://analytics-track-digest256.dummytracker.org", - ] -) +ALLOWED_COOKIES = set([ + "https://ads-track-digest256.dummytracker.org", + "https://social-track-digest256.dummytracker.org", + "https://analytics-track-digest256.dummytracker.org", +]) @pytest.fixture() @@ -30,7 +28,6 @@ def add_to_prefs_list(): ] -@pytest.mark.ci def test_cross_site_tracking_cookies_blocked(driver: Firefox): """ C446402: Ensures the cross tracking cookies are displayed in the tracker panel diff --git a/tests/tabs/test_active_tab.py b/tests/tabs/test_active_tab.py index 9ff4c8904..12245ddd4 100644 --- a/tests/tabs/test_active_tab.py +++ b/tests/tabs/test_active_tab.py @@ -11,7 +11,6 @@ def test_case(): return "134646" -@pytest.mark.ci def test_active_tab(driver: Firefox): """ C134646, ensures that the selected tab is highlighted diff --git a/tests/theme_and_toolbar/test_customize_themes_and_redirect.py b/tests/theme_and_toolbar/test_customize_themes_and_redirect.py index a732c01ad..21df4b0f0 100644 --- a/tests/theme_and_toolbar/test_customize_themes_and_redirect.py +++ b/tests/theme_and_toolbar/test_customize_themes_and_redirect.py @@ -61,7 +61,6 @@ def colors_match(a: str, b: str, tolerance: float = 0.14) -> bool: return True -@pytest.mark.ci def test_redirect_to_addons(driver: Firefox) -> None: """ C118173: Ensure the user is redirected to about:addons via the UI panel. @@ -83,9 +82,8 @@ def test_redirect_to_addons(driver: Firefox) -> None: @pytest.mark.parametrize("theme_name", list(THEMES.keys())) -def test_activate_theme_background_matches_expected( - driver: Firefox, theme_name: str -) -> None: +def test_activate_theme_background_matches_expected(driver: Firefox, + theme_name: str) -> None: """ C118173: Ensure that activating each theme in about:addons applies the expected background color. Handles Developer Edition vs standard Firefox defaults. @@ -103,12 +101,14 @@ def test_activate_theme_background_matches_expected( if theme_name == "firefox-compact-light_mozilla_org-heading": pytest.skip("Compact Light is default on Firefox, skipping.") - current_bg = abt_addons.activate_theme(nav, theme_name, "", perform_assert=False) + current_bg = abt_addons.activate_theme(nav, + theme_name, + "", + perform_assert=False) expected_list = THEMES[theme_name] assert any(colors_match(current_bg, exp) for exp in expected_list), ( - f"Got {current_bg} for {theme_name}; expected one of {expected_list}" - ) + f"Got {current_bg} for {theme_name}; expected one of {expected_list}") def test_alpenglow_theme(driver: Firefox) -> None: @@ -123,9 +123,8 @@ def test_alpenglow_theme(driver: Firefox) -> None: abt_addons.choose_sidebar_option("theme") current_bg = abt_addons.activate_theme( - nav, "firefox-alpenglow_mozilla_org-heading", "", perform_assert=False - ) + nav, "firefox-alpenglow_mozilla_org-heading", "", perform_assert=False) assert colors_match(current_bg, ALPENGLOW_MAP["light"]) or colors_match( - current_bg, ALPENGLOW_MAP["dark"] - ) + current_bg, ALPENGLOW_MAP["dark"]) + From 42f408802f7635bbaa2fb2f31da7bf67a826298a Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Thu, 20 Nov 2025 13:54:01 -0800 Subject: [PATCH 20/29] lint --- choose_ci_set.py | 50 ++++++------- ...ault_search_provider_change_awesome_bar.py | 6 +- .../test_set_always_ask_file_type.py | 3 +- .../find_toolbar/test_find_toolbar_search.py | 5 +- .../test_telephone_autofill_attribute.py | 10 ++- tests/menus/test_copy_paste_actions.py | 3 +- .../menus/test_image_context_menu_actions.py | 6 +- tests/networking/test_http_site.py | 13 ++-- .../test_notifications_displayed.py | 6 +- .../test_pdf_data_can_be_cleared.py | 14 ++-- .../test_improved_type_control_panel.py | 71 ++++++++++--------- .../test_zoom_text_only.py | 53 +++++++------- ...est_cross_site_tracking_cookies_blocked.py | 12 ++-- .../test_customize_themes_and_redirect.py | 20 +++--- 14 files changed, 143 insertions(+), 129 deletions(-) diff --git a/choose_ci_set.py b/choose_ci_set.py index d80d849a9..2ec8575a8 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -47,8 +47,9 @@ def localify(path: str) -> str: return path.replace(SCRIPT_DIR, ".") -def get_tests_by_model(model_name: str, test_paths_and_contents: dict, - run_list: list) -> list: +def get_tests_by_model( + model_name: str, test_paths_and_contents: dict, run_list: list +) -> list: """ Given a model name, a dict of paths and their file contents, and the list of existing tests/dirs to check, return matches by local paths @@ -73,8 +74,11 @@ def dedupe(run_list: list) -> list: removes = [] for i, entry in enumerate(run_list): - if (not entry.startswith(".") and not entry.startswith("\\") - and not entry.startswith("/")): + if ( + not entry.startswith(".") + and not entry.startswith("\\") + and not entry.startswith("/") + ): dotslashes.append(i) for dotslash in dotslashes: @@ -138,15 +142,15 @@ def convert_manifest_to_list(manifest_loc): addtest = True elif isinstance(mkey[suite][testfile][subtest], dict): if sysname() in mkey[suite][testfile][subtest]: - if mkey[suite][testfile][subtest][ - sysname()] == "pass": + if mkey[suite][testfile][subtest][sysname()] == "pass": test_name = f"{test_name}::{subtest}" addtest = True if addtest: test_to_add = SLASH.join(toplevel + [suite, test_name]) assert os.path.exists(test_to_add.split("::")[0]), ( - f"{test_to_add} could not be found") + f"{test_to_add} could not be found" + ) tests.append(test_to_add) addtest = False return tests @@ -192,9 +196,12 @@ def convert_manifest_to_list(manifest_loc): run_list = [] check_output(["git", "fetch", "--quiet", "--depth=1", "origin", "main"]) - committed_files = (check_output( - ["git", "--no-pager", "diff", "--name-only", - "origin/main"]).decode().replace("/", SLASH).splitlines()) + committed_files = ( + check_output(["git", "--no-pager", "diff", "--name-only", "origin/main"]) + .decode() + .replace("/", SLASH) + .splitlines() + ) main_conftest = "conftest.py" base_page = os.path.join("modules", "page_base.py") @@ -212,8 +219,7 @@ def convert_manifest_to_list(manifest_loc): for root, _, files in os.walk(os.path.join(SCRIPT_DIR, "tests")): for f in files: this_file = os.path.join(root, f) - if re_obj.get("test_re").search( - this_file) and "__pycache" not in this_file: + if re_obj.get("test_re").search(this_file) and "__pycache" not in this_file: all_tests.append(os.path.join(this_file)) with open(this_file, encoding="utf-8") as fh: lines = fh.readlines() @@ -228,9 +234,7 @@ def convert_manifest_to_list(manifest_loc): changed_models = [ f for f in committed_files if re_obj.get("object_model_re").match(f) ] - changed_tests = [ - f for f in committed_files if re_obj.get("test_re").match(f) - ] + changed_tests = [f for f in committed_files if re_obj.get("test_re").match(f)] if changed_suite_conftests: run_list = [ @@ -242,9 +246,9 @@ def convert_manifest_to_list(manifest_loc): for selector_file in changed_selectors: (_, filename) = os.path.split(selector_file) model_name = pascalify(filename.split(".")[0]) - for test_name in get_tests_by_model(model_name, - test_paths_and_contents, - run_list): + for test_name in get_tests_by_model( + model_name, test_paths_and_contents, run_list + ): run_list.append(test_name) if changed_models: @@ -252,9 +256,9 @@ def convert_manifest_to_list(manifest_loc): model_file_contents = "".join([line for line in open(model_file)]) classes = re_obj.get("class_re").findall(model_file_contents) for model_name in classes: - for test_name in get_tests_by_model(model_name, - test_paths_and_contents, - run_list): + for test_name in get_tests_by_model( + model_name, test_paths_and_contents, run_list + ): run_list.append(test_name) if changed_tests: @@ -286,8 +290,6 @@ def convert_manifest_to_list(manifest_loc): if SLASH == "\\": run_list = [entry.replace("/", SLASH) for entry in run_list] run_list = dedupe(run_list) - run_list = [ - entry for entry in run_list if os.path.exists(entry.split("::")[0]) - ] + run_list = [entry for entry in run_list if os.path.exists(entry.split("::")[0])] with open(OUTPUT_FILE, "w") as fh: fh.write("\n".join(run_list)) diff --git a/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py b/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py index 9659ebb48..4ab88b435 100644 --- a/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py +++ b/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py @@ -29,6 +29,6 @@ def test_default_search_provider_change_awesome_bar(driver: Firefox): # Step 3: Re-open new tab and verify placeholder driver.get("about:newtab") - nav.expect_element_attribute_contains("awesome-bar", "placeholder", - EXPECTED_PLACEHOLDER) - + nav.expect_element_attribute_contains( + "awesome-bar", "placeholder", EXPECTED_PLACEHOLDER + ) diff --git a/tests/downloads/test_set_always_ask_file_type.py b/tests/downloads/test_set_always_ask_file_type.py index 3acb0c3c2..330b1cc5a 100644 --- a/tests/downloads/test_set_always_ask_file_type.py +++ b/tests/downloads/test_set_always_ask_file_type.py @@ -16,7 +16,8 @@ def delete_files_regex_string(): CONTENT_DISPOSITION_ATTACHMENT_URL = ( - "https://download.novapdf.com/download/samples/pdf-example-bookmarks.pdf") + "https://download.novapdf.com/download/samples/pdf-example-bookmarks.pdf" +) def test_set_always_ask_file_type(driver: Firefox, delete_files): diff --git a/tests/find_toolbar/test_find_toolbar_search.py b/tests/find_toolbar/test_find_toolbar_search.py index 0fd98ddf3..9f06198ac 100644 --- a/tests/find_toolbar/test_find_toolbar_search.py +++ b/tests/find_toolbar/test_find_toolbar_search.py @@ -27,8 +27,9 @@ def are_lists_different(a: int, b: int) -> bool: return abs(a - b) > TOLERANCE -def test_find_toolbar_search(driver: Firefox, find_toolbar: FindToolbar, - browser_actions: BrowserActions): +def test_find_toolbar_search( + driver: Firefox, find_toolbar: FindToolbar, browser_actions: BrowserActions +): """ C127239: Perform a search (using the Find Toolbar) diff --git a/tests/form_autofill/test_telephone_autofill_attribute.py b/tests/form_autofill/test_telephone_autofill_attribute.py index a312be24c..11803d8e4 100644 --- a/tests/form_autofill/test_telephone_autofill_attribute.py +++ b/tests/form_autofill/test_telephone_autofill_attribute.py @@ -43,16 +43,14 @@ def test_telephone_attribute_autofill( # Get the first element from the autocomplete dropdown first_item = autofill_popup.get_nth_element(1) - actual_value = autofill_popup.hover(first_item).get_primary_value( - first_item) + actual_value = autofill_popup.hover(first_item).get_primary_value(first_item) # normalize phone number actual_value = util.normalize_phone_number(actual_value) # Get the primary value (telephone) from the first item in the dropdown and assert that the actual value # matches the expected value - expected_telephone = util.normalize_phone_number( - autofill_sample_data.telephone) + expected_telephone = util.normalize_phone_number(autofill_sample_data.telephone) assert expected_telephone == actual_value, ( - f"Expected {expected_telephone}, but got {actual_value}") - + f"Expected {expected_telephone}, but got {actual_value}" + ) diff --git a/tests/menus/test_copy_paste_actions.py b/tests/menus/test_copy_paste_actions.py index dbc7d8347..e1d97c53d 100644 --- a/tests/menus/test_copy_paste_actions.py +++ b/tests/menus/test_copy_paste_actions.py @@ -24,8 +24,7 @@ def test_login_form_copy_paste(driver: Firefox): random_text = util.generate_random_text("sentence") # Get the field and send text - password_field = login_fill.get_element("input-field", - labels=["current-password"]) + password_field = login_fill.get_element("input-field", labels=["current-password"]) password_field.send_keys(random_text) logging.info(f"Sent the text {random_text} to the textarea.") diff --git a/tests/menus/test_image_context_menu_actions.py b/tests/menus/test_image_context_menu_actions.py index 7831f7b2b..8aaa42796 100644 --- a/tests/menus/test_image_context_menu_actions.py +++ b/tests/menus/test_image_context_menu_actions.py @@ -20,7 +20,8 @@ def delete_files_regex_string(): LINK_IMAGE_URL = ( - "https://en.wikipedia.org/wiki/Firefox#/media/File:Firefox_logo,_2019.svg") + "https://en.wikipedia.org/wiki/Firefox#/media/File:Firefox_logo,_2019.svg" +) LOADED_IMAGE_URL = r"https://upload\.wikimedia\.org/wikipedia/commons/thumb/a/a0/Firefox_logo%2C_2019\.svg/\d+px-Firefox_logo%2C_2019\.svg\.png" SAVED_FILENAME = "Firefox_logo,_2019.svg.png" @@ -42,8 +43,7 @@ def test_open_image_in_new_tab(driver: Firefox): wiki_image_page.context_click(image_logo) # open in a new tab - image_context_menu.click_and_hide_menu( - "context-menu-open-image-in-new-tab") + image_context_menu.click_and_hide_menu("context-menu-open-image-in-new-tab") # switch to the second tab and verify the URL tabs.wait_for_num_tabs(2) diff --git a/tests/networking/test_http_site.py b/tests/networking/test_http_site.py index 563f3bdf4..d53f67153 100644 --- a/tests/networking/test_http_site.py +++ b/tests/networking/test_http_site.py @@ -39,12 +39,12 @@ def test_http_site(driver: Firefox): # Basic functionality prefs.open() - prefs.select_https_only_setting( - prefs.HTTPS_ONLY_STATUS.HTTPS_ONLY_DISABLED) + prefs.select_https_only_setting(prefs.HTTPS_ONLY_STATUS.HTTPS_ONLY_DISABLED) panel_ui.open_and_switch_to_new_window("tab") driver.get(HTTP_SITE) - nav.expect_element_attribute_contains("lock-icon", "tooltiptext", - CONNECTION_NOT_SECURE) + nav.expect_element_attribute_contains( + "lock-icon", "tooltiptext", CONNECTION_NOT_SECURE + ) # Blocking driver.switch_to.window(driver.window_handles[0]) @@ -61,8 +61,9 @@ def test_http_site(driver: Firefox): prefs.select_https_only_setting(prefs.HTTPS_ONLY_STATUS.HTTPS_ONLY_PRIVATE) driver.switch_to.window(driver.window_handles[1]) driver.refresh() - nav.expect_element_attribute_contains("lock-icon", "tooltiptext", - CONNECTION_NOT_SECURE) + nav.expect_element_attribute_contains( + "lock-icon", "tooltiptext", CONNECTION_NOT_SECURE + ) # Private browsing - blocked panel_ui.open_and_switch_to_new_window("private") diff --git a/tests/notifications/test_notifications_displayed.py b/tests/notifications/test_notifications_displayed.py index 58cbd1c5e..3fdcb94e8 100644 --- a/tests/notifications/test_notifications_displayed.py +++ b/tests/notifications/test_notifications_displayed.py @@ -40,11 +40,7 @@ def temp_selectors(): "strategy": "id", "groups": [], }, - "notification-log": { - "selectorData": "log", - "strategy": "id", - "groups": [] - }, + "notification-log": {"selectorData": "log", "strategy": "id", "groups": []}, } diff --git a/tests/pdf_viewer/test_pdf_data_can_be_cleared.py b/tests/pdf_viewer/test_pdf_data_can_be_cleared.py index 589d1ea80..dd3fc291c 100644 --- a/tests/pdf_viewer/test_pdf_data_can_be_cleared.py +++ b/tests/pdf_viewer/test_pdf_data_can_be_cleared.py @@ -46,8 +46,7 @@ def test_pdf_data_can_be_cleared( # Step 1: Click and type inside the text field for the name section pdf_viewer.fill("first-name-field", TEST_NAME) - pdf_viewer.expect_element_attribute_contains("first-name-field", "value", - TEST_NAME) + pdf_viewer.expect_element_attribute_contains("first-name-field", "value", TEST_NAME) # Step 2: Click over any checkbox and assert the status is updated @@ -57,13 +56,15 @@ def test_pdf_data_can_be_cleared( # Step 3: Select an option from a dropdown and verify the selection dropdown_option = pdf_viewer.select_and_return_dropdown_option( - "state-dropdown-field", By.XPATH, "//option[@value='CA']") + "state-dropdown-field", By.XPATH, "//option[@value='CA']" + ) pdf_viewer.expect(lambda _: dropdown_option.is_displayed()) # Step 4: Delete the text added in step 1 and ensure the input field is empty pdf_viewer.get_element("first-name-field").clear() - pdf_viewer.expect(lambda _: not pdf_viewer.get_element("first-name-field"). - get_attribute("value")) + pdf_viewer.expect( + lambda _: not pdf_viewer.get_element("first-name-field").get_attribute("value") + ) # Step 5: Click the checkbox from step 2 again and ensure it returns to its previous state checkbox.click() @@ -71,6 +72,7 @@ def test_pdf_data_can_be_cleared( # Step 6: Clear the state selection and ensure the field is empty default_option = pdf_viewer.select_and_return_dropdown_option( - "state-dropdown-field", By.XPATH, "//option[@value=' ']") + "state-dropdown-field", By.XPATH, "//option[@value=' ']" + ) default_option.click() pdf_viewer.expect(lambda _: default_option.is_selected()) diff --git a/tests/reader_view/test_improved_type_control_panel.py b/tests/reader_view/test_improved_type_control_panel.py index acc3c7a7a..47a5ca795 100644 --- a/tests/reader_view/test_improved_type_control_panel.py +++ b/tests/reader_view/test_improved_type_control_panel.py @@ -39,8 +39,7 @@ def test_case() -> str: # Helpers -def _open_reader_type_panel(web_page: GenericPage, - reader_view: ReaderView) -> None: +def _open_reader_type_panel(web_page: GenericPage, reader_view: ReaderView) -> None: """ Open the target page, enter Reader View, and open the 'Type' panel. """ @@ -53,14 +52,13 @@ def _css_int(util: Utilities, element, prop: str) -> int: """ Read a CSS property and normalize it to an integer by stripping non-numeric chars. """ - return int(util.remove_all_non_numbers( - element.value_of_css_property(prop))) + return int(util.remove_all_non_numbers(element.value_of_css_property(prop))) @pytest.mark.parametrize("font", FONTS) def test_type_control_panel_font( - driver: Firefox, font: Literal["sans-serif", "serif", - "monospace"]) -> None: + driver: Firefox, font: Literal["sans-serif", "serif", "monospace"] +) -> None: """ C130919.1: Ensure the functionality of the type control panels works (font family). """ @@ -73,25 +71,24 @@ def test_type_control_panel_font( # Ensure default is sans-serif first so the next wait has a stable baseline reader_view.wait.until( - lambda _: "sans-serif" in body.value_of_css_property("font-family")) + lambda _: "sans-serif" in body.value_of_css_property("font-family") + ) font_dropdown_root = reader_view.get_element("toolbar-font-selector") - font_dropdown = Dropdown(page=reader_view, - require_shadow=False, - root=font_dropdown_root) + font_dropdown = Dropdown( + page=reader_view, require_shadow=False, root=font_dropdown_root + ) font_dropdown.select_option( f"about-reader-font-type-{font}", option_tag="option", label_name="data-l10n-id", ) - reader_view.wait.until( - lambda _: font in body.value_of_css_property("font-family")) + reader_view.wait.until(lambda _: font in body.value_of_css_property("font-family")) @pytest.mark.parametrize("control", SIZE_CONTROLS) -def test_type_control_panel_size(driver: Firefox, - control: SizeControl) -> None: +def test_type_control_panel_size(driver: Firefox, control: SizeControl) -> None: """ C130919.2: Ensure the functionality of the type control panels works (text size). """ @@ -108,10 +105,12 @@ def test_type_control_panel_size(driver: Firefox, if control == "minus": reader_view.wait.until( - lambda _: _css_int(util, body, "--font-size") < size_before) + lambda _: _css_int(util, body, "--font-size") < size_before + ) else: reader_view.wait.until( - lambda _: _css_int(util, body, "--font-size") > size_before) + lambda _: _css_int(util, body, "--font-size") > size_before + ) @pytest.mark.parametrize("alignment,intended_alignment", ALIGNMENTS) @@ -132,13 +131,16 @@ def test_type_control_panel_text_alignment( reader_view.open_advanced_options() reader_view.get_element(f"toolbar-text-align-{alignment}").click() - reader_view.wait.until(lambda _: container.value_of_css_property( - "--text-alignment") == intended_alignment) + reader_view.wait.until( + lambda _: container.value_of_css_property("--text-alignment") + == intended_alignment + ) @pytest.mark.parametrize("direction", SLIDER_DIRS) -def test_type_control_panel_content_width(driver: Firefox, - direction: SliderDirection) -> None: +def test_type_control_panel_content_width( + driver: Firefox, direction: SliderDirection +) -> None: """ C130919.4: Ensure the functionality of the type control panels works (content width slider). """ @@ -158,15 +160,18 @@ def test_type_control_panel_content_width(driver: Firefox, if direction == "decrease": reader_view.wait.until( - lambda _: _css_int(util, body, "--content-width") < width_before) + lambda _: _css_int(util, body, "--content-width") < width_before + ) else: reader_view.wait.until( - lambda _: _css_int(util, body, "--content-width") > width_before) + lambda _: _css_int(util, body, "--content-width") > width_before + ) @pytest.mark.parametrize("direction", SLIDER_DIRS) -def test_type_control_panel_line_spacing(driver: Firefox, - direction: SliderDirection) -> None: +def test_type_control_panel_line_spacing( + driver: Firefox, direction: SliderDirection +) -> None: """ C130919.5: Ensure the functionality of the type control panels works (line spacing slider). """ @@ -186,10 +191,12 @@ def test_type_control_panel_line_spacing(driver: Firefox, if direction == "decrease": reader_view.wait.until( - lambda _: _css_int(util, body, "block-size") < block_before) + lambda _: _css_int(util, body, "block-size") < block_before + ) else: reader_view.wait.until( - lambda _: _css_int(util, body, "block-size") > block_before) + lambda _: _css_int(util, body, "block-size") > block_before + ) def test_type_control_panel_character_spacing(driver: Firefox) -> None: @@ -203,8 +210,7 @@ def test_type_control_panel_character_spacing(driver: Firefox) -> None: _open_reader_type_panel(web_page, reader_view) reader_view.open_advanced_options() - reader_view.change_slider_element_shadow_parent( - "toolbar-character-spacing") + reader_view.change_slider_element_shadow_parent("toolbar-character-spacing") container = web_page.get_element("container-div") letter_before = _css_int(util, container, "--letter-spacing") @@ -212,8 +218,9 @@ def test_type_control_panel_character_spacing(driver: Firefox) -> None: reader_view.change_slider_value(slider, increase=True) - reader_view.wait.until(lambda _: _css_int( - util, container, "--letter-spacing") > letter_before) + reader_view.wait.until( + lambda _: _css_int(util, container, "--letter-spacing") > letter_before + ) def test_type_control_panel_word_spacing(driver: Firefox) -> None: @@ -236,5 +243,5 @@ def test_type_control_panel_word_spacing(driver: Firefox) -> None: reader_view.change_slider_value(slider, increase=True) reader_view.wait.until( - lambda _: _css_int(util, container, "--word-spacing") > word_before) - + lambda _: _css_int(util, container, "--word-spacing") > word_before + ) diff --git a/tests/scrolling_panning_zooming/test_zoom_text_only.py b/tests/scrolling_panning_zooming/test_zoom_text_only.py index e6ca4d755..cc1c6a2d5 100644 --- a/tests/scrolling_panning_zooming/test_zoom_text_only.py +++ b/tests/scrolling_panning_zooming/test_zoom_text_only.py @@ -78,8 +78,7 @@ def reject_consent_page(web_page: GenericPage): try: if web_page.element_clickable("yahoo-consent-page-scroll"): web_page.click_on("yahoo-consent-page-scroll") - web_page.wait.until( - lambda _: web_page.element_clickable("yahoo-reject-cookie")) + web_page.wait.until(lambda _: web_page.element_clickable("yahoo-reject-cookie")) web_page.click_on("yahoo-reject-cookie") except TimeoutException: pass @@ -87,8 +86,9 @@ def reject_consent_page(web_page: GenericPage): @pytest.mark.skip(reason="Tracked in bug 1991139") @pytest.mark.noxvfb -def test_zoom_text_only_from_settings(driver: Firefox, web_page: GenericPage, - reject_consent_page): +def test_zoom_text_only_from_settings( + driver: Firefox, web_page: GenericPage, reject_consent_page +): """ C545733.1: Verify that ticking the zoom text only box would only affect the scale of text. Verify setting the default zoom level applies the chosen zoom level to all websites. @@ -123,8 +123,9 @@ def test_zoom_text_only_from_settings(driver: Firefox, web_page: GenericPage, about_prefs.click_on("zoom-text-only") -def test_zoom_text_only_after_restart(driver: Firefox, web_page: GenericPage, - reject_consent_page): +def test_zoom_text_only_after_restart( + driver: Firefox, web_page: GenericPage, reject_consent_page +): """ C545733.2: Verify that the zoom text only option works after restart @@ -160,15 +161,16 @@ def save_original_data(driver, web_page): Saves the original positions and sizes for comparison. """ driver.switch_to.window(driver.window_handles[0]) - original_website1_image_position = web_page.get_element( - "yahoo-logo").location["x"] + original_website1_image_position = web_page.get_element("yahoo-logo").location["x"] original_website1_text_position = web_page.get_element( - "yahoo-login-button").location["x"] + "yahoo-login-button" + ).location["x"] driver.switch_to.window(driver.window_handles[1]) original_website2_image_size = web_page.get_element("duckduckgo-logo").size original_website2_text_position = web_page.get_element( - "duckduckgo-tagline").location["x"] + "duckduckgo-tagline" + ).location["x"] return ( original_website1_image_position, @@ -192,8 +194,7 @@ def zoom_text_only_functionality_test(driver, nav, web_page, original_data): # Verify Yahoo: image position unchanged, text position changed driver.switch_to.window(driver.window_handles[0]) new_image_position = web_page.get_element("yahoo-logo").location["x"] - new_text_position = web_page.get_element( - "yahoo-login-button").location["x"] + new_text_position = web_page.get_element("yahoo-login-button").location["x"] assert new_image_position == original_website1_image_position assert new_text_position < original_website1_text_position @@ -205,20 +206,24 @@ def zoom_text_only_functionality_test(driver, nav, web_page, original_data): # Verify that zoom level badge is correct with driver.context(driver.CONTEXT_CHROME): - nav.expect_element_attribute_contains("toolbar-zoom-level", "label", - "90%") + nav.expect_element_attribute_contains("toolbar-zoom-level", "label", "90%") # Verify Yahoo at 90%: image position still unchanged, text position changed - assert (web_page.get_element("yahoo-logo").location["x"] == - original_website1_image_position) - assert (web_page.get_element("yahoo-login-button").location["x"] - > original_website1_text_position) + assert ( + web_page.get_element("yahoo-logo").location["x"] + == original_website1_image_position + ) + assert ( + web_page.get_element("yahoo-login-button").location["x"] + > original_website1_text_position + ) # Verify DuckDuckGo: image SIZE unchanged, text position changed driver.switch_to.window(driver.window_handles[1]) - assert (web_page.get_element( - "duckduckgo-logo").size == original_website2_image_size - ), "DuckDuckGo image size should not change (text-only zoom)" - assert (web_page.get_element("duckduckgo-tagline").location["x"] - < original_website2_text_position) - + assert ( + web_page.get_element("duckduckgo-logo").size == original_website2_image_size + ), "DuckDuckGo image size should not change (text-only zoom)" + assert ( + web_page.get_element("duckduckgo-tagline").location["x"] + < original_website2_text_position + ) diff --git a/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py b/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py index 800749676..7875dcbb3 100644 --- a/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py +++ b/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py @@ -11,11 +11,13 @@ def test_case(): FIRST_TRACKER_WEBSITE = "https://senglehardt.com/test/trackingprotection/test_pages/tracking_protection.html" -ALLOWED_COOKIES = set([ - "https://ads-track-digest256.dummytracker.org", - "https://social-track-digest256.dummytracker.org", - "https://analytics-track-digest256.dummytracker.org", -]) +ALLOWED_COOKIES = set( + [ + "https://ads-track-digest256.dummytracker.org", + "https://social-track-digest256.dummytracker.org", + "https://analytics-track-digest256.dummytracker.org", + ] +) @pytest.fixture() diff --git a/tests/theme_and_toolbar/test_customize_themes_and_redirect.py b/tests/theme_and_toolbar/test_customize_themes_and_redirect.py index 21df4b0f0..c506a31d4 100644 --- a/tests/theme_and_toolbar/test_customize_themes_and_redirect.py +++ b/tests/theme_and_toolbar/test_customize_themes_and_redirect.py @@ -82,8 +82,9 @@ def test_redirect_to_addons(driver: Firefox) -> None: @pytest.mark.parametrize("theme_name", list(THEMES.keys())) -def test_activate_theme_background_matches_expected(driver: Firefox, - theme_name: str) -> None: +def test_activate_theme_background_matches_expected( + driver: Firefox, theme_name: str +) -> None: """ C118173: Ensure that activating each theme in about:addons applies the expected background color. Handles Developer Edition vs standard Firefox defaults. @@ -101,14 +102,12 @@ def test_activate_theme_background_matches_expected(driver: Firefox, if theme_name == "firefox-compact-light_mozilla_org-heading": pytest.skip("Compact Light is default on Firefox, skipping.") - current_bg = abt_addons.activate_theme(nav, - theme_name, - "", - perform_assert=False) + current_bg = abt_addons.activate_theme(nav, theme_name, "", perform_assert=False) expected_list = THEMES[theme_name] assert any(colors_match(current_bg, exp) for exp in expected_list), ( - f"Got {current_bg} for {theme_name}; expected one of {expected_list}") + f"Got {current_bg} for {theme_name}; expected one of {expected_list}" + ) def test_alpenglow_theme(driver: Firefox) -> None: @@ -123,8 +122,9 @@ def test_alpenglow_theme(driver: Firefox) -> None: abt_addons.choose_sidebar_option("theme") current_bg = abt_addons.activate_theme( - nav, "firefox-alpenglow_mozilla_org-heading", "", perform_assert=False) + nav, "firefox-alpenglow_mozilla_org-heading", "", perform_assert=False + ) assert colors_match(current_bg, ALPENGLOW_MAP["light"]) or colors_match( - current_bg, ALPENGLOW_MAP["dark"]) - + current_bg, ALPENGLOW_MAP["dark"] + ) From 764d5c4c3228e156cc6755c3b0e511d0eced9a2d Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Thu, 20 Nov 2025 13:57:15 -0800 Subject: [PATCH 21/29] all -> smoke as default manifest --- choose_ci_set.py | 54 +++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/choose_ci_set.py b/choose_ci_set.py index 2ec8575a8..6f50888dd 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -47,9 +47,8 @@ def localify(path: str) -> str: return path.replace(SCRIPT_DIR, ".") -def get_tests_by_model( - model_name: str, test_paths_and_contents: dict, run_list: list -) -> list: +def get_tests_by_model(model_name: str, test_paths_and_contents: dict, + run_list: list) -> list: """ Given a model name, a dict of paths and their file contents, and the list of existing tests/dirs to check, return matches by local paths @@ -74,11 +73,8 @@ def dedupe(run_list: list) -> list: removes = [] for i, entry in enumerate(run_list): - if ( - not entry.startswith(".") - and not entry.startswith("\\") - and not entry.startswith("/") - ): + if (not entry.startswith(".") and not entry.startswith("\\") + and not entry.startswith("/")): dotslashes.append(i) for dotslash in dotslashes: @@ -142,15 +138,15 @@ def convert_manifest_to_list(manifest_loc): addtest = True elif isinstance(mkey[suite][testfile][subtest], dict): if sysname() in mkey[suite][testfile][subtest]: - if mkey[suite][testfile][subtest][sysname()] == "pass": + if mkey[suite][testfile][subtest][ + sysname()] == "pass": test_name = f"{test_name}::{subtest}" addtest = True if addtest: test_to_add = SLASH.join(toplevel + [suite, test_name]) assert os.path.exists(test_to_add.split("::")[0]), ( - f"{test_to_add} could not be found" - ) + f"{test_to_add} could not be found") tests.append(test_to_add) addtest = False return tests @@ -196,19 +192,16 @@ def convert_manifest_to_list(manifest_loc): run_list = [] check_output(["git", "fetch", "--quiet", "--depth=1", "origin", "main"]) - committed_files = ( - check_output(["git", "--no-pager", "diff", "--name-only", "origin/main"]) - .decode() - .replace("/", SLASH) - .splitlines() - ) + committed_files = (check_output( + ["git", "--no-pager", "diff", "--name-only", + "origin/main"]).decode().replace("/", SLASH).splitlines()) main_conftest = "conftest.py" base_page = os.path.join("modules", "page_base.py") if main_conftest in committed_files or base_page in committed_files: - # Run all the tests (no files as arguments) if main conftest or basepage changed - run_list = convert_manifest_to_list("manifests/all.yaml") + # Run smoke tests if main conftest or basepage changed + run_list = convert_manifest_to_list("manifests/smoke.yaml") run_list = dedupe(run_list) with open(OUTPUT_FILE, "w") as fh: fh.write("\n".join(run_list)) @@ -219,7 +212,8 @@ def convert_manifest_to_list(manifest_loc): for root, _, files in os.walk(os.path.join(SCRIPT_DIR, "tests")): for f in files: this_file = os.path.join(root, f) - if re_obj.get("test_re").search(this_file) and "__pycache" not in this_file: + if re_obj.get("test_re").search( + this_file) and "__pycache" not in this_file: all_tests.append(os.path.join(this_file)) with open(this_file, encoding="utf-8") as fh: lines = fh.readlines() @@ -234,7 +228,9 @@ def convert_manifest_to_list(manifest_loc): changed_models = [ f for f in committed_files if re_obj.get("object_model_re").match(f) ] - changed_tests = [f for f in committed_files if re_obj.get("test_re").match(f)] + changed_tests = [ + f for f in committed_files if re_obj.get("test_re").match(f) + ] if changed_suite_conftests: run_list = [ @@ -246,9 +242,9 @@ def convert_manifest_to_list(manifest_loc): for selector_file in changed_selectors: (_, filename) = os.path.split(selector_file) model_name = pascalify(filename.split(".")[0]) - for test_name in get_tests_by_model( - model_name, test_paths_and_contents, run_list - ): + for test_name in get_tests_by_model(model_name, + test_paths_and_contents, + run_list): run_list.append(test_name) if changed_models: @@ -256,9 +252,9 @@ def convert_manifest_to_list(manifest_loc): model_file_contents = "".join([line for line in open(model_file)]) classes = re_obj.get("class_re").findall(model_file_contents) for model_name in classes: - for test_name in get_tests_by_model( - model_name, test_paths_and_contents, run_list - ): + for test_name in get_tests_by_model(model_name, + test_paths_and_contents, + run_list): run_list.append(test_name) if changed_tests: @@ -290,6 +286,8 @@ def convert_manifest_to_list(manifest_loc): if SLASH == "\\": run_list = [entry.replace("/", SLASH) for entry in run_list] run_list = dedupe(run_list) - run_list = [entry for entry in run_list if os.path.exists(entry.split("::")[0])] + run_list = [ + entry for entry in run_list if os.path.exists(entry.split("::")[0]) + ] with open(OUTPUT_FILE, "w") as fh: fh.write("\n".join(run_list)) From 20385dd737f8879018b679c16f232c2c79a0d7d8 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Thu, 20 Nov 2025 15:32:43 -0800 Subject: [PATCH 22/29] update manifest key; return win to explicit gecko --- .github/workflows/smoke.yml | 4 +-- choose_ci_set.py | 50 +++++++++++++++++++------------------ manifests/key.yaml | 35 ++++++++++++++------------ 3 files changed, 47 insertions(+), 42 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 1d343215c..b12512b80 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -112,7 +112,7 @@ jobs: $env:FX_EXECUTABLE = "C:\Program Files\Custom Firefox\firefox.exe" Start-Process -FilePath $env:FX_EXECUTABLE -ArgumentList "--version" -Wait -NoNewWindow pipenv run python choose_ci_set.py - pipenv run pytest $(cat selected_tests) + pipenv run pytest $(cat selected_tests) --geckodriver geckodriver $env:TEST_EXIT_CODE = $LASTEXITCODE mv artifacts artifacts-win || true exit $env:TEST_EXIT_CODE @@ -125,7 +125,7 @@ jobs: mv ./ci_pyproject_headed.toml ./pyproject.toml; $env:FX_EXECUTABLE = "C:\Program Files\Custom Firefox\firefox.exe" pipenv run python choose_ci_set.py - pipenv run pytest $(cat selected_tests) + pipenv run pytest $(cat selected_tests) --geckodriver geckodriver $env:TEST_EXIT_CODE = $LASTEXITCODE rm artifacts/assets -r -Force Get-ChildItem -Path "artifacts" | ForEach-Object { diff --git a/choose_ci_set.py b/choose_ci_set.py index 6f50888dd..80c3c60cd 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -47,8 +47,9 @@ def localify(path: str) -> str: return path.replace(SCRIPT_DIR, ".") -def get_tests_by_model(model_name: str, test_paths_and_contents: dict, - run_list: list) -> list: +def get_tests_by_model( + model_name: str, test_paths_and_contents: dict, run_list: list +) -> list: """ Given a model name, a dict of paths and their file contents, and the list of existing tests/dirs to check, return matches by local paths @@ -73,8 +74,11 @@ def dedupe(run_list: list) -> list: removes = [] for i, entry in enumerate(run_list): - if (not entry.startswith(".") and not entry.startswith("\\") - and not entry.startswith("/")): + if ( + not entry.startswith(".") + and not entry.startswith("\\") + and not entry.startswith("/") + ): dotslashes.append(i) for dotslash in dotslashes: @@ -138,15 +142,15 @@ def convert_manifest_to_list(manifest_loc): addtest = True elif isinstance(mkey[suite][testfile][subtest], dict): if sysname() in mkey[suite][testfile][subtest]: - if mkey[suite][testfile][subtest][ - sysname()] == "pass": + if mkey[suite][testfile][subtest][sysname()] == "pass": test_name = f"{test_name}::{subtest}" addtest = True if addtest: test_to_add = SLASH.join(toplevel + [suite, test_name]) assert os.path.exists(test_to_add.split("::")[0]), ( - f"{test_to_add} could not be found") + f"{test_to_add} could not be found" + ) tests.append(test_to_add) addtest = False return tests @@ -192,9 +196,12 @@ def convert_manifest_to_list(manifest_loc): run_list = [] check_output(["git", "fetch", "--quiet", "--depth=1", "origin", "main"]) - committed_files = (check_output( - ["git", "--no-pager", "diff", "--name-only", - "origin/main"]).decode().replace("/", SLASH).splitlines()) + committed_files = ( + check_output(["git", "--no-pager", "diff", "--name-only", "origin/main"]) + .decode() + .replace("/", SLASH) + .splitlines() + ) main_conftest = "conftest.py" base_page = os.path.join("modules", "page_base.py") @@ -212,8 +219,7 @@ def convert_manifest_to_list(manifest_loc): for root, _, files in os.walk(os.path.join(SCRIPT_DIR, "tests")): for f in files: this_file = os.path.join(root, f) - if re_obj.get("test_re").search( - this_file) and "__pycache" not in this_file: + if re_obj.get("test_re").search(this_file) and "__pycache" not in this_file: all_tests.append(os.path.join(this_file)) with open(this_file, encoding="utf-8") as fh: lines = fh.readlines() @@ -228,9 +234,7 @@ def convert_manifest_to_list(manifest_loc): changed_models = [ f for f in committed_files if re_obj.get("object_model_re").match(f) ] - changed_tests = [ - f for f in committed_files if re_obj.get("test_re").match(f) - ] + changed_tests = [f for f in committed_files if re_obj.get("test_re").match(f)] if changed_suite_conftests: run_list = [ @@ -242,9 +246,9 @@ def convert_manifest_to_list(manifest_loc): for selector_file in changed_selectors: (_, filename) = os.path.split(selector_file) model_name = pascalify(filename.split(".")[0]) - for test_name in get_tests_by_model(model_name, - test_paths_and_contents, - run_list): + for test_name in get_tests_by_model( + model_name, test_paths_and_contents, run_list + ): run_list.append(test_name) if changed_models: @@ -252,9 +256,9 @@ def convert_manifest_to_list(manifest_loc): model_file_contents = "".join([line for line in open(model_file)]) classes = re_obj.get("class_re").findall(model_file_contents) for model_name in classes: - for test_name in get_tests_by_model(model_name, - test_paths_and_contents, - run_list): + for test_name in get_tests_by_model( + model_name, test_paths_and_contents, run_list + ): run_list.append(test_name) if changed_tests: @@ -286,8 +290,6 @@ def convert_manifest_to_list(manifest_loc): if SLASH == "\\": run_list = [entry.replace("/", SLASH) for entry in run_list] run_list = dedupe(run_list) - run_list = [ - entry for entry in run_list if os.path.exists(entry.split("::")[0]) - ] + run_list = [entry for entry in run_list if os.path.exists(entry.split("::")[0])] with open(OUTPUT_FILE, "w") as fh: fh.write("\n".join(run_list)) diff --git a/manifests/key.yaml b/manifests/key.yaml index f1911b40c..4f352633c 100644 --- a/manifests/key.yaml +++ b/manifests/key.yaml @@ -16,10 +16,7 @@ address_bar_and_search: linux: pass mac: unstable win: pass - test_google_withads_url_bar_us: - linux: unstable - mac: unstable - win: pass + test_google_withads_url_bar_us: unstable test_insertion_point_no_search_terms_display: pass test_no_suggestions_for_empty_query: pass test_non_sponsored_topsite_context_menu_option: pass @@ -34,7 +31,10 @@ address_bar_and_search: test_search_modes_for_sites: unstable test_search_string_displayed_when_addressbar_unfocused: pass test_search_suggestions: unstable - test_search_term_persists: pass + test_search_term_persists: + mac: pass + win: pass + linux: unstable test_searchbar_display_alpenglow_theme: pass test_searchbar_on_engines_remove_restore: pass test_searchbar_results_shown_in_a_new_tab: pass @@ -79,14 +79,20 @@ bookmarks_and_history: test_toggle_bookmarks_toolbar: pass test_user_can_forget_history: pass downloads: - test_add_mime_type_doc: pass + test_add_mime_type_doc: + linux: pass + mac: pass + win: unstable test_add_zip_type: linux: unstable mac: pass win: pass test_download_pdf: pass test_download_pdf_from_context_menu: unstable - test_mixed_content_download_via_https: pass + test_mixed_content_download_via_https: + win: unstable + mac: unstable + linux: pass test_set_always_ask_file_type: pass drag_and_drop: test_copy_entire_row_column: pass @@ -274,14 +280,8 @@ security_and_privacy: test_tracking_content_custom_mode: pass test_undo_close_tab_private_browsing: pass sync_and_fxa: - test_existing_fxa: - linux: unstable - mac: unstable - win: pass - test_new_fxa: - linux: unstable - mac: unstable - win: pass + test_existing_fxa: unstable + test_new_fxa: unstable tabs: test_active_tab: pass test_change_position_of_pinned_tabs: pass @@ -289,7 +289,10 @@ tabs: test_close_tab_through_middle_mouse_click: pass test_display_customize_button: pass test_list_all_tabs: pass - test_mute_tabs: pass + test_mute_tabs: + win: unstable + mac: pass + linux: pass test_navigation_multiple_tabs: pass test_open_bookmark_in_new_tab: pass test_open_new_bg_tab_via_mouse_and_keyboard: pass From 5057c72a9a5d5833b38f4736cbeb84330f835cbb Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Thu, 20 Nov 2025 15:54:32 -0800 Subject: [PATCH 23/29] update manifest key; return win to explicit gecko --- .github/workflows/smoke.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index b12512b80..0572e2f1f 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -112,7 +112,7 @@ jobs: $env:FX_EXECUTABLE = "C:\Program Files\Custom Firefox\firefox.exe" Start-Process -FilePath $env:FX_EXECUTABLE -ArgumentList "--version" -Wait -NoNewWindow pipenv run python choose_ci_set.py - pipenv run pytest $(cat selected_tests) --geckodriver geckodriver + pipenv run pytest $(cat selected_tests) --geckodriver ./geckodriver $env:TEST_EXIT_CODE = $LASTEXITCODE mv artifacts artifacts-win || true exit $env:TEST_EXIT_CODE @@ -125,7 +125,7 @@ jobs: mv ./ci_pyproject_headed.toml ./pyproject.toml; $env:FX_EXECUTABLE = "C:\Program Files\Custom Firefox\firefox.exe" pipenv run python choose_ci_set.py - pipenv run pytest $(cat selected_tests) --geckodriver geckodriver + pipenv run pytest $(cat selected_tests) --geckodriver ./geckodriver $env:TEST_EXIT_CODE = $LASTEXITCODE rm artifacts/assets -r -Force Get-ChildItem -Path "artifacts" | ForEach-Object { From ecb7e3851f95306e4f52d36eefdc1650c3f3ad66 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Thu, 20 Nov 2025 16:00:14 -0800 Subject: [PATCH 24/29] update manifest key; return win to explicit gecko --- .github/workflows/smoke.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 0572e2f1f..a4819b53b 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -112,7 +112,7 @@ jobs: $env:FX_EXECUTABLE = "C:\Program Files\Custom Firefox\firefox.exe" Start-Process -FilePath $env:FX_EXECUTABLE -ArgumentList "--version" -Wait -NoNewWindow pipenv run python choose_ci_set.py - pipenv run pytest $(cat selected_tests) --geckodriver ./geckodriver + pipenv run pytest $(cat selected_tests) --geckodriver=geckodriver.exe $env:TEST_EXIT_CODE = $LASTEXITCODE mv artifacts artifacts-win || true exit $env:TEST_EXIT_CODE @@ -125,7 +125,7 @@ jobs: mv ./ci_pyproject_headed.toml ./pyproject.toml; $env:FX_EXECUTABLE = "C:\Program Files\Custom Firefox\firefox.exe" pipenv run python choose_ci_set.py - pipenv run pytest $(cat selected_tests) --geckodriver ./geckodriver + pipenv run pytest $(cat selected_tests) --geckodriver=geckodriver.exe $env:TEST_EXIT_CODE = $LASTEXITCODE rm artifacts/assets -r -Force Get-ChildItem -Path "artifacts" | ForEach-Object { From 08a1a0778e72292c8099cdcfb170455a72b56327 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Thu, 20 Nov 2025 18:49:40 -0800 Subject: [PATCH 25/29] update manifest key; return win to explicit gecko --- manifests/key.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/key.yaml b/manifests/key.yaml index 4f352633c..67dc48a21 100644 --- a/manifests/key.yaml +++ b/manifests/key.yaml @@ -33,7 +33,7 @@ address_bar_and_search: test_search_suggestions: unstable test_search_term_persists: mac: pass - win: pass + win: unstable linux: unstable test_searchbar_display_alpenglow_theme: pass test_searchbar_on_engines_remove_restore: pass From 871e766c58cc99c9cfdd422dae690a6f627baa0e Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 21 Nov 2025 07:44:16 -0800 Subject: [PATCH 26/29] update manifest key; return win to explicit gecko --- manifests/key.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/manifests/key.yaml b/manifests/key.yaml index 67dc48a21..1f4f3eb3c 100644 --- a/manifests/key.yaml +++ b/manifests/key.yaml @@ -31,10 +31,7 @@ address_bar_and_search: test_search_modes_for_sites: unstable test_search_string_displayed_when_addressbar_unfocused: pass test_search_suggestions: unstable - test_search_term_persists: - mac: pass - win: unstable - linux: unstable + test_search_term_persists: unstable test_searchbar_display_alpenglow_theme: pass test_searchbar_on_engines_remove_restore: pass test_searchbar_results_shown_in_a_new_tab: pass From 0becacd1d6984a67b8af270bf8135d9746d51436 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Fri, 21 Nov 2025 12:18:59 -0800 Subject: [PATCH 27/29] README --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index fb6478b95..3eae60623 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,47 @@ You may find that if you are re-running all previously executed test runs that y "Session is not reportable." If you wish to overwrite a previously reported session, add `REPORTABLE=true` to your environment. +### Marking Tests For Skipping + +The file `manifests/key.yaml` is the single source of truth for whether a test exists, and whether it +should or should not be skipped. The other files in `manifests/` are test lists. The schema for the key +file is: + +```yaml +suite_name_which_is_the_folder_under_tests: + test_file_without_the_dot_py: pass +address_bar_and_search: + test_thing_does_stuff: + test_a_subtest_inside_this_file: pass + test_another_thing: + mac: pass + win: unstable + linux: pass +tabs: + test_tab_says_hi: + test_clever_subtest_name: + mac: unstable + win: pass + linux: pass +``` + +Any value other than `pass` will skip the test or subtest (for the given OS if applicable). It is good +practice to keep non-pass values limited. Good values are `unstable`, `deprecated`, `out-of-scope` etc. +**Do not use `fail` for tests you wish to see pass again one day.** Future work will include testing +items marked `fail` as xfail, and may implement `strict_xfail`, which will throw if tests pass. + +The test lists in `manifests/` have the following schema: + +```yaml +suite_name_a: +- test_name_b +- test_name_c +suite_name_d: +- test_name_e +``` + +We currently do not assume ma + ### Manual Execution of Smoke Tests To run the smoke tests manually against an arbitrary version of Firefox **where the installer or (for Linux) From 69b444b73f05a33900248beb4e11d7460c0c6932 Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Mon, 24 Nov 2025 07:30:19 -0800 Subject: [PATCH 28/29] README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3eae60623..4177da159 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,8 @@ suite_name_d: - test_name_e ``` -We currently do not assume ma +We currently do not assume that test lists need to identify individual test functions inside test +files ("subtests"), as TestRail reporting is on the testfile level. ### Manual Execution of Smoke Tests From 2cfd72dc23b44514dd8eee1cd093980f54f33c4b Mon Sep 17 00:00:00 2001 From: Ben Chatterton Date: Mon, 24 Nov 2025 08:43:28 -0800 Subject: [PATCH 29/29] mark test unstable --- manifests/key.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/manifests/key.yaml b/manifests/key.yaml index 1f4f3eb3c..ca0a75fd6 100644 --- a/manifests/key.yaml +++ b/manifests/key.yaml @@ -113,7 +113,10 @@ form_autofill: test_create_new_cc_profile: pass test_create_profile_autofill: pass test_delete_cc_profile: pass - test_edit_credit_card: pass + test_edit_credit_card: + win: unstable + mac: pass + linux: pass test_enable_disable_autofill: pass test_form_autofill_suggestions: pass test_name_autofill_attribute: pass