diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index ec0da223b..a4819b53b 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -93,6 +93,10 @@ jobs: - name: Download Executable if: ${{ inputs.win_installer_link }} run: | + $env_contents = @" + STARFOX_MANIFEST='.\manifests\incident.yaml' + "@ + New-Item -Name .env -Value $env_contents -ItemType File -Force Invoke-WebRequest -Uri ${{ inputs.win_installer_link }} -OutFile "${{ github.workspace }}\setup.exe" New-Item -ItemType Directory -Path "C:\Program Files\Custom Firefox" -Force shell: pwsh @@ -108,7 +112,7 @@ jobs: $env:FX_EXECUTABLE = "C:\Program Files\Custom Firefox\firefox.exe" Start-Process -FilePath $env:FX_EXECUTABLE -ArgumentList "--version" -Wait -NoNewWindow pipenv run python choose_ci_set.py - pipenv run pytest $(cat selected_tests) + pipenv run pytest $(cat selected_tests) --geckodriver=geckodriver.exe $env:TEST_EXIT_CODE = $LASTEXITCODE mv artifacts artifacts-win || true exit $env:TEST_EXIT_CODE @@ -121,7 +125,7 @@ jobs: mv ./ci_pyproject_headed.toml ./pyproject.toml; $env:FX_EXECUTABLE = "C:\Program Files\Custom Firefox\firefox.exe" pipenv run python choose_ci_set.py - pipenv run pytest $(cat selected_tests) + pipenv run pytest $(cat selected_tests) --geckodriver=geckodriver.exe $env:TEST_EXIT_CODE = $LASTEXITCODE rm artifacts/assets -r -Force Get-ChildItem -Path "artifacts" | ForEach-Object { @@ -165,6 +169,7 @@ jobs: if: ${{ inputs.mac_installer_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; + echo "STARFOX_MANIFEST=manifests/incident.yaml" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; - name: Install dependencies run: | @@ -228,6 +233,7 @@ jobs: MANUAL_DOWNLOAD_LINK: ${{ inputs.linux_tarball_link }} run: | echo "MANUAL='true'" >> "$GITHUB_ENV"; + echo "STARFOX_MANIFEST=manifests/incident.yaml" >> "$GITHUB_ENV" echo "Running smoke tests on supplied executable"; sudo apt install gnome-screenshot uname -m; diff --git a/README.md b/README.md index fb6478b95..4177da159 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,48 @@ You may find that if you are re-running all previously executed test runs that y "Session is not reportable." If you wish to overwrite a previously reported session, add `REPORTABLE=true` to your environment. +### Marking Tests For Skipping + +The file `manifests/key.yaml` is the single source of truth for whether a test exists, and whether it +should or should not be skipped. The other files in `manifests/` are test lists. The schema for the key +file is: + +```yaml +suite_name_which_is_the_folder_under_tests: + test_file_without_the_dot_py: pass +address_bar_and_search: + test_thing_does_stuff: + test_a_subtest_inside_this_file: pass + test_another_thing: + mac: pass + win: unstable + linux: pass +tabs: + test_tab_says_hi: + test_clever_subtest_name: + mac: unstable + win: pass + linux: pass +``` + +Any value other than `pass` will skip the test or subtest (for the given OS if applicable). It is good +practice to keep non-pass values limited. Good values are `unstable`, `deprecated`, `out-of-scope` etc. +**Do not use `fail` for tests you wish to see pass again one day.** Future work will include testing +items marked `fail` as xfail, and may implement `strict_xfail`, which will throw if tests pass. + +The test lists in `manifests/` have the following schema: + +```yaml +suite_name_a: +- test_name_b +- test_name_c +suite_name_d: +- test_name_e +``` + +We currently do not assume that test lists need to identify individual test functions inside test +files ("subtests"), as TestRail reporting is on the testfile level. + ### Manual Execution of Smoke Tests To run the smoke tests manually against an arbitrary version of Firefox **where the installer or (for Linux) diff --git a/choose_ci_set.py b/choose_ci_set.py index 01e67cf36..80c3c60cd 100644 --- a/choose_ci_set.py +++ b/choose_ci_set.py @@ -1,24 +1,19 @@ import os +import platform import re import sys from subprocess import check_output -import pytest +import yaml -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -CI_MARK = "@pytest.mark.ci" +CI_MANIFEST = "manifests/ci.yaml" +MANIFEST_KEY = "manifests/key.yaml" +SUPPORTED_OSES = ["mac", "win", "linux"] HEADED_MARK = "@pytest.mark.headed" +MIN_RUN_SIZE = 7 OUTPUT_FILE = "selected_tests" - - -class CollectionPlugin: - """Mini plugin to get test names""" - - def __init__(self): - self.tests = [] - - def pytest_report_collectionfinish(self, items): - self.tests = [item.nodeid for item in items] +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +SLASH = "/" if "/" in SCRIPT_DIR else "\\" def snakify(pascal: str) -> str: @@ -72,7 +67,7 @@ def get_tests_by_model( return matching_tests -def dedupe(run_list: list, slash: str) -> list: +def dedupe(run_list: list) -> list: """For a run list, remove entries that are covered by more general entries.""" run_list = list(set(run_list)) dotslashes = [] @@ -87,7 +82,7 @@ def dedupe(run_list: list, slash: str) -> list: dotslashes.append(i) for dotslash in dotslashes: - run_list[dotslash] = f".{slash}{run_list[dotslash]}" + run_list[dotslash] = f".{SLASH}{run_list[dotslash]}" for i, entry_a in enumerate(run_list): for j, entry_b in enumerate(run_list): @@ -105,7 +100,64 @@ def dedupe(run_list: list, slash: str) -> list: return run_list +def sysname(): + sys_platform = platform.system().lower() + if sys_platform.startswith("darwin"): + return "mac" + elif sys_platform.startswith("win"): + return "win" + elif sys_platform.startswith("linux"): + return "linux" + raise OSError("Unsupported system.") + + +def convert_manifest_to_list(manifest_loc): + manifest = yaml.safe_load(open(manifest_loc)) + mkey = yaml.safe_load(open(MANIFEST_KEY)) + toplevel = [".", "tests"] + tests = [] + if manifest: + print(f"Reading {manifest_loc}") + for suite in manifest: + if suite not in mkey: + print(f"{suite} not in {MANIFEST_KEY}") + continue + + for testfile in manifest[suite]: + addtest = False + test_name = f"{testfile}.py" + if testfile not in mkey[suite]: + print(f"{suite}/{testfile} not in {MANIFEST_KEY}::{suite}") + continue + if mkey[suite][testfile] == "pass": + addtest = True + elif isinstance(mkey[suite][testfile], dict): + if any([x in mkey[suite][testfile] for x in SUPPORTED_OSES]): + if mkey[suite][testfile][sysname()] == "pass": + addtest = True + else: + for subtest in mkey[suite][testfile]: + if mkey[suite][testfile][subtest] == "pass": + test_name = f"{test_name}::{subtest}" + addtest = True + elif isinstance(mkey[suite][testfile][subtest], dict): + if sysname() in mkey[suite][testfile][subtest]: + if mkey[suite][testfile][subtest][sysname()] == "pass": + test_name = f"{test_name}::{subtest}" + addtest = True + + if addtest: + test_to_add = SLASH.join(toplevel + [suite, test_name]) + assert os.path.exists(test_to_add.split("::")[0]), ( + f"{test_to_add} could not be found" + ) + tests.append(test_to_add) + addtest = False + return tests + + if __name__ == "__main__": + print("Selecting test set...") if os.path.exists(".env"): with open(".env") as fh: contents = fh.read() @@ -114,13 +166,20 @@ def dedupe(run_list: list, slash: str) -> list: if "RUN_ALL='true'" in contents: os.environ["MANUAL"] = "true" - if os.environ.get("TESTRAIL_REPORT") or os.environ.get("MANUAL"): - # Run all tests if this is a scheduled beta or a manual run + if os.environ.get("TESTRAIL_REPORT"): + # Run all tests if this is a scheduled run + run_list = convert_manifest_to_list("manifests/smoke.yaml") + run_list = dedupe(run_list) with open(OUTPUT_FILE, "w") as fh: - fh.write("tests") + fh.write("\n".join(run_list)) sys.exit(0) - slash = "/" if "/" in SCRIPT_DIR else "\\" + if os.environ.get("STARFOX_MANIFEST"): + run_list = convert_manifest_to_list(os.environ["STARFOX_MANIFEST"]) + run_list = dedupe(run_list) + with open(OUTPUT_FILE, "w") as fh: + fh.write("\n".join(run_list)) + sys.exit(0) re_obj = { "test_re_string": r".*/.*/test_.*\.py", @@ -130,7 +189,7 @@ def dedupe(run_list: list, slash: str) -> list: "class_re_string": r"\s*class (\w+)[(A-Za-z0-9_)]*:", } for k in list(re_obj.keys()): - if slash == "\\": + if SLASH == "\\": re_obj[k] = re_obj.get(k).replace("/", r"\\") short_name = "_".join(k.split("_")[:-1]) re_obj[short_name] = re.compile(re_obj.get(k)) @@ -140,7 +199,7 @@ def dedupe(run_list: list, slash: str) -> list: committed_files = ( check_output(["git", "--no-pager", "diff", "--name-only", "origin/main"]) .decode() - .replace("/", slash) + .replace("/", SLASH) .splitlines() ) @@ -148,9 +207,11 @@ def dedupe(run_list: list, slash: str) -> list: base_page = os.path.join("modules", "page_base.py") if main_conftest in committed_files or base_page in committed_files: - # Run all the tests (no files as arguments) if main conftest or basepage changed + # Run smoke tests if main conftest or basepage changed + run_list = convert_manifest_to_list("manifests/smoke.yaml") + run_list = dedupe(run_list) with open(OUTPUT_FILE, "w") as fh: - fh.write("tests") + fh.write("\n".join(run_list)) sys.exit(0) all_tests = [] @@ -164,13 +225,6 @@ def dedupe(run_list: list, slash: str) -> list: lines = fh.readlines() test_paths_and_contents[this_file] = "".join(lines) - p = CollectionPlugin() - pytest.main(["--collect-only", "-m", "ci", "-s"], plugins=[p]) - ci_paths = [f".{slash}{test}" for test in p.tests] - - # Dedupe just in case - ci_paths = list(set(ci_paths)) - changed_suite_conftests = [ f for f in committed_files if re_obj.get("suite_conftest_re").match(f) ] @@ -184,7 +238,7 @@ def dedupe(run_list: list, slash: str) -> list: if changed_suite_conftests: run_list = [ - "." + slash + os.path.join(*suite.split(slash)[-3:-1]) + "." + SLASH + os.path.join(*suite.split(SLASH)[-3:-1]) for suite in changed_suite_conftests ] @@ -212,7 +266,7 @@ def dedupe(run_list: list, slash: str) -> list: found = False for file in run_list: # Don't add if already exists in suite changes - pieces = file.split(slash) + pieces = file.split(SLASH) if len(pieces) == 3 and pieces[-1] in changed_test: found = True @@ -223,15 +277,19 @@ def dedupe(run_list: list, slash: str) -> list: run_list.append(changed_test) if not run_list: + ci_paths = convert_manifest_to_list(CI_MANIFEST) + ci_paths = dedupe(ci_paths) with open(OUTPUT_FILE, "w") as fh: fh.write("\n".join(ci_paths)) - else: + sys.exit(0) + + if len(run_list) < MIN_RUN_SIZE: run_list.extend(ci_paths) - # Dedupe just in case - if slash == "\\": - run_list = [entry.replace("/", slash) for entry in run_list] - run_list = dedupe(run_list, slash) - run_list = [entry for entry in run_list if os.path.exists(entry.split("::")[0])] - with open(OUTPUT_FILE, "w") as fh: - fh.write("\n".join(run_list)) + # Dedupe just in case + if SLASH == "\\": + run_list = [entry.replace("/", SLASH) for entry in run_list] + run_list = dedupe(run_list) + run_list = [entry for entry in run_list if os.path.exists(entry.split("::")[0])] + with open(OUTPUT_FILE, "w") as fh: + fh.write("\n".join(run_list)) diff --git a/choose_test_channel.py b/choose_test_channel.py index 06198725d..705a5859f 100644 --- a/choose_test_channel.py +++ b/choose_test_channel.py @@ -3,7 +3,7 @@ import sys from subprocess import check_output -ALL_CHANNELS = ["smoke", "l10n"] +ALL_CHANNELS = ["smoke", "l10n", "functional"] SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) SLASH = "/" if "/" in SCRIPT_DIR else "\\" @@ -29,6 +29,10 @@ ] ) +if len(sys.argv) > 1: + print(sys.argv[1:]) + sys.exit(0) + check_output(["git", "fetch", "--quiet", "--depth=1", "origin", "main"]) committed_files = ( diff --git a/ci_pyproject.toml b/ci_pyproject.toml index 8548738f5..c178ae8e1 100644 --- a/ci_pyproject.toml +++ b/ci_pyproject.toml @@ -5,17 +5,14 @@ log_cli_level = "warn" markers = [ "audio: test is reliant on audio", "headed: test must run in headed mode (e.g. uses pynput)", - "incident: incident smoke tests", - "unstable: temporary mark for unstable tests", "slow: test is clocked at more than 30s on modern machines", - "ci: basic tests to run in ci", "locale_de: tests run in DE locale versions", "locale_fr: tests run in FR locale versions", "locale_gb: tests run in GB locale versions", "noxvfb: tests that should not run in xvfb sessions" ] -addopts = "-vs --ci --run-headless --json-report --json-report-file artifacts/report.json --reruns 3 -n auto --reruns-delay 3 -m 'not incident and not unstable and not headed' --html=artifacts/report.html" +addopts = "-vs --ci --run-headless --json-report --json-report-file artifacts/report.json --reruns 3 -n auto --reruns-delay 3 -m 'not headed' --html=artifacts/report.html" [tool.ruff] target-version = "py310" diff --git a/ci_pyproject_headed.toml b/ci_pyproject_headed.toml index 5916a6a44..281f8ceb2 100644 --- a/ci_pyproject_headed.toml +++ b/ci_pyproject_headed.toml @@ -5,16 +5,13 @@ log_cli_level = "warn" markers = [ "audio: test is reliant on audio", "headed: test must run in headed mode (e.g. uses pynput)", - "incident: incident smoke tests", - "unstable: temporary mark for unstable tests", "slow: test is clocked at more than 30s on modern machines", - "ci: basic tests to run in ci", "locale_de: tests run in DE locale versions", "locale_fr: tests run in FR locale versions", "locale_gb: tests run in GB locale versions", "noxvfb: tests that should not run in xvfb sessions" ] -addopts = "-vs --ci --json-report --json-report-file artifacts/report_headed.json --reruns 3 --reruns-delay 2 -m 'not incident and not unstable and headed' --html=artifacts/report_headed.html" +addopts = "-vs --ci --json-report --json-report-file artifacts/report_headed.json --reruns 3 --reruns-delay 2 -m 'headed' --html=artifacts/report_headed.html" [tool.ruff] target-version = "py310" diff --git a/ci_xvfb_pyproject.toml b/ci_xvfb_pyproject.toml index 11f51f8f1..1f8dd217c 100644 --- a/ci_xvfb_pyproject.toml +++ b/ci_xvfb_pyproject.toml @@ -5,16 +5,13 @@ log_cli_level = "warn" markers = [ "audio: test is reliant on audio", "headed: test must run in headed mode (e.g. uses pynput)", - "incident: incident smoke tests", - "unstable: temporary mark for unstable tests", "slow: test is clocked at more than 30s on modern machines", - "ci: basic tests to run in ci", "locale_de: tests run in DE locale versions", "locale_fr: tests run in FR locale versions", "locale_gb: tests run in GB locale versions", "noxvfb: tests that should not run in xvfb sessions" ] -addopts = "-vs --ci --run-headless --json-report --json-report-file artifacts/report.json -n auto --reruns 3 --reruns-delay 3 -m 'not unstable and not headed and not noxvfb' --html=artifacts/report.html" +addopts = "-vs --ci --run-headless --json-report --json-report-file artifacts/report.json -n auto --reruns 3 --reruns-delay 3 -m 'not headed and not noxvfb' --html=artifacts/report.html" [tool.ruff] target-version = "py310" diff --git a/ci_xvfb_pyproject_headed.toml b/ci_xvfb_pyproject_headed.toml index 7b5483c05..660e6f1a9 100644 --- a/ci_xvfb_pyproject_headed.toml +++ b/ci_xvfb_pyproject_headed.toml @@ -5,16 +5,13 @@ log_cli_level = "warn" markers = [ "audio: test is reliant on audio", "headed: test must run in headed mode (e.g. uses pynput)", - "incident: incident smoke tests", - "unstable: temporary mark for unstable tests", "slow: test is clocked at more than 30s on modern machines", - "ci: basic tests to run in ci", "locale_de: tests run in DE locale versions", "locale_fr: tests run in FR locale versions", "locale_gb: tests run in GB locale versions", "noxvfb: tests that should not run in xvfb sessions" ] -addopts = "-vs --ci --json-report --json-report-file artifacts/report_headed.json -n auto --reruns 3 --reruns-delay 2 -m 'not unstable and not noxvfb and headed' --html=artifacts/report_headed.html" +addopts = "-vs --ci --json-report --json-report-file artifacts/report_headed.json -n auto --reruns 3 --reruns-delay 2 -m 'not noxvfb and headed' --html=artifacts/report_headed.html" [tool.ruff] target-version = "py310" diff --git a/conftest.py b/conftest.py index 18b45e369..48c02472f 100644 --- a/conftest.py +++ b/conftest.py @@ -200,6 +200,14 @@ def _screenshot_whole_screen(filename: str, driver: Firefox, opt_ci: bool): return fullpath +def _get_version(driver: Firefox): + driver.get("chrome://browser/content/aboutDialog.xhtml") + version_el = driver.find_element(By.ID, "version") + version = version_el.text + driver.get("about:blank") + return version + + @pytest.fixture() def opt_headless(request): return request.config.getoption("--run-headless") @@ -496,7 +504,7 @@ def driver( WebDriverWait(driver, timeout=40).until( EC.presence_of_element_located((By.TAG_NAME, "body")) ) - json_metadata["fx_version"] = version + json_metadata["fx_version"] = _get_version(driver) json_metadata["machine_config"] = machine_config json_metadata["suite_id"] = suite_id json_metadata["test_case"] = test_case @@ -506,7 +514,7 @@ def driver( finally: if hard_quit: return - if "driver" in locals() or "driver" in globals(): + if ("driver" in locals() or "driver" in globals()) and driver: driver.quit() diff --git a/make_manifest.py b/make_manifest.py new file mode 100644 index 000000000..2495c8f21 --- /dev/null +++ b/make_manifest.py @@ -0,0 +1,36 @@ +import os +import re + +import yaml + +TESTFILE_RE = re.compile(r"test_.*\.py") +TEST_RE = re.compile(r"def (test.*)\(") + +if __name__ == "__main__": + testdict = {} + for root, _, files in os.walk("tests"): + for f in files: + if not TESTFILE_RE.match(f): + continue + pointer = testdict + for level in os.path.split(root)[1:]: + if level not in testdict: + pointer[level] = {} + pointer = pointer[level] + status = "pass" + filename = os.path.join(root, f) + with open(filename) as fh: + metatest = f.rsplit(".", 1)[0] + for line in fh.readlines(): + m = TEST_RE.search(line) + if m and m[1] != "test_case": + if metatest not in pointer: + pointer[metatest] = {m[1]: status} + else: + pointer[metatest][m[1]] = status + print(yaml.safe_dump(testdict), "\n\n\n\n===\n\n\n\n") + status = "pass" + if "mark.unstable" in line: + status = "unstable" + with open("new_manifest.yaml", "w") as fh: + yaml.safe_dump(testdict, fh) diff --git a/manifests/ci.yaml b/manifests/ci.yaml new file mode 100644 index 000000000..9594d6eea --- /dev/null +++ b/manifests/ci.yaml @@ -0,0 +1,33 @@ +address_bar_and_search: + - test_default_search_provider_change_awesome_bar +bookmarks_and_history: + - test_bookmark_website_via_star_button +downloads: + - test_set_always_ask_file_type +find_toolbar: + - test_find_toolbar_search +form_autofill: + - test_telephone_autofill_attribute +menus: + - test_copy_paste_actions + - test_image_context_menu_actions +networking: + - test_http_site +notifications: + - test_notifications_displayed +password_manager: + - test_about_logins_navigation_from_context_menu +pdf_viewer: + - test_pdf_data_can_be_cleared +printing_ui: + - test_print_preview +reader_view: + - test_improved_type_control_panel +scrolling_panning_zooming: + - test_zoom_text_only +security_and_privacy: + - test_cross_site_tracking_cookies_blocked +tabs: + - test_active_tab +theme_and_toolbar: + - test_customize_themes_and_redirect diff --git a/manifests/incident.yaml b/manifests/incident.yaml new file mode 100644 index 000000000..774f2c3d4 --- /dev/null +++ b/manifests/incident.yaml @@ -0,0 +1,159 @@ +address_bar_and_search: +- test_add_engine_address_bar +- test_addon_suggestion +- test_default_search_provider_change_awesome_bar +- test_google_search_counts_us +- test_google_withads_url_bar_us +- test_sap_google_adclick +- test_search_code_google_non_us +- test_search_code_google_us +- test_search_engine_selector +- test_search_suggestions +- test_search_term_persists +- test_server_not_found_error +- test_suggestion_engine_selection +audio_video: +- test_allow_audio_video_functionality +- test_block_audio_video_functionality +- test_users_actions_saved_on_reload +bookmarks_and_history: +- test_add_new_other_bookmark +- test_bookmark_via_bookmark_menu +- test_bookmark_website_via_star_button +- test_clear_all_history +- test_clear_recent_history_displayed +- test_delete_bookmarks_from_toolbar +- test_delete_other_bookmarks +- test_deleted_page_not_remembered +- test_open_bookmark_in_new_window_via_toolbar_context_menu +- test_open_bookmark_in_private_window_via_toolbar_context_menu +- test_open_bookmarks_from_toolbar +- test_open_websites_from_history +- test_private_window_website_not_in_history +- test_toggle_bookmarks_toolbar +- test_user_can_forget_history +downloads: +- test_add_zip_type +- test_download_pdf +- test_download_pdf_from_context_menu +drag_and_drop: +- test_copy_entire_row_column +find_toolbar: +- test_find_in_pdf +- test_find_toolbar_nav +- test_find_toolbar_search +form_autofill: +- test_address_autofill_attribute +- test_autofill_cc_cvv +- test_autofill_credit_card +- test_autofill_credit_card_doorhanger +- test_autofill_credit_card_enable +- test_autofill_credit_card_four_fields +- test_cc_clear_form +- test_clear_form +- test_create_new_cc_profile +- test_create_profile_autofill +- test_delete_cc_profile +- test_enable_disable_autofill +- test_form_autofill_suggestions +- test_name_autofill_attribute +- test_private_mode_info_not_saved +- test_telephone_autofill_attribute +- test_updating_address +geolocation: +- test_geolocation_shared_via_html5 +- test_geolocation_shared_via_w3c_api +language_packs: +- test_language_pack_install_addons +- test_language_pack_install_preferences +menus: +- test_frequently_used_context_menu +- test_hyperlink_context_menu +- test_tab_context_menu_actions +- test_tab_context_menu_close +networking: +- test_default_dns_protection +- test_http_site +notifications: +- test_audio_video_permissions_notification +- test_camera_permissions_notification +- test_deny_geolocation +- test_deny_screen_capture +- test_geolocation_prompt_presence +- test_microphone_permissions_notification +- test_notifications_displayed +- test_screen_share_permission_prompt +- test_webextension_completed_installation_successfully_displayed +password_manager: +- test_add_password_non_ascii_chars +- test_add_password_save_valid_data +- test_auto_saved_generated_password_context_menu +- test_can_view_password_when_PP_enabled +- test_changes_made_in_edit_mode_are_saved +- test_delete_login +- test_multiple_saved_logins +- test_never_save_login_via_doorhanger +- test_primary_password_triggered_on_about_logins_access +- test_save_login_via_doorhanger +- test_update_login_via_doorhanger +pdf_viewer: +- test_download_pdf_with_form_fields +- test_download_triggered_on_content_disposition_attachment +- test_open_pdf_in_FF +- test_pdf_navigation +- test_zoom_pdf_viewer +preferences: +- test_check_for_updates +- test_clear_cookie_data +- test_firefox_home_new_tabs +- test_firefox_home_on_launch +- test_lang_pack_changed_from_about_prefs +- test_never_remember_history +- test_notifications_change_set +printing_ui: +- test_print_preview +profile: +- test_set_default_profile +reader_view: +- test_improved_type_control_panel +- test_reader_view_location_bar +scrolling_panning_zooming: +- test_default_zoom_persists +- test_mouse_wheel_zoom +- test_zoom_text_only +security_and_privacy: +- test_blocking_cryptominers +- test_blocking_fingerprinters +- test_blocking_social_media_trackers +- test_cookies_not_saved_private_browsing +- test_cross_site_tracking_cookies_blocked +- test_cryptominers_blocked_and_shown_in_info_panel +- test_detected_blocked_trackers_found +- test_downloads_from_private_not_leaked +- test_https_enabled_private_browsing +- test_never_remember_browsing_history +- test_no_trackers_detected +- test_open_link_in_private_window +- test_phishing_and_malware_warnings +- test_private_browser_password_doorhanger +- test_private_session_awesome_bar_exclusion +- test_private_session_history_exclusion +- test_private_window_from_panelui +- test_third_party_content_blocked_private_browsing +- test_tls_v1_2_protocol +- test_trackers_crypto_fingerprint_blocked +- test_tracking_content_custom_mode +- test_undo_close_tab_private_browsing +tabs: +- test_active_tab +- test_navigation_multiple_tabs +- test_open_bookmark_in_new_tab +- test_open_new_tab +- test_open_new_tab_keys +- test_open_new_tab_via_hyperlink +- test_pin_tab +- test_reopen_tab_through_context_menu +- test_reopen_tab_through_history_menu +- test_reopen_tabs_through_keys +theme_and_toolbar: +- test_customize_themes_and_redirect diff --git a/manifests/key.yaml b/manifests/key.yaml new file mode 100644 index 000000000..ca0a75fd6 --- /dev/null +++ b/manifests/key.yaml @@ -0,0 +1,312 @@ +address_bar_and_search: + test_adaptive_history_autofill: + linux: unstable + mac: unstable + win: pass + test_add_engine_address_bar: pass + test_added_open_search_engine_default: pass + test_addon_suggestion: unstable + test_addressbar_search_engine_keywords: pass + test_copied_url_contains_https: pass + test_ctrl_enter_fixes_url: pass + test_default_search_provider_change_awesome_bar: unstable + test_default_search_provider_change_legacy_search_bar: deprecated + test_dont_show_search_suggestions_in_private_window: pass + test_google_search_counts_us: + linux: pass + mac: unstable + win: pass + test_google_withads_url_bar_us: unstable + test_insertion_point_no_search_terms_display: pass + test_no_suggestions_for_empty_query: pass + test_non_sponsored_topsite_context_menu_option: pass + test_refresh_firefox_dialog: pass + test_sap_google_adclick: unstable + test_seach_suggestions_can_be_disabled: pass + test_search_code_google_non_us: pass + test_search_code_google_us: pass + test_search_engine_selector: pass + test_search_mode_appears_after_input: pass + test_search_mode_persists_mixed_with_bing: pass + test_search_modes_for_sites: unstable + test_search_string_displayed_when_addressbar_unfocused: pass + test_search_suggestions: unstable + test_search_term_persists: unstable + test_searchbar_display_alpenglow_theme: pass + test_searchbar_on_engines_remove_restore: pass + test_searchbar_results_shown_in_a_new_tab: pass + test_searchmode_change_tab: pass + test_server_not_found_error: pass + test_server_not_found_error_pb: pass + test_suggestion_engine_selection: pass + test_tile_menu_options: pass +audio_video: + test_allow_audio_video_functionality: + linux: unstable + mac: pass + win: unstable + test_block_audio_video_functionality: pass + test_users_actions_saved_on_reload: pass +bookmarks_and_history: + test_add_new_other_bookmark: pass + test_bookmark_via_bookmark_menu: pass + test_bookmark_website_via_star_button: pass + test_clear_all_history: pass + test_clear_recent_history_displayed: pass + test_delete_bookmarks_from_toolbar: pass + test_delete_other_bookmarks: pass + test_deleted_page_not_remembered: pass + test_edit_bookmark_from_bookmark_menu: pass + test_edit_bookmark_via_star_button: pass + test_history_menu_from_different_places: pass + test_import_bookmarks_chrome: pass + test_import_bookmarks_edge: + linux: unstable + mac: unstable + win: pass + test_open_all_bookmarks_from_bookmarks_toolbar: pass + test_open_bookmark_in_new_window_via_toolbar_context_menu: pass + test_open_bookmark_in_private_window_via_toolbar_context_menu: pass + test_open_bookmarks_from_toolbar: pass + test_open_websites_from_history: pass + test_opened_website_in_new_tab_present_in_hamburger_history_menu: pass + test_opened_website_in_new_window_present_in_hamburger_history_menu: pass + test_opened_website_present_in_hamburger_history_menu: pass + test_private_window_website_not_in_history: pass + test_toggle_bookmarks_toolbar: pass + test_user_can_forget_history: pass +downloads: + test_add_mime_type_doc: + linux: pass + mac: pass + win: unstable + test_add_zip_type: + linux: unstable + mac: pass + win: pass + test_download_pdf: pass + test_download_pdf_from_context_menu: unstable + test_mixed_content_download_via_https: + win: unstable + mac: unstable + linux: pass + test_set_always_ask_file_type: pass +drag_and_drop: + test_copy_entire_row_column: pass + test_copy_from_an_editor_paste_in_another: pass + test_copy_hyperlink_table: pass + test_copy_table_header: pass + test_paste_image_text: pass +find_toolbar: + test_find_in_pdf: pass + test_find_toolbar_nav: pass + test_find_toolbar_search: pass +form_autofill: + test_address_autofill_attribute: pass + test_autofill_cc_cvv: pass + test_autofill_credit_card: pass + test_autofill_credit_card_doorhanger: pass + test_autofill_credit_card_enable: pass + test_autofill_credit_card_four_fields: pass + test_cc_clear_form: pass + test_clear_form: pass + test_create_new_cc_profile: pass + test_create_profile_autofill: pass + test_delete_cc_profile: pass + test_edit_credit_card: + win: unstable + mac: pass + linux: pass + test_enable_disable_autofill: pass + test_form_autofill_suggestions: pass + test_name_autofill_attribute: pass + test_private_mode_info_not_saved: pass + test_telephone_autofill_attribute: pass + test_updating_address: pass + test_updating_credit_card: pass +geolocation: + test_geolocation_shared_via_html5: + linux: pass + mac: pass + win: unstable + test_geolocation_shared_via_w3c_api: + linux: pass + mac: pass + win: unstable +language_packs: + test_language_pack_install_addons: pass + test_language_pack_install_preferences: pass +menus: + test_copy_paste_actions: pass + test_frequently_used_context_menu: + linux: pass + mac: pass + win: unstable + test_hyperlink_context_menu: pass + test_image_context_menu_actions: pass + test_tab_context_menu_actions: pass + test_tab_context_menu_close: pass +meta: + test_selectors: + test_a_selector: pass + test_version: + test_version: pass +networking: + test_default_dns_protection: pass + test_http_site: pass +notifications: + test_audio_video_permissions_notification: + linux: pass + mac: unstable + win: pass + test_camera_permissions_notification: + linux: pass + mac: unstable + win: pass + test_deny_geolocation: pass + test_deny_screen_capture: pass + test_geolocation_prompt_presence: pass + test_microphone_permissions_notification: + linux: pass + mac: unstable + win: pass + test_notifications_displayed: pass + test_screen_share_permission_prompt: pass + test_webextension_completed_installation_successfully_displayed: pass +password_manager: + test_about_logins_navigation_from_context_menu: pass + test_about_logins_navigation_from_hamburger_menu: pass + test_about_logins_search_username: pass + test_about_logins_search_website: pass + test_add_password_non_ascii_chars: pass + test_add_password_save_valid_data: pass + test_add_primary_password: pass + test_auto_saved_generated_password_context_menu: pass + test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load: pass + test_can_view_password_when_PP_enabled: pass + test_changes_made_in_edit_mode_are_saved: pass + test_delete_login: pass + test_multiple_saved_logins: pass + test_never_save_login_via_doorhanger: pass + test_password_csv_correctness: + linux: unstable + mac: pass + win: pass + test_password_csv_export: + linux: unstable + mac: pass + win: pass + test_primary_password_triggered_on_about_logins_access: pass + test_save_login_via_doorhanger: pass + test_saved_hyperlink_redirects_to_corresponding_page: pass + test_update_login_via_doorhanger: pass +pdf_viewer: + test_add_image_pdf: + linux: unstable + mac: pass + win: pass + test_download_pdf_with_form_fields: pass + test_download_triggered_on_content_disposition_attachment: pass + test_open_pdf_in_FF: pass + test_pdf_data_can_be_cleared: pass + test_pdf_download: pass + test_pdf_input_numbers: pass + test_pdf_navigation: pass + test_zoom_pdf_viewer: pass +pocket: + test_basic_de: + test_localized_pocket_layout_DE: deprecated + test_basic_fr: + test_localized_pocket_layout_FR: deprecated + test_basic_gb: + test_localized_pocket_layout_GB: deprecated + test_basic_us: + test_localized_pocket_layout_US: deprecated + test_new_tab_about_blank: pass +preferences: + test_check_for_updates: pass + test_clear_cookie_data: + linux: pass + mac: pass + win: unstable + test_firefox_home_new_tabs: pass + test_firefox_home_on_launch: pass + test_lang_pack_changed_from_about_prefs: pass + test_manage_cookie_data: pass + test_never_remember_history: pass + test_notifications_change_set: pass +printing_ui: + test_print_preview: pass + test_print_to_pdf: + linux: unstable + mac: pass + win: pass +profile: + test_set_default_profile: pass +reader_view: + test_improved_type_control_panel: pass + test_reader_view_close_sidebar: pass + test_reader_view_location_bar: pass +scrolling_panning_zooming: + test_default_zoom_persists: pass + test_mouse_wheel_zoom: + linux: pass + mac: unstable + win: pass + test_zoom_from_menu_bar: pass + test_zoom_menu_correlation: pass + test_zoom_text_only: pass +security_and_privacy: + test_blocking_cryptominers: pass + test_blocking_fingerprinters: pass + test_blocking_social_media_trackers: unstable + test_cookies_not_saved_private_browsing: pass + test_cross_site_tracking_cookies_blocked: pass + test_cryptominers_blocked_and_shown_in_info_panel: pass + test_detected_blocked_trackers_found: pass + test_downloads_from_private_not_leaked: pass + test_https_enabled_private_browsing: pass + test_never_remember_browsing_history: pass + test_no_trackers_detected: pass + test_open_link_in_private_window: pass + test_phishing_and_malware_warnings: pass + test_private_browser_password_doorhanger: pass + test_private_session_awesome_bar_exclusion: pass + test_private_session_history_exclusion: pass + test_private_window_from_panelui: pass + test_third_party_content_blocked_private_browsing: pass + test_tls_v1_2_protocol: pass + test_trackers_crypto_fingerprint_blocked: pass + test_tracking_content_custom_mode: pass + test_undo_close_tab_private_browsing: pass +sync_and_fxa: + test_existing_fxa: unstable + test_new_fxa: unstable +tabs: + test_active_tab: pass + test_change_position_of_pinned_tabs: pass + test_close_pinned_tab_via_mouse: pass + test_close_tab_through_middle_mouse_click: pass + test_display_customize_button: pass + test_list_all_tabs: pass + test_mute_tabs: + win: unstable + mac: pass + linux: pass + test_navigation_multiple_tabs: pass + test_open_bookmark_in_new_tab: pass + test_open_new_bg_tab_via_mouse_and_keyboard: pass + test_open_new_tab: pass + test_open_new_tab_keys: pass + test_open_new_tab_via_hyperlink: pass + test_pin_tab: pass + test_pin_unpin_selected_tabs: pass + test_play_mute_unmute_tabs_via_toggle: unstable + test_reload_overiding_cache_keys: pass + test_reload_tab_via_keyboard: pass + test_reopen_tab_through_context_menu: pass + test_reopen_tab_through_history_menu: pass + test_reopen_tabs_through_keys: pass +theme_and_toolbar: + test_customize_themes_and_redirect: pass + test_installed_theme_enabled: pass diff --git a/manifests/smoke.yaml b/manifests/smoke.yaml new file mode 100644 index 000000000..ad77aeb39 --- /dev/null +++ b/manifests/smoke.yaml @@ -0,0 +1,237 @@ +address_bar_and_search: +- test_adaptive_history_autofill +- test_add_engine_address_bar +- test_added_open_search_engine_default +- test_addon_suggestion +- test_addressbar_search_engine_keywords +- test_copied_url_contains_https +- test_ctrl_enter_fixes_url +- test_default_search_provider_change_awesome_bar +- test_default_search_provider_change_legacy_search_bar +- test_dont_show_search_suggestions_in_private_window +- test_google_search_counts_us +- test_google_withads_url_bar_us +- test_insertion_point_no_search_terms_display +- test_no_suggestions_for_empty_query +- test_non_sponsored_topsite_context_menu_option +- test_refresh_firefox_dialog +- test_sap_google_adclick +- test_seach_suggestions_can_be_disabled +- test_search_code_google_non_us +- test_search_code_google_us +- test_search_engine_selector +- test_search_mode_appears_after_input +- test_search_mode_persists_mixed_with_bing +- test_search_modes_for_sites +- test_search_string_displayed_when_addressbar_unfocused +- test_search_suggestions +- test_search_term_persists +- test_searchbar_display_alpenglow_theme +- test_searchbar_on_engines_remove_restore +- test_searchbar_results_shown_in_a_new_tab +- test_searchmode_change_tab +- test_server_not_found_error +- test_server_not_found_error_pb +- test_suggestion_engine_selection +- test_tile_menu_options +audio_video: +- test_allow_audio_video_functionality +- test_block_audio_video_functionality +- test_users_actions_saved_on_reload +bookmarks_and_history: +- test_add_new_other_bookmark +- test_bookmark_via_bookmark_menu +- test_bookmark_website_via_star_button +- test_clear_all_history +- test_clear_recent_history_displayed +- test_delete_bookmarks_from_toolbar +- test_delete_other_bookmarks +- test_deleted_page_not_remembered +- test_edit_bookmark_from_bookmark_menu +- test_edit_bookmark_via_star_button +- test_history_menu_from_different_places +- test_import_bookmarks_chrome +- test_import_bookmarks_edge +- test_open_all_bookmarks_from_bookmarks_toolbar +- test_open_bookmark_in_new_window_via_toolbar_context_menu +- test_open_bookmark_in_private_window_via_toolbar_context_menu +- test_open_bookmarks_from_toolbar +- test_open_websites_from_history +- test_opened_website_in_new_tab_present_in_hamburger_history_menu +- test_opened_website_in_new_window_present_in_hamburger_history_menu +- test_opened_website_present_in_hamburger_history_menu +- test_private_window_website_not_in_history +- test_toggle_bookmarks_toolbar +- test_user_can_forget_history +downloads: +- test_add_mime_type_doc +- test_add_zip_type +- test_download_pdf +- test_download_pdf_from_context_menu +- test_mixed_content_download_via_https +- test_set_always_ask_file_type +drag_and_drop: +- test_copy_entire_row_column +- test_copy_from_an_editor_paste_in_another +- test_copy_hyperlink_table +- test_copy_table_header +- test_paste_image_text +find_toolbar: +- test_find_in_pdf +- test_find_toolbar_nav +- test_find_toolbar_search +form_autofill: +- test_address_autofill_attribute +- test_autofill_cc_cvv +- test_autofill_credit_card +- test_autofill_credit_card_doorhanger +- test_autofill_credit_card_enable +- test_autofill_credit_card_four_fields +- test_cc_clear_form +- test_clear_form +- test_create_new_cc_profile +- test_create_profile_autofill +- test_delete_cc_profile +- test_edit_credit_card +- test_enable_disable_autofill +- test_form_autofill_suggestions +- test_name_autofill_attribute +- test_private_mode_info_not_saved +- test_telephone_autofill_attribute +- test_updating_address +- test_updating_credit_card +geolocation: +- test_geolocation_shared_via_html5 +- test_geolocation_shared_via_w3c_api +language_packs: +- test_language_pack_install_addons +- test_language_pack_install_preferences +menus: +- test_copy_paste_actions +- test_frequently_used_context_menu +- test_hyperlink_context_menu +- test_image_context_menu_actions +- test_tab_context_menu_actions +- test_tab_context_menu_close +meta: +- test_selectors +- test_version +networking: +- test_default_dns_protection +- test_http_site +notifications: +- test_audio_video_permissions_notification +- test_camera_permissions_notification +- test_deny_geolocation +- test_deny_screen_capture +- test_geolocation_prompt_presence +- test_microphone_permissions_notification +- test_notifications_displayed +- test_screen_share_permission_prompt +- test_webextension_completed_installation_successfully_displayed +password_manager: +- test_about_logins_navigation_from_context_menu +- test_about_logins_navigation_from_hamburger_menu +- test_about_logins_search_username +- test_about_logins_search_website +- test_add_password_non_ascii_chars +- test_add_password_save_valid_data +- test_add_primary_password +- test_auto_saved_generated_password_context_menu +- test_autocomplete_dropdown_is_toggled_for_focused_login_fields_on_page_load +- test_can_view_password_when_PP_enabled +- test_changes_made_in_edit_mode_are_saved +- test_delete_login +- test_multiple_saved_logins +- test_never_save_login_via_doorhanger +- test_password_csv_correctness +- test_password_csv_export +- test_primary_password_triggered_on_about_logins_access +- test_save_login_via_doorhanger +- test_saved_hyperlink_redirects_to_corresponding_page +- test_update_login_via_doorhanger +pdf_viewer: +- test_add_image_pdf +- test_download_pdf_with_form_fields +- test_download_triggered_on_content_disposition_attachment +- test_open_pdf_in_FF +- test_pdf_data_can_be_cleared +- test_pdf_download +- test_pdf_input_numbers +- test_pdf_navigation +- test_zoom_pdf_viewer +preferences: +- test_check_for_updates +- test_clear_cookie_data +- test_firefox_home_new_tabs +- test_firefox_home_on_launch +- test_lang_pack_changed_from_about_prefs +- test_manage_cookie_data +- test_never_remember_history +- test_notifications_change_set +printing_ui: +- test_print_preview +- test_print_to_pdf +profile: +- test_set_default_profile +reader_view: +- test_improved_type_control_panel +- test_reader_view_close_sidebar +- test_reader_view_location_bar +scrolling_panning_zooming: +- test_default_zoom_persists +- test_mouse_wheel_zoom +- test_zoom_from_menu_bar +- test_zoom_menu_correlation +- test_zoom_text_only +security_and_privacy: +- test_blocking_cryptominers +- test_blocking_fingerprinters +- test_blocking_social_media_trackers +- test_cookies_not_saved_private_browsing +- test_cross_site_tracking_cookies_blocked +- test_cryptominers_blocked_and_shown_in_info_panel +- test_detected_blocked_trackers_found +- test_downloads_from_private_not_leaked +- test_https_enabled_private_browsing +- test_never_remember_browsing_history +- test_no_trackers_detected +- test_open_link_in_private_window +- test_phishing_and_malware_warnings +- test_private_browser_password_doorhanger +- test_private_session_awesome_bar_exclusion +- test_private_session_history_exclusion +- test_private_window_from_panelui +- test_third_party_content_blocked_private_browsing +- test_tls_v1_2_protocol +- test_trackers_crypto_fingerprint_blocked +- test_tracking_content_custom_mode +- test_undo_close_tab_private_browsing +sync_and_fxa: +- test_existing_fxa +- test_new_fxa +tabs: +- test_active_tab +- test_change_position_of_pinned_tabs +- test_close_pinned_tab_via_mouse +- test_close_tab_through_middle_mouse_click +- test_display_customize_button +- test_list_all_tabs +- test_mute_tabs +- test_navigation_multiple_tabs +- test_open_bookmark_in_new_tab +- test_open_new_bg_tab_via_mouse_and_keyboard +- test_open_new_tab +- test_open_new_tab_keys +- test_open_new_tab_via_hyperlink +- test_pin_tab +- test_pin_unpin_selected_tabs +- test_play_mute_unmute_tabs_via_toggle +- test_reload_overiding_cache_keys +- test_reload_tab_via_keyboard +- test_reopen_tab_through_context_menu +- test_reopen_tab_through_history_menu +- test_reopen_tabs_through_keys +theme_and_toolbar: +- test_customize_themes_and_redirect +- test_installed_theme_enabled diff --git a/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py b/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py index de55b2d51..4ab88b435 100644 --- a/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py +++ b/tests/address_bar_and_search/test_default_search_provider_change_awesome_bar.py @@ -13,7 +13,6 @@ def test_case(): return "3028795" -@pytest.mark.ci def test_default_search_provider_change_awesome_bar(driver: Firefox): """ C2860208 - Verify that changing the default search provider updates the address bar placeholder. diff --git a/tests/address_bar_and_search/test_google_search_counts_us.py b/tests/address_bar_and_search/test_google_search_counts_us.py index 893a4f06a..c1b4974d0 100644 --- a/tests/address_bar_and_search/test_google_search_counts_us.py +++ b/tests/address_bar_and_search/test_google_search_counts_us.py @@ -1,5 +1,3 @@ -import sys -from os import environ from time import sleep import pytest @@ -16,15 +14,13 @@ WAIT_AFTER_SEARCH = 5 WAIT_TELEMETRY_LOAD = 2 -MAC_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("darwin") - @pytest.fixture() def test_case(): return "3029528" -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in macOS GitHub Actions") +# This test is unstable in MacOS GHA for now def test_google_search_counts_us(driver: Firefox): """ C1365026 - Verify Google search counts in telemetry from the URL bar (US region). diff --git a/tests/address_bar_and_search/test_google_withads_url_bar_us.py b/tests/address_bar_and_search/test_google_withads_url_bar_us.py index dd06b786a..501548b30 100644 --- a/tests/address_bar_and_search/test_google_withads_url_bar_us.py +++ b/tests/address_bar_and_search/test_google_withads_url_bar_us.py @@ -8,8 +8,6 @@ from modules.page_object import AboutTelemetry from modules.util import Utilities -MAC_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("darwin") - @pytest.fixture() def test_case(): @@ -21,8 +19,8 @@ def add_to_prefs_list(): return [("cookiebanners.service.mode", 1)] -@pytest.mark.unstable(reason="Google re-captcha") -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Google re-captcha makes test unstable for now +# Mac GHA makes test unstable for now def test_google_withads_url_bar_us(driver): """ C1365070 - Retry up to 5 times if Google CAPTCHA blocks telemetry path. diff --git a/tests/address_bar_and_search/test_sap_google_adclick.py b/tests/address_bar_and_search/test_sap_google_adclick.py index 0c6caf75e..395b643db 100644 --- a/tests/address_bar_and_search/test_sap_google_adclick.py +++ b/tests/address_bar_and_search/test_sap_google_adclick.py @@ -17,7 +17,7 @@ def test_case(): return "3029662" -@pytest.mark.unstable(reason="Google re-captcha") +# Google recaptcha makes this test unstable for now def test_sap_google_adclick(driver: Firefox): """ C1365108 - Verify Google ad click from URL bar is recorded in telemetry (US region). diff --git a/tests/address_bar_and_search/test_search_modes_for_sites.py b/tests/address_bar_and_search/test_search_modes_for_sites.py index 56fed538e..2a67223ae 100644 --- a/tests/address_bar_and_search/test_search_modes_for_sites.py +++ b/tests/address_bar_and_search/test_search_modes_for_sites.py @@ -17,7 +17,8 @@ def test_case(): return "3028754" -@pytest.mark.unstable(reason="Google re-captcha and manual tagged this test as removed") +# Google recaptcha makes this test unstable for now +# Test has been removed from main TestRail suites @pytest.mark.parametrize("search_engine, prefix, url", SEARCH_MODES) def test_search_modes_for_sites( driver: Firefox, search_engine: str, prefix: str, url: str diff --git a/tests/address_bar_and_search/test_search_term_persists.py b/tests/address_bar_and_search/test_search_term_persists.py index 37e332d9d..f409fc361 100644 --- a/tests/address_bar_and_search/test_search_term_persists.py +++ b/tests/address_bar_and_search/test_search_term_persists.py @@ -24,7 +24,7 @@ def add_to_prefs_list(): SECOND_SEARCH = "lion" -@pytest.mark.unstable +# Google re-captcha makes this test unstable for now def test_search_term_persists(driver: Firefox): """ C2153943 - Persist search term basic functionality diff --git a/tests/audio_video/test_allow_audio_video_functionality.py b/tests/audio_video/test_allow_audio_video_functionality.py index 9826a0dc8..23e8178d8 100644 --- a/tests/audio_video/test_allow_audio_video_functionality.py +++ b/tests/audio_video/test_allow_audio_video_functionality.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox @@ -14,13 +11,10 @@ def test_case(): return "330155" -WIN_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") TEST_URL = "https://www.w3schools.com/html/mov_bbb.mp4" -@pytest.mark.skipif( - WIN_GHA, reason="Audio playback not supported in Windows CI environment" -) +# Test is unstable in Windows GHA because audio playback is not allowed @pytest.mark.audio @pytest.mark.noxvfb def test_allow_audio_video_functionality(driver: Firefox): diff --git a/tests/bookmarks_and_history/test_bookmark_website_via_star_button.py b/tests/bookmarks_and_history/test_bookmark_website_via_star_button.py index 9b71b0cab..d01def77c 100644 --- a/tests/bookmarks_and_history/test_bookmark_website_via_star_button.py +++ b/tests/bookmarks_and_history/test_bookmark_website_via_star_button.py @@ -15,7 +15,6 @@ def test_case(): BOOKMARK_TITLE = "Internet for people" -@pytest.mark.ci def test_bookmark_website_via_star(driver: Firefox): """ C2084539: Verify that the Websites can be bookmarked via star-shaped button diff --git a/tests/bookmarks_and_history/test_import_bookmarks_edge.py b/tests/bookmarks_and_history/test_import_bookmarks_edge.py index 779d52896..6e34a58ed 100644 --- a/tests/bookmarks_and_history/test_import_bookmarks_edge.py +++ b/tests/bookmarks_and_history/test_import_bookmarks_edge.py @@ -1,5 +1,4 @@ import os -import sys from shutil import copyfile import pytest @@ -46,15 +45,7 @@ def edge_bookmarks(sys_platform, home_folder): return target -@pytest.mark.skipif( - sys.platform.lower().startswith("linux"), - reason="Only testing Edge on Win and MacOS", -) -@pytest.mark.skipif( - os.environ.get("GITHUB_ACTIONS") == "true" - and not sys.platform.lower().startswith("win"), - reason="No Edge on GHA Mac", -) +# Test should only be run on Windows machines. def test_edge_bookmarks_imported(driver: Firefox, edge_bookmarks, sys_platform): about_prefs = AboutPrefs(driver, category="General") about_prefs.open() diff --git a/tests/downloads/test_add_mime_type_doc.py b/tests/downloads/test_add_mime_type_doc.py index 96492fd46..1102ceb16 100644 --- a/tests/downloads/test_add_mime_type_doc.py +++ b/tests/downloads/test_add_mime_type_doc.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox @@ -14,7 +11,6 @@ def test_case(): DOC_LINK = "https://sapphire-hendrika-5.tiiny.site/" -WIN_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") @pytest.fixture() @@ -28,9 +24,7 @@ def expected_app_name(sys_platform: str, opt_ci: bool) -> str: return "LibreOffice Writer" -@pytest.mark.skipif( - WIN_GHA, reason="Most runners don’t have a .doc handler registered on Windows" -) +# Test is unstable on Windows GHA because the image does not have a .doc handler @pytest.mark.noxvfb def test_mime_type_doc(driver: Firefox, sys_platform: str, opt_ci: bool, delete_files): """ diff --git a/tests/downloads/test_mixed_content_download_via_https.py b/tests/downloads/test_mixed_content_download_via_https.py index e3ceb24ee..b150c32f9 100644 --- a/tests/downloads/test_mixed_content_download_via_https.py +++ b/tests/downloads/test_mixed_content_download_via_https.py @@ -21,7 +21,7 @@ def delete_files_regex_string(): MAX_CHECKS = 30 -@pytest.mark.unstable(reason="Unstable in CI environment") +# This test has been found to be unstable in CI def test_mixed_content_download_via_https(driver: Firefox, delete_files): """ C1756722: Verify that the user can download mixed content via HTTPS diff --git a/tests/downloads/test_set_always_ask_file_type.py b/tests/downloads/test_set_always_ask_file_type.py index b9abd7244..330b1cc5a 100644 --- a/tests/downloads/test_set_always_ask_file_type.py +++ b/tests/downloads/test_set_always_ask_file_type.py @@ -20,7 +20,6 @@ def delete_files_regex_string(): ) -@pytest.mark.ci def test_set_always_ask_file_type(driver: Firefox, delete_files): """ C1756752 - Ensure that the Always ask option in Firefox Applications settings diff --git a/tests/find_toolbar/test_find_toolbar_search.py b/tests/find_toolbar/test_find_toolbar_search.py index 3c9dabae8..9f06198ac 100644 --- a/tests/find_toolbar/test_find_toolbar_search.py +++ b/tests/find_toolbar/test_find_toolbar_search.py @@ -27,7 +27,6 @@ def are_lists_different(a: int, b: int) -> bool: return abs(a - b) > TOLERANCE -@pytest.mark.ci def test_find_toolbar_search( driver: Firefox, find_toolbar: FindToolbar, browser_actions: BrowserActions ): diff --git a/tests/form_autofill/test_telephone_autofill_attribute.py b/tests/form_autofill/test_telephone_autofill_attribute.py index eeda769c1..11803d8e4 100644 --- a/tests/form_autofill/test_telephone_autofill_attribute.py +++ b/tests/form_autofill/test_telephone_autofill_attribute.py @@ -11,7 +11,6 @@ def test_case(): return "122361" -@pytest.mark.ci def test_telephone_attribute_autofill( driver: Firefox, address_autofill: AddressFill, diff --git a/tests/geolocation/test_geolocation_shared_via_html5.py b/tests/geolocation/test_geolocation_shared_via_html5.py index 971b48337..b9b754954 100644 --- a/tests/geolocation/test_geolocation_shared_via_html5.py +++ b/tests/geolocation/test_geolocation_shared_via_html5.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox from selenium.webdriver.common.by import By @@ -28,8 +25,6 @@ def add_to_prefs_list(): TEST_URL = "https://browserleaks.com/geo" -WIN_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") - def wait_for_geolocation_data(web_page, timeout=20): """Wait until both latitude and longitude data are available.""" @@ -45,7 +40,7 @@ def wait_for_geolocation_data(web_page, timeout=20): ) -@pytest.mark.skipif(WIN_GHA, reason="Recent permission changes at their side") +# Test is unstable on Windows GHA because of permission changes on the CI image def test_allow_permission_on_geolocation_via_html5(driver: Firefox): """ C15189 - Verify that geolocation is successfully shared when the user allows permission via the HTML5 Geolocation API @@ -77,7 +72,7 @@ def test_allow_permission_on_geolocation_via_html5(driver: Firefox): assert permission_icon.is_displayed() -@pytest.mark.skipif(WIN_GHA, reason="Recent permission changes at their side") +# Test is unstable on Windows GHA because of permission changes on the CI image def test_block_permission_on_geolocation_via_w3c_api(driver: Firefox): """ C15189 - Verify that geolocation is not shared when the user blocks permission via the HTML5 Geolocation API diff --git a/tests/geolocation/test_geolocation_shared_via_w3c_api.py b/tests/geolocation/test_geolocation_shared_via_w3c_api.py index 4b76e4638..bd192f570 100644 --- a/tests/geolocation/test_geolocation_shared_via_w3c_api.py +++ b/tests/geolocation/test_geolocation_shared_via_w3c_api.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox @@ -27,8 +24,6 @@ def add_to_prefs_list(): TEST_URL = "https://www.w3schools.com/html/html5_geolocation.asp" -WIN_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") - @pytest.fixture() def temp_selectors(): @@ -51,7 +46,7 @@ def temp_selectors(): } -@pytest.mark.skipif(WIN_GHA, reason="Recent permission changes at their side") +# Test is unstable on Windows GHA because of permission changes on the CI image def test_allow_permission_on_geolocation_via_w3c_api(driver: Firefox, temp_selectors): """ C15186 - Verify that geolocation is successfully shared when the user allows permission via the W3C Geolocation API @@ -97,7 +92,7 @@ def test_allow_permission_on_geolocation_via_w3c_api(driver: Firefox, temp_selec assert permission_icon.is_displayed() -@pytest.mark.skipif(WIN_GHA, reason="Recent permission changes at their side") +# Test is unstable on Windows GHA because of permission changes on the CI image def test_block_permission_on_geolocation_via_w3c_api(driver: Firefox, temp_selectors): """ C15186 - Verify that geolocation is not shared when the user blocks permission via the W3C Geolocation API diff --git a/tests/menus/test_copy_paste_actions.py b/tests/menus/test_copy_paste_actions.py index 1ccbe090d..e1d97c53d 100644 --- a/tests/menus/test_copy_paste_actions.py +++ b/tests/menus/test_copy_paste_actions.py @@ -15,7 +15,6 @@ def test_case(): return "2264626" -@pytest.mark.ci def test_login_form_copy_paste(driver: Firefox): """C2264626 - Verify that copy and paste actions are displayed in the context menu and work as expected""" # Instantiate objects diff --git a/tests/menus/test_frequently_used_context_menu.py b/tests/menus/test_frequently_used_context_menu.py index dfe0ed72d..1cdb94a67 100644 --- a/tests/menus/test_frequently_used_context_menu.py +++ b/tests/menus/test_frequently_used_context_menu.py @@ -1,6 +1,5 @@ import os import platform -import sys from time import sleep import pytest @@ -16,10 +15,7 @@ def test_case(): return "2637623" -WIN_GHA = os.environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") - - -@pytest.mark.skipif(WIN_GHA, reason="Test unstable in Windows Github Actions") +# Test is unstable in Windows GHA for now @pytest.mark.headed def test_save_page_as(driver: Firefox): """ diff --git a/tests/menus/test_image_context_menu_actions.py b/tests/menus/test_image_context_menu_actions.py index c7fcbe088..8aaa42796 100644 --- a/tests/menus/test_image_context_menu_actions.py +++ b/tests/menus/test_image_context_menu_actions.py @@ -52,7 +52,6 @@ def test_open_image_in_new_tab(driver: Firefox): wiki_image_page.verify_opened_image_url("wikimedia", LOADED_IMAGE_URL) -@pytest.mark.ci @pytest.mark.headed def test_save_image_as(driver: Firefox, sys_platform, delete_files): """ diff --git a/tests/networking/test_http_site.py b/tests/networking/test_http_site.py index 2ffa78536..d53f67153 100644 --- a/tests/networking/test_http_site.py +++ b/tests/networking/test_http_site.py @@ -27,7 +27,6 @@ def add_to_prefs_list(): CONNECTION_NOT_SECURE = "Connection is not secure" -@pytest.mark.ci def test_http_site(driver: Firefox): """C2300294 Check that HTTP is allowed when appropriate""" diff --git a/tests/notifications/test_audio_video_permissions_notification.py b/tests/notifications/test_audio_video_permissions_notification.py index da53480aa..5b3457263 100644 --- a/tests/notifications/test_audio_video_permissions_notification.py +++ b/tests/notifications/test_audio_video_permissions_notification.py @@ -1,5 +1,3 @@ -import sys -from os import environ from time import sleep import pytest @@ -32,10 +30,9 @@ def add_to_prefs_list(): TEST_URL = "https://mozilla.github.io/webrtc-landing/gum_test.html" -MAC_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("darwin") -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Test is unstable in MacOS GHA for now def test_camera_and_microphone_permissions_notification( driver: Firefox, temp_selectors ): diff --git a/tests/notifications/test_camera_permissions_notification.py b/tests/notifications/test_camera_permissions_notification.py index aacba7e48..a2d3c1d37 100644 --- a/tests/notifications/test_camera_permissions_notification.py +++ b/tests/notifications/test_camera_permissions_notification.py @@ -1,5 +1,3 @@ -import sys -from os import environ from time import sleep import pytest @@ -32,10 +30,9 @@ def add_to_prefs_list(): TEST_URL = "https://mozilla.github.io/webrtc-landing/gum_test.html" -MAC_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("darwin") -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Test is unstable in MacOS GHA for now def test_camera_permissions_notification(driver: Firefox, temp_selectors): """ C122536 - Verify that Camera only permission prompt is successfully displayed when the website asks for camera permissions diff --git a/tests/notifications/test_microphone_permissions_notification.py b/tests/notifications/test_microphone_permissions_notification.py index 423e807e1..e28cbd960 100644 --- a/tests/notifications/test_microphone_permissions_notification.py +++ b/tests/notifications/test_microphone_permissions_notification.py @@ -1,5 +1,3 @@ -import sys -from os import environ from time import sleep import pytest @@ -32,10 +30,9 @@ def add_to_prefs_list(): TEST_URL = "https://mozilla.github.io/webrtc-landing/gum_test.html" -MAC_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("darwin") -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Test is unstable in MacOS GHA for now def test_microphone_permissions_notification(driver: Firefox, temp_selectors): """ C122539 - Verify that Microphone only permission prompt is successfully displayed when the website asks for microphone permissions diff --git a/tests/notifications/test_notifications_displayed.py b/tests/notifications/test_notifications_displayed.py index 4dc293477..3fdcb94e8 100644 --- a/tests/notifications/test_notifications_displayed.py +++ b/tests/notifications/test_notifications_displayed.py @@ -44,7 +44,6 @@ def temp_selectors(): } -@pytest.mark.ci def test_notifications_displayed(driver: Firefox, temp_page, temp_selectors): """ This test does not (and SHOULD not) test that the OS displays web notifications diff --git a/tests/password_manager/test_about_logins_navigation_from_context_menu.py b/tests/password_manager/test_about_logins_navigation_from_context_menu.py index 5518f9961..107028c9c 100644 --- a/tests/password_manager/test_about_logins_navigation_from_context_menu.py +++ b/tests/password_manager/test_about_logins_navigation_from_context_menu.py @@ -12,7 +12,6 @@ def test_case(): return "2241087" -@pytest.mark.ci def test_about_logins_navigation_from_login_form_context_menu(driver: Firefox): """ C2241087 - Verify that right-clicking the Username field in a login form and then the Manage Passwords option diff --git a/tests/password_manager/test_password_csv_correctness.py b/tests/password_manager/test_password_csv_correctness.py index 10cf7ee5c..c84333d32 100644 --- a/tests/password_manager/test_password_csv_correctness.py +++ b/tests/password_manager/test_password_csv_correctness.py @@ -14,7 +14,7 @@ def test_case(): return "2241522" -@pytest.mark.unstable(reason="Bug 1996004") +# This test is unstable for now: Bug 1996004 @pytest.mark.headed @pytest.mark.noxvfb def test_password_csv_correctness( diff --git a/tests/password_manager/test_password_csv_export.py b/tests/password_manager/test_password_csv_export.py index 576b1e9a1..4f4d36676 100644 --- a/tests/password_manager/test_password_csv_export.py +++ b/tests/password_manager/test_password_csv_export.py @@ -13,7 +13,7 @@ def test_case(): return "2241521" -@pytest.mark.unstable(reason="Bug 1996005") +# This test is unstable for now: Bug 1996005 @pytest.mark.headed @pytest.mark.noxvfb def test_password_csv_export( diff --git a/tests/pdf_viewer/test_add_image_pdf.py b/tests/pdf_viewer/test_add_image_pdf.py index 4174697bb..a5df7e415 100644 --- a/tests/pdf_viewer/test_add_image_pdf.py +++ b/tests/pdf_viewer/test_add_image_pdf.py @@ -1,5 +1,4 @@ from pathlib import Path -from platform import system from time import sleep import pytest @@ -33,7 +32,7 @@ def add_to_prefs_list(): ] -@pytest.mark.skipif(system().lower().startswith("linux"), reason="Bug 1983852") +# This test is unstable in Linux (all CI) for now: Bug 1983852 @pytest.mark.headed @pytest.mark.noxvfb def test_add_image_pdf(driver: Firefox, sys_platform, pdf_viewer: GenericPdf): diff --git a/tests/pdf_viewer/test_pdf_data_can_be_cleared.py b/tests/pdf_viewer/test_pdf_data_can_be_cleared.py index 7976bb619..dd3fc291c 100644 --- a/tests/pdf_viewer/test_pdf_data_can_be_cleared.py +++ b/tests/pdf_viewer/test_pdf_data_can_be_cleared.py @@ -27,7 +27,6 @@ def file_name(): return "i-9.pdf" -@pytest.mark.ci def test_pdf_data_can_be_cleared( driver: Firefox, pdf_viewer: GenericPdf, diff --git a/tests/printing_ui/test_print_preview.py b/tests/printing_ui/test_print_preview.py index f4fe4f112..bbc730ec7 100644 --- a/tests/printing_ui/test_print_preview.py +++ b/tests/printing_ui/test_print_preview.py @@ -19,7 +19,6 @@ def test_print_preview_menu(driver: Firefox): print_preview.open() -@pytest.mark.ci def test_print_preview_keys(driver: Firefox): """C965139 - Check for print preview modal (Key Combo)""" driver.get(TEST_PAGE) diff --git a/tests/printing_ui/test_print_to_pdf.py b/tests/printing_ui/test_print_to_pdf.py index 09df9b887..69218a8f7 100644 --- a/tests/printing_ui/test_print_to_pdf.py +++ b/tests/printing_ui/test_print_to_pdf.py @@ -1,6 +1,5 @@ import logging import os -from platform import system import pytest from selenium.webdriver import Firefox @@ -45,10 +44,7 @@ def file_is_somewhere(): return False -@pytest.mark.skipif( - system().lower().startswith("win") or system().lower().startswith("linux"), - reason="Bug 1974011", -) +# Test is unstable in Windows GHA and Linux Taskcluster for now: Bug 1974011 @pytest.mark.headed def test_print_to_pdf( driver: Firefox, diff --git a/tests/reader_view/test_improved_type_control_panel.py b/tests/reader_view/test_improved_type_control_panel.py index b94c4d0d8..47a5ca795 100644 --- a/tests/reader_view/test_improved_type_control_panel.py +++ b/tests/reader_view/test_improved_type_control_panel.py @@ -55,7 +55,6 @@ def _css_int(util: Utilities, element, prop: str) -> int: return int(util.remove_all_non_numbers(element.value_of_css_property(prop))) -@pytest.mark.ci @pytest.mark.parametrize("font", FONTS) def test_type_control_panel_font( driver: Firefox, font: Literal["sans-serif", "serif", "monospace"] diff --git a/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py b/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py index ef5c0bdf6..3db3177f2 100644 --- a/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py +++ b/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py @@ -1,5 +1,4 @@ import logging -import platform import time import pytest @@ -19,11 +18,7 @@ def test_case(): TEST_PAGE = "https://www.example.com" -# Skip this test if running on macOS -@pytest.mark.skipif( - platform.system() == "Darwin", - reason="Test skipped on macOS due to incompatible zoom controls", -) +# This test is not compatible with MacOS wheel controls def test_mouse_wheel_zoom(driver: Firefox): """ This test verifies that the X-coordinate of a
element's position @@ -38,7 +33,8 @@ def test_mouse_wheel_zoom(driver: Firefox): # Locate the main
element on the page div = driver.find_element(By.TAG_NAME, "div") - initial_position = div.location["x"] # Get the initial X position of the div + # Get the initial X position of the div + initial_position = div.location["x"] logging.info(f"Initial X position of div: {initial_position}") # Initialize ActionChains for zooming with Ctrl + Mouse Wheel diff --git a/tests/scrolling_panning_zooming/test_zoom_text_only.py b/tests/scrolling_panning_zooming/test_zoom_text_only.py index c67e32d37..cc1c6a2d5 100644 --- a/tests/scrolling_panning_zooming/test_zoom_text_only.py +++ b/tests/scrolling_panning_zooming/test_zoom_text_only.py @@ -85,7 +85,6 @@ def reject_consent_page(web_page: GenericPage): @pytest.mark.skip(reason="Tracked in bug 1991139") -@pytest.mark.ci @pytest.mark.noxvfb def test_zoom_text_only_from_settings( driver: Firefox, web_page: GenericPage, reject_consent_page diff --git a/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py b/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py index b8e48d669..7875dcbb3 100644 --- a/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py +++ b/tests/security_and_privacy/test_cross_site_tracking_cookies_blocked.py @@ -30,7 +30,6 @@ def add_to_prefs_list(): ] -@pytest.mark.ci def test_cross_site_tracking_cookies_blocked(driver: Firefox): """ C446402: Ensures the cross tracking cookies are displayed in the tracker panel diff --git a/tests/tabs/test_active_tab.py b/tests/tabs/test_active_tab.py index 9ff4c8904..12245ddd4 100644 --- a/tests/tabs/test_active_tab.py +++ b/tests/tabs/test_active_tab.py @@ -11,7 +11,6 @@ def test_case(): return "134646" -@pytest.mark.ci def test_active_tab(driver: Firefox): """ C134646, ensures that the selected tab is highlighted diff --git a/tests/tabs/test_mute_tabs.py b/tests/tabs/test_mute_tabs.py index 80aa03f9c..0fac924db 100644 --- a/tests/tabs/test_mute_tabs.py +++ b/tests/tabs/test_mute_tabs.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox from selenium.webdriver.support import expected_conditions as EC @@ -11,7 +8,6 @@ COOKIE_CONSENT_SELECTOR = ( "button[aria-label^='Accept all'], button[aria-label^='Accept the use']" ) -WIN_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win") @pytest.fixture() @@ -24,7 +20,7 @@ def add_to_prefs_list(): return [("network.cookie.cookieBehavior", "2")] -@pytest.mark.skipif(WIN_GHA, reason="Test unstable on Windows in Github Actions") +# This test is unstable in Windows GHA for now @pytest.mark.audio def test_mute_unmute_tab(screenshot, driver: Firefox, video_url: str): """C134719, test that tabs can be muted and unmuted""" diff --git a/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py b/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py index f8535a614..dd619f8a3 100644 --- a/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py +++ b/tests/tabs/test_play_mute_unmute_tabs_via_toggle.py @@ -1,5 +1,3 @@ -from os import environ -from platform import system from time import sleep import pytest @@ -14,8 +12,6 @@ from modules.browser_object_tabbar import TabBar from modules.page_object_generics import GenericPage -GHA = environ.get("GITHUB_ACTIONS") == "true" - @pytest.fixture() def test_case(): @@ -27,10 +23,9 @@ def add_to_prefs_list(): return [("network.cookie.cookieBehavior", "2")] +# This test is unstable in Windows GHA for now @pytest.mark.audio @pytest.mark.headed -@pytest.mark.skipif(GHA, reason="Test unstable in Github Actions") -@pytest.mark.skipif(system().lower().startswith("linux"), reason="Bug 2001204") def test_play_mute_unmute_tabs_via_toggle(driver: Firefox, sys_platform: str): """ C246981 - Verify that play/mute/unmute tabs via toggle audio works diff --git a/tests/tabs/test_reopen_tab_through_history_menu.py b/tests/tabs/test_reopen_tab_through_history_menu.py index 53b8f37ab..09e8fe6d6 100644 --- a/tests/tabs/test_reopen_tab_through_history_menu.py +++ b/tests/tabs/test_reopen_tab_through_history_menu.py @@ -1,5 +1,4 @@ import logging -from platform import system import pytest from selenium.webdriver import Firefox @@ -26,9 +25,7 @@ def test_case(): NUM_TABS = 6 -@pytest.mark.skipif( - system().lower().startswith("linux"), reason="Currently unstable in linux" -) +# This test is unstable in Linux Taskcluster for now def test_reopen_tab_through_history_menu(driver: Firefox): """C134650 - Verify that the recently closed tab can be reopened from the history menu""" diff --git a/tests/theme_and_toolbar/test_customize_themes_and_redirect.py b/tests/theme_and_toolbar/test_customize_themes_and_redirect.py index a732c01ad..c506a31d4 100644 --- a/tests/theme_and_toolbar/test_customize_themes_and_redirect.py +++ b/tests/theme_and_toolbar/test_customize_themes_and_redirect.py @@ -61,7 +61,6 @@ def colors_match(a: str, b: str, tolerance: float = 0.14) -> bool: return True -@pytest.mark.ci def test_redirect_to_addons(driver: Firefox) -> None: """ C118173: Ensure the user is redirected to about:addons via the UI panel. diff --git a/tests/theme_and_toolbar/test_installed_theme_enabled.py b/tests/theme_and_toolbar/test_installed_theme_enabled.py index ec39cc802..42cbb006f 100644 --- a/tests/theme_and_toolbar/test_installed_theme_enabled.py +++ b/tests/theme_and_toolbar/test_installed_theme_enabled.py @@ -1,6 +1,3 @@ -import sys -from os import environ - import pytest from selenium.webdriver import Firefox @@ -12,15 +9,11 @@ def test_case(): return "118174" -MAC_GHA: bool = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith( - "darwin" -) - AMO_HOST: str = "addons.mozilla.org" AMO_THEMES_PATH: str = "firefox/themes" -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Test is unstable in MacOS GHA for now def test_find_more_themes(driver: Firefox) -> None: """ C118174 (part 1): From about:addons > Themes, 'Find more themes' opens AMO in a new tab. @@ -37,7 +30,7 @@ def test_find_more_themes(driver: Firefox) -> None: base.url_contains(AMO_THEMES_PATH) -@pytest.mark.skipif(MAC_GHA, reason="Test unstable in MacOS Github Actions") +# Test is unstable in MacOS GHA for now def test_installed_theme_enabled(driver: Firefox) -> None: """ C118174 (part 2): Install a recommended theme from AMO and ensure it becomes enabled immediately.