[IGNORE] Execute incident tests based on a manifest

.github/workflows/smoke.yml

@@ -88,12 +88,16 @@ jobs:
           echo $fx_url
           Invoke-WebRequest -Uri $fx_url -OutFile "${{ github.workspace }}\setup.exe"
           New-Item -ItemType Directory -Path "C:\Program Files\Custom Firefox" -Force
+          "MANIFEST=" >> $env:GITHUB_ENV
+          "MANIFEST_ARG=" >> $env:GITHUB_ENV
         shell: pwsh
       - name: Download Executable
         if: ${{ inputs.win_installer_link }}
         run: |
           Invoke-WebRequest -Uri ${{ inputs.win_installer_link }} -OutFile "${{ github.workspace }}\setup.exe"
           New-Item -ItemType Directory -Path "C:\Program Files\Custom Firefox" -Force
+          "MANIFEST=incident_manifest.yaml" >> $env:GITHUB_ENV
+          "MANIFEST_ARG=\"--manifest incident_manifest.yaml\"" >> $env:GITHUB_ENV
         shell: pwsh
       - name: Install Beta
         id: setup
@@ -106,8 +110,8 @@ jobs:
           pipenv run python -c "import sys; print(sys.platform)"
           $env:FX_EXECUTABLE = "C:\Program Files\Custom Firefox\firefox.exe"
           Start-Process -FilePath $env:FX_EXECUTABLE -ArgumentList "--version" -Wait -NoNewWindow
-          pipenv run python choose_ci_set.py
-          pipenv run pytest $(cat selected_tests)
+          pipenv run python choose_ci_set.py $env:MANIFEST
+          pipenv run pytest $(cat selected_tests) $env:MANIFEST_ARG
           $env:TEST_EXIT_CODE = $LASTEXITCODE
           mv artifacts artifacts-win || true
           exit $env:TEST_EXIT_CODE
@@ -119,8 +123,8 @@ jobs:
           rm ./pyproject.toml;
           mv ./ci_pyproject_headed.toml ./pyproject.toml;
           $env:FX_EXECUTABLE = "C:\Program Files\Custom Firefox\firefox.exe"
-          pipenv run python choose_ci_set.py
-          pipenv run pytest $(cat selected_tests)
+          pipenv run python choose_ci_set.py $env:MANIFEST
+          pipenv run pytest $(cat selected_tests) $env:MANIFEST_ARG
           $env:TEST_EXIT_CODE = $LASTEXITCODE
           rm artifacts/assets -r -Force
           Get-ChildItem -Path "artifacts" | ForEach-Object {
@@ -179,14 +183,19 @@ jobs:
           MANUAL_DOWNLOAD_LINK: ${{ inputs.mac_installer_link }}
         run:
           echo app_name=$(bash ./collect_executables.sh | xargs -0 ./parse_executables.sh) >> "$GITHUB_OUTPUT"
+      - name: Identify Manifests
+        if: ${{ inputs.mac_installer_link }}
+        run: |
+          echo "MANIFEST=incident_manifest.yaml" >> "$GITHUB_ENV";
+          echo "MANIFEST_ARG=\"--manifest incident_manifest.yaml" >> "$GITHUB_ENV";
       - name: Run Smoke Tests in MacOS
         if: steps.setup.conclusion == 'success'
         env:
           FX_EXECUTABLE: /Volumes/${{ steps.setup.outputs.app_name }}/${{ steps.setup.outputs.app_name }}.app/Contents/MacOS/firefox
         run: |
           "$FX_EXECUTABLE" --version
-          pipenv run python choose_ci_set.py
-          pipenv run pytest --fx-executable="$FX_EXECUTABLE"  $(cat selected_tests) || TEST_EXIT_CODE=$?
+          pipenv run python choose_ci_set.py $MANIFEST
+          pipenv run pytest --fx-executable="$FX_EXECUTABLE" $MANIFEST_ARG $(cat selected_tests) || TEST_EXIT_CODE=$?
           mv artifacts artifacts-mac || true
           exit $TEST_EXIT_CODE
       - name: Run Smoke Tests in MacOS (Headed)
@@ -196,8 +205,8 @@ jobs:
           REPORTABLE: true
         run: |
           mv ./ci_pyproject_headed.toml ./pyproject.toml;
-          pipenv run python choose_ci_set.py
-          pipenv run pytest --fx-executable="$FX_EXECUTABLE" $(cat selected_tests) || TEST_EXIT_CODE=$?
+          pipenv run python choose_ci_set.py $MANIFEST
+          pipenv run pytest --fx-executable="$FX_EXECUTABLE" $MANIFEST_ARG $(cat selected_tests) || TEST_EXIT_CODE=$?
           mv -n artifacts/* artifacts-mac/ || true
           exit $TEST_EXIT_CODE
       - name: Upload artifacts
@@ -239,24 +248,29 @@ jobs:
           ./collect_executables.sh
           ./firefox/firefox --version;
           #. ./keyring-unlock.sh;
+      - name: Identify Manifests
+        if: ${{ inputs.linux_tarball_link }}
+        run: |
+          echo "MANIFEST=incident_manifest.yaml" >> "$GITHUB_ENV";
+          echo "MANIFEST_ARG=\"--manifest incident_manifest.yaml" >> "$GITHUB_ENV";
       - name: Run Smoke Tests in Ubuntu
         if: steps.setup.conclusion == 'success'
         env:
           FX_EXECUTABLE: ./firefox/firefox
         run: |
           "$FX_EXECUTABLE" --version;
-          pipenv run python choose_ci_set.py;
+          pipenv run python choose_ci_set.py $MANIFEST;
           Xvfb :99 -screen 0 '1600x1200x24' > artifacts/xvfb.log &
-          DISPLAY=:99 pipenv run pytest --fx-executable="$FX_EXECUTABLE"  $(cat selected_tests) || TEST_EXIT_CODE=$?
+          DISPLAY=:99 pipenv run pytest --fx-executable="$FX_EXECUTABLE" $MANIFEST_ARG $(cat selected_tests) || TEST_EXIT_CODE=$?
           exit $TEST_EXIT_CODE
       - name: Run Smoke Tests in Ubuntu (Headed)
         if: steps.setup.conclusion == 'success' && always()
         env:
           FX_EXECUTABLE: ./firefox/firefox
         run: |
           mv ./ci_xvfb_pyproject_headed.toml ./pyproject.toml;
-          pipenv run python choose_ci_set.py
-          DISPLAY=:99 pipenv run pytest --fx-executable="$FX_EXECUTABLE" $(cat selected_tests) || TEST_EXIT_CODE=$?
+          pipenv run python choose_ci_set.py $MANIFEST
+          DISPLAY=:99 pipenv run pytest --fx-executable="$FX_EXECUTABLE" $MANIFEST_ARG $(cat selected_tests) || TEST_EXIT_CODE=$?
           exit $TEST_EXIT_CODE
 
   Use-Artifacts:

choose_ci_set.py

@@ -4,6 +4,7 @@
 from subprocess import check_output
 
 import pytest
+import yaml
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 CI_MARK = "@pytest.mark.ci"
@@ -105,7 +106,25 @@ def dedupe(run_list: list, slash: str) -> list:
     return run_list
 
 
+def parse_manifest(manifest_list, slash):
+    tests = []
+    for manifest_file in manifest_list:
+        if not manifest_file.endswith(".yaml"):
+            manifest_file = f"{manifest_file}.yaml"
+            with open(manifest_file) as fh:
+                manifest = yaml.safe_load(fh)
+                for suite, suite_manifest in manifest.items():
+                    for test in suite_manifest.keys():
+                        tests.append(slash.join(["tests", suite, test]))
+
+    with open(OUTPUT_FILE, "w") as fh:
+        fh.write("\n".join(tests))
+
+
 if __name__ == "__main__":
+    slash = "/" if "/" in SCRIPT_DIR else "\\"
+    if len(sys.argv) > 1:
+        parse_manifest(sys.argv[1:], slash)
     if os.path.exists(".env"):
         with open(".env") as fh:
             contents = fh.read()
@@ -120,8 +139,6 @@ def dedupe(run_list: list, slash: str) -> list:
             fh.write("tests")
             sys.exit(0)
 
-    slash = "/" if "/" in SCRIPT_DIR else "\\"
-
     re_obj = {
         "test_re_string": r".*/.*/test_.*\.py",
         "suite_conftest_re_string": r".*/.*/conftest\.py",

conftest.py

@@ -9,6 +9,7 @@
 from typing import Callable, List, Tuple, Union
 
 import pytest
+import yaml
 from PIL import Image, ImageGrab
 from selenium.common.exceptions import TimeoutException, WebDriverException
 from selenium.webdriver import Firefox
@@ -153,6 +154,13 @@ def pytest_addoption(parser):
         help="Size for Fx window, default is '1152x864'",
     )
 
+    parser.addoption(
+        "--manifest",
+        action="store",
+        default=None,
+        help="Location of test manifest yaml file",
+    )
+
 
 def _screenshot(filename: str, driver: Firefox, opt_ci: bool):
     if not filename.endswith(".png"):
@@ -196,6 +204,11 @@ def opt_headless(request):
     return request.config.getoption("--run-headless")
 
 
+@pytest.fixture(scope="session")
+def opt_manifest(request):
+    return request.config.getoption("--manifest")
+
+
 @pytest.fixture()
 def opt_implicit_timeout(request):
     return request.config.getoption("--implicit-timeout")
@@ -389,6 +402,36 @@ def hard_quit():
     return False
 
 
+@pytest.fixture(scope="session")
+def read_manifest(opt_manifest, sys_platform):
+    syskey = {"Windows": "win", "Darwin": "mac", "Linux": "linux"}
+    if not opt_manifest:
+        return None
+    with open(opt_manifest) as fh:
+        manifest = yaml.safe_load(fh)
+    for suite in manifest:
+        result = None
+        for entry in suite:
+            if isinstance(manifest[suite][entry], dict):
+                result = manifest[suite][entry][syskey[sys_platform]]
+            else:
+                result = manifest[suite][entry]
+            if result == "unstable":
+                os.environ[f"SKIP_{entry}"] = True
+            elif result == "fail":
+                os.environ[f"XFAIL_{entry}"] = True
+
+
+@pytest.fixture()
+def modtests(request, read_manifest):
+    filename = os.path.split(request.module.__file__)[-1]
+    entry_name = filename.rsplit(".", 1)[0]
+    if os.environ.get(f"XFAIL_{entry_name}"):
+        request.node.add_marker(
+            pytest.mark.xfail(reason="Test was marked xfail in manifest.")
+        )
+
+
 @pytest.fixture(autouse=True)
 def driver(
     fx_executable: str,
@@ -408,6 +451,7 @@ def driver(
     json_metadata,
     hard_quit,
     create_profiles,
+    modtests,
 ):
     """
     Return the webdriver object.