Merge branch 'ci/fixes'

Author: lucasssvaz

.github/scripts/runtime_table_generator.py

@@ -1,6 +1,7 @@
 import json
 import sys
 import os
+import re
 from datetime import datetime
 
 SUCCESS_SYMBOL = ":white_check_mark:"
@@ -12,16 +13,206 @@
     data = json.load(f)
     tests = sorted(data["stats"]["suite_details"], key=lambda x: x["name"])
 
+def _safe_env_list(name: str):
+    """Parse list-like env variables that look like ['esp32','esp32s2'].
+    Accepts empty/None and returns a Python list of strings.
+    """
+    raw = os.environ.get(name, "").strip()
+    if not raw:
+        return []
+    # Normalize to JSON by replacing single quotes with double quotes
+    try:
+        normalized = raw.replace("'", '"')
+        return json.loads(normalized)
+    except Exception:
+        # Fallback: split on commas and strip brackets
+        raw = raw.strip("[]")
+        return [p.strip().strip("'\"") for p in raw.split(",") if p.strip()]
+
+
+def _scan_artifacts_for_tests():
+    """Discover built tests from downloaded artifacts.
+    Returns mapping test_name -> set(targets) for which a build*.tmp dir exists containing ci.yml (or even without).
+    """
+    results = {}
+    root = os.path.join(os.getcwd(), "runtime-test-results", "build_artifacts")
+    if not os.path.isdir(root):
+        return results
+    for dirpath, dirnames, filenames in os.walk(root):
+        # Match .../<target>/<test_name>/buildX.tmp
+        if not re.search(r"build\d*\.tmp$", dirpath):
+            continue
+        parts = dirpath.split(os.sep)
+        if len(parts) < 3:
+            continue
+        test_name = parts[-2]
+        target = parts[-3]
+        results.setdefault(test_name, set()).add(target)
+    return results
+
+
+def _read_local_ci_yaml_if_available(test_name: str) -> str:
+    """Try to find a ci.yml stored beside compiled binaries downloaded by results workflow.
+    Expected paths: runtime-test-results/build_artifacts/**/build*.tmp/ci.yml with sketch folder named test_name.
+    """
+    # Walk known root
+    root = os.path.join(os.getcwd(), "runtime-test-results", "build_artifacts")
+    if not os.path.isdir(root):
+        return ""
+    # Traverse limited depth for performance
+    for dirpath, dirnames, filenames in os.walk(root):
+        # Quick filter: folder path must contain the test folder name and a build*.tmp
+        if test_name not in dirpath:
+            continue
+        if not re.search(r"build\d*\.tmp", dirpath):
+            continue
+        if "ci.yml" in filenames:
+            try:
+                with open(os.path.join(dirpath, "ci.yml"), "r") as f:
+                    return f.read()
+            except Exception:
+                continue
+    return ""
+
+
+def _parse_ci_yml_minimal(content: str):
+    """Very small YAML reader for the fields we need: targets, platforms, requires, requires_any.
+    Returns dict: {targets: {t: bool}, platforms: {p: bool}, requires: [str], requires_any: [str]}
+    """
+    parsed = {"targets": {}, "platforms": {}, "requires": [], "requires_any": []}
+    if not content:
+        return parsed
+
+    lines = content.splitlines()
+    current = None
+    for line in lines:
+        # Remove comments and trailing spaces
+        line = line.split("#", 1)[0].rstrip()
+        if not line.strip():
+            continue
+        # Top-level keys
+        if not line.startswith(" "):
+            if line.startswith("targets:"):
+                current = "targets"
+                continue
+            if line.startswith("platforms:"):
+                current = "platforms"
+                continue
+            if line.startswith("requires:"):
+                current = "requires"
+                continue
+            if line.startswith("requires_any:"):
+                current = "requires_any"
+                continue
+            current = None
+            continue
+
+        # Nested under a current section (2+ spaces)
+        if current in ("targets", "platforms"):
+            m = re.match(r"\s*([A-Za-z0-9_\-]+)\s*:\s*(\w+)", line)
+            if m:
+                key = m.group(1)
+                val = m.group(2).lower()
+                parsed[current][key] = (val != "false" and val != "null")
+        elif current in ("requires", "requires_any"):
+            m = re.match(r"\s*-[\s]*(.+)$", line)
+            if m:
+                parsed[current].append(m.group(1).strip())
+
+    return parsed
+
+
+def _read_local_sdkconfig_for_target(target: str) -> str:
+    """Find sdkconfig for target inside downloaded build artifacts.
+    Searches any build*.tmp directory under .arduino/tests/<target>/<test>/build*.tmp
+    Returns first found content.
+    """
+    root = os.path.join(os.getcwd(), "runtime-test-results", "build_artifacts")
+    if not os.path.isdir(root):
+        return ""
+    for dirpath, dirnames, filenames in os.walk(root):
+        if target not in dirpath:
+            continue
+        if not re.search(r"build\d*\.tmp$", dirpath):
+            continue
+        if "sdkconfig" in filenames:
+            try:
+                with open(os.path.join(dirpath, "sdkconfig"), "r") as f:
+                    return f.read()
+            except Exception:
+                continue
+    return ""
+
+
+def _meets_requirements(ci_cfg: dict, sdkconfig_text: str) -> bool:
+    # Missing files -> assume requirements met (match CI behavior to build and fail later if needed)
+    if not sdkconfig_text:
+        return True
+    # AND list
+    for req in ci_cfg.get("requires", []):
+        req = req.strip()
+        if not req:
+            continue
+        if not any(line.startswith(req) for line in sdkconfig_text.splitlines()):
+            return False
+    # OR list
+    req_any = ci_cfg.get("requires_any", [])
+    if req_any:
+        if not any(any(line.startswith(r.strip()) for line in sdkconfig_text.splitlines()) for r in req_any):
+            return False
+    return True
+
+
+def _parse_fqbn_counts(content: str):
+    """Parse ci.yml 'fqbn' section and count entries per target.
+    Returns dict target -> int
+    """
+    counts = {}
+    if not content:
+        return counts
+    lines = content.splitlines()
+    in_fqbn = False
+    current_target = None
+    for line in lines:
+        raw = line.split("#", 1)[0].rstrip()
+        if not raw:
+            continue
+        if not raw.startswith(" "):
+            # top-level key
+            if raw.startswith("fqbn:"):
+                in_fqbn = True
+                current_target = None
+                continue
+            else:
+                if in_fqbn:
+                    # leaving fqbn section
+                    break
+                continue
+        if not in_fqbn:
+            continue
+        # two-space indent target: line
+        m_target = re.match(r"\s{2}([A-Za-z0-9_\-]+)\s*:\s*", raw)
+        if m_target:
+            current_target = m_target.group(1)
+            counts.setdefault(current_target, 0)
+            continue
+        # four-space list item under current target
+        if current_target:
+            if re.match(r"\s{4}-\s", raw):
+                counts[current_target] = counts.get(current_target, 0) + 1
+    return counts
+
+
 # Get commit SHA from command line argument or environment variable
 commit_sha = None
 if len(sys.argv) < 2 or len(sys.argv) > 3:
     print(f"Usage: python {sys.argv[0]} <test_results.json> [commit_sha]", file=sys.stderr)
     sys.exit(1)
-elif len(sys.argv) == 3: # Commit SHA is provided as argument
+elif len(sys.argv) == 3:  # Commit SHA is provided as argument
     commit_sha = sys.argv[2]
-elif "GITHUB_SHA" in os.environ: # Commit SHA is provided as environment variable
+elif "GITHUB_SHA" in os.environ:  # Commit SHA is provided as environment variable
     commit_sha = os.environ["GITHUB_SHA"]
-else: # Commit SHA is not provided
+else:  # Commit SHA is not provided
     print("Commit SHA is not provided. Please provide it as an argument or set the GITHUB_SHA environment variable.", file=sys.stderr)
     sys.exit(1)
 
@@ -45,12 +236,23 @@
 proc_test_data = {}
 target_list = []
 
+# Build executed tests map and collect targets
+executed_tests_index = {}  # {(platform, target, test_name): {tests, failures, errors}}
+executed_run_counts = {}   # {(platform, target, test_name): int}
+
 for test in tests:
     if test["name"].startswith("performance_"):
         continue
 
-    _, platform, target, test_name = test["name"].split("_", 3)
-    test_name = test_name[:-1]
+    try:
+        test_type, platform, target, rest = test["name"].split("_", 3)
+    except ValueError:
+        # Unexpected name, skip
+        continue
+
+    # Remove an optional trailing numeric index (multi-FQBN builds)
+    m = re.match(r"(.+?)(\d+)?$", rest)
+    test_name = m.group(1) if m else rest
 
     if target not in target_list:
         target_list.append(target)
@@ -72,9 +274,70 @@
     proc_test_data[platform][test_name][target]["failures"] += test["failures"]
     proc_test_data[platform][test_name][target]["errors"] += test["errors"]
 
+    executed_tests_index[(platform, target, test_name)] = proc_test_data[platform][test_name][target]
+    executed_run_counts[(platform, target, test_name)] = executed_run_counts.get((platform, target, test_name), 0) + 1
+
 target_list = sorted(target_list)
 
-for platform in proc_test_data:
+# Determine expected tests from repo, ci.yml and sdkconfig for validation type
+repo = os.environ.get("GITHUB_REPOSITORY", "")
+hw_enabled = os.environ.get("HW_TESTS_ENABLED", "false").lower() == "true"
+wokwi_enabled = os.environ.get("WOKWI_TESTS_ENABLED", "false").lower() == "true"
+qemu_enabled = os.environ.get("QEMU_TESTS_ENABLED", "false").lower() == "true"
+
+hw_targets = _safe_env_list("HW_TARGETS") if hw_enabled else []
+wokwi_targets = _safe_env_list("WOKWI_TARGETS") if wokwi_enabled else []
+qemu_targets = _safe_env_list("QEMU_TARGETS") if qemu_enabled else []
+
+# Only consider validation tests in this report
+expected_by_platform = {"hardware": {}, "wokwi": {}, "qemu": {}}
+expected_counts_by_key = {}  # {(platform, target, test_name): int}
+try:
+    built_tests = _scan_artifacts_for_tests()
+    # Prefetch sdkconfigs per target
+    sdk_by_target = {}
+    all_needed_targets = sorted(set(hw_targets + wokwi_targets + qemu_targets))
+    for t in all_needed_targets:
+        sdk_by_target[t] = _read_local_sdkconfig_for_target(t)
+
+    for test_name, test_targets in built_tests.items():
+        # Load ci.yml if exists
+        ci_content = _read_local_ci_yaml_if_available(test_name)
+        ci = _parse_ci_yml_minimal(ci_content)
+        fqbn_counts = _parse_fqbn_counts(ci_content)
+
+        # Evaluate for each platform
+        for platform, targets in (("hardware", hw_targets), ("wokwi", wokwi_targets), ("qemu", qemu_targets)):
+            if not targets:
+                continue
+            # platforms.<platform> == false -> skip
+            if platform in ci.get("platforms", {}) and ci["platforms"].get(platform) is False:
+                continue
+            # Only consider targets that were actually built (we don't fetch repo)
+            for target in [t for t in targets if t in test_targets]:
+                # targets.<target> == false -> skip
+                if target in ci.get("targets", {}) and ci["targets"].get(target) is False:
+                    continue
+                # requirements
+                if not _meets_requirements(ci, sdk_by_target.get(target, "")):
+                    continue
+                expected_by_platform.setdefault(platform, {}).setdefault(test_name, set()).add(target)
+                # Determine expected run count by number of FQBN variants (default 1)
+                expected_runs = fqbn_counts.get(target, 0)
+                if expected_runs <= 0:
+                    expected_runs = 1
+                expected_counts_by_key[(platform, target, test_name)] = expected_runs
+except Exception as e:
+    print(f"WARN: Failed to compute expected tests: {e}", file=sys.stderr)
+
+# Ensure target_list also includes expected targets even if nothing executed
+for t in hw_targets + wokwi_targets + qemu_targets:
+    if t not in target_list:
+        target_list.append(t)
+target_list = sorted(set(target_list))
+
+platforms_to_render = set(proc_test_data.keys()) | {p for p, mp in expected_by_platform.items() if any(mp.values())}
+for platform in platforms_to_render:
     print("")
     print(f"#### {platform.capitalize()}")
     print("")
@@ -90,19 +353,37 @@
     print("")
     print("-" + "|:-:" * len(target_list))
 
-    for test_name, targets in proc_test_data[platform].items():
+    # Determine all test names to render for this platform: executed ∪ expected
+    platform_executed = proc_test_data.get(platform, {})
+    platform_expected = expected_by_platform.get(platform, {})
+    all_test_names = sorted(set(platform_executed.keys()) | set(platform_expected.keys()))
+
+    for test_name in all_test_names:
         print(f"{test_name}", end="")
         for target in target_list:
-            if target in targets:
-                test_data = targets[target]
-                if test_data["errors"] > 0:
+            # Executed?
+            executed_cell = platform_executed.get(test_name, {}).get(target)
+            exp_runs = expected_counts_by_key.get((platform, target, test_name), 0)
+            exec_runs = executed_run_counts.get((platform, target, test_name), 0)
+            if executed_cell:
+                # If fewer runs executed than expected, mark as error
+                if exp_runs > 0 and exec_runs < exp_runs:
+                    print(f"|Error {ERROR_SYMBOL}", end="")
+                    continue
+                if executed_cell["errors"] > 0:
                     print(f"|Error {ERROR_SYMBOL}", end="")
                 else:
-                    print(f"|{test_data['total']-test_data['failures']}/{test_data['total']}", end="")
-                    if test_data["failures"] > 0:
+                    print(f"|{executed_cell['total']-executed_cell['failures']}/{executed_cell['total']}", end="")
+                    if executed_cell["failures"] > 0:
                         print(f" {FAILURE_SYMBOL}", end="")
                     else:
                         print(f" {SUCCESS_SYMBOL}", end="")
+                continue
+
+            # Not executed. Was it expected?
+            expected_targets = platform_expected.get(test_name, set())
+            if target in expected_targets:
+                print(f"|Error {ERROR_SYMBOL}", end="")
             else:
                 print("|-", end="")
         print("")

.github/scripts/sketch_utils.sh

@@ -294,6 +294,11 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
                 exit "$exit_status"
             fi
 
+            # Copy ci.yml alongside compiled binaries for later consumption by reporting tools
+            if [ -f "$sketchdir/ci.yml" ]; then
+                cp -f "$sketchdir/ci.yml" "$build_dir/ci.yml" 2>/dev/null || true
+            fi
+
             if [ -n "$log_compilation" ]; then
                 #Extract the program storage space and dynamic memory usage in bytes and percentage in separate variables from the output, just the value without the string
                 flash_bytes=$(grep -oE 'Sketch uses ([0-9]+) bytes' "$output_file" | awk '{print $3}')
@@ -337,6 +342,10 @@ function build_sketch { # build_sketch <ide_path> <user_path> <path-to-ino> [ext
                 echo "ERROR: Compilation failed with error code $exit_status"
                 exit $exit_status
             fi
+            # Copy ci.yml alongside compiled binaries for later consumption by reporting tools
+            if [ -f "$sketchdir/ci.yml" ]; then
+                cp -f "$sketchdir/ci.yml" "$build_dir/ci.yml" 2>/dev/null || true
+            fi
             # $ide_path/arduino-builder -compile -logger=human -core-api-version=10810 \
             #     -fqbn=\"$currfqbn\" \
             #     -warnings="all" \