From a3f497f8ba812522274c007dcebb11ba1ff72b5c Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 2 Jul 2026 15:37:07 -0700 Subject: [PATCH 1/4] Add test counts to CI summary --- .github/scripts/tox-summary.py | 40 ++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/scripts/tox-summary.py b/.github/scripts/tox-summary.py index 618e168dbc..94a6905df2 100755 --- a/.github/scripts/tox-summary.py +++ b/.github/scripts/tox-summary.py @@ -25,6 +25,10 @@ RESULTS_FILE_RE = re.compile( r"(?P[a-zA-Z0-9_-]+)-(?P\d+)-(?P[a-zA-Z0-9]+)-(?P[a-zA-Z0-9_-]+)-results.json" ) +ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-9;]*[a-zA-Z]") +PYTEST_SUMMARY_RE = re.compile(r"=+ (?P.+?) in (?P[\d.]+)s =+") +PYTEST_COUNT_RE = re.compile(r"(\d+) (passed|failed|skipped|xfailed|xpassed|errors?|warnings?|deselected|rerun)") +PYTEST_COUNT_NORMALIZE = {"errors": "error", "warnings": "warning"} GITHUB_SERVER_URL = os.environ.get("GITHUB_SERVER_URL", "https://github.com") GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "newrelic/newrelic-python-agent") @@ -32,8 +36,12 @@ TABLE_HEADER = """ # Tox Results Summary -| Environment | Status | Duration (s) | Setup Duration (s) | Test Duration (s) | Runner | -|-------------|--------|--------------|--------------------|-------------------|--------| +| Total Passed | Total Failed | +|--------------|--------------| +| {total_passed} | {total_failed} | + +| Environment | Status | Passed | Failed | Duration (s) | Setup Duration (s) | Test Duration (s) | Runner | +|-------------|--------|--------|--------|--------------|--------------------|-------------------|--------| """ TABLE_HEADER = dedent(TABLE_HEADER).strip() @@ -67,11 +75,13 @@ def main(): with GITHUB_SUMMARY.open("w") as output_fp: summary = summarize_results(results) + total_passed = sum(r["passed"] for r in summary) + total_failed = sum(r["failed"] for r in summary) # Print table header - print(TABLE_HEADER, file=output_fp) + print(TABLE_HEADER.format(total_passed=total_passed, total_failed=total_failed), file=output_fp) for result in summary: - line = "| {env_name} | {status} | {duration} | {setup_duration} | {test_duration} | {runner} |".format( + line = "| {env_name} | {status} | {passed} | {failed} | {duration} | {setup_duration} | {test_duration} | {runner} |".format( **result ) print(line, file=output_fp) @@ -96,6 +106,26 @@ def summarize_results(results): test_duration += cmd.get("elapsed", 0) test_duration = f"{test_duration:.2f}" if test_duration >= 0 else "N/A" + # Get test counts from test run + passed, failed = 0, 0 + try: + # Remove ANSI color control characters + raw_output = ANSI_ESCAPE_RE.sub("", result["test"][0]["output"]) + # Read backwards through the output since the summary is at the bottom + output = reversed(list(raw_output.splitlines())) + for line in output: + if match := PYTEST_SUMMARY_RE.match(line): + counts = { + PYTEST_COUNT_NORMALIZE.get(name, name): int(num) + for num, name in PYTEST_COUNT_RE.findall(match.group("summary")) + } + passed = counts.get("passed", 0) + counts.get("xfailed", 0) + failed = counts.get("failed", 0) + counts.get("xpassed", 0) + break + + except Exception: + pass + summary.append( { "env_name": env, @@ -103,6 +133,8 @@ def summarize_results(results): "duration": duration, "setup_duration": setup_duration, "test_duration": test_duration, + "passed": passed, + "failed": failed, "runner": runner, } ) From 666a4cd591ddf79bcf8716da3614230c3773ea1b Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 2 Jul 2026 16:02:52 -0700 Subject: [PATCH 2/4] Separate other failure modes --- .github/scripts/tox-summary.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/scripts/tox-summary.py b/.github/scripts/tox-summary.py index 94a6905df2..f975ba0138 100755 --- a/.github/scripts/tox-summary.py +++ b/.github/scripts/tox-summary.py @@ -28,7 +28,7 @@ ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-9;]*[a-zA-Z]") PYTEST_SUMMARY_RE = re.compile(r"=+ (?P.+?) in (?P[\d.]+)s =+") PYTEST_COUNT_RE = re.compile(r"(\d+) (passed|failed|skipped|xfailed|xpassed|errors?|warnings?|deselected|rerun)") -PYTEST_COUNT_NORMALIZE = {"errors": "error", "warnings": "warning"} +PYTEST_COUNT_NORMALIZE = {"error": "errors", "warning": "warnings"} GITHUB_SERVER_URL = os.environ.get("GITHUB_SERVER_URL", "https://github.com") GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "newrelic/newrelic-python-agent") @@ -36,12 +36,12 @@ TABLE_HEADER = """ # Tox Results Summary -| Total Passed | Total Failed | -|--------------|--------------| -| {total_passed} | {total_failed} | +| Total Passed | Total Failed | Total XFailed | Total XPassed | Total Errors | Total Warnings | +|--------------|--------------|---------------|---------------|--------------|----------------| +| {total_passed} | {total_failed} | {total_xfailed} | {total_xpassed} | {total_errors} | {total_warnings} | -| Environment | Status | Passed | Failed | Duration (s) | Setup Duration (s) | Test Duration (s) | Runner | -|-------------|--------|--------|--------|--------------|--------------------|-------------------|--------| +| Environment | Status | Duration (s) | Setup Duration (s) | Test Duration (s) | Runner | Passed | Failed | XFailed | XPassed | Errors | Warnings | +|-------------|--------|--------------|--------------------|-------------------|--------|--------|--------|---------|---------|--------|----------| """ TABLE_HEADER = dedent(TABLE_HEADER).strip() @@ -75,13 +75,15 @@ def main(): with GITHUB_SUMMARY.open("w") as output_fp: summary = summarize_results(results) - total_passed = sum(r["passed"] for r in summary) - total_failed = sum(r["failed"] for r in summary) + totals = { + f"total_{key}": sum(r[key] for r in summary) + for key in ("passed", "failed", "xfailed", "xpassed", "errors", "warnings") + } # Print table header - print(TABLE_HEADER.format(total_passed=total_passed, total_failed=total_failed), file=output_fp) + print(TABLE_HEADER.format(**totals), file=output_fp) for result in summary: - line = "| {env_name} | {status} | {passed} | {failed} | {duration} | {setup_duration} | {test_duration} | {runner} |".format( + line = "| {env_name} | {status} | {duration} | {setup_duration} | {test_duration} | {runner} | {passed} | {failed} | {xfailed} | {xpassed} | {errors} | {warnings} |".format( **result ) print(line, file=output_fp) @@ -107,7 +109,7 @@ def summarize_results(results): test_duration = f"{test_duration:.2f}" if test_duration >= 0 else "N/A" # Get test counts from test run - passed, failed = 0, 0 + counts = {} try: # Remove ANSI color control characters raw_output = ANSI_ESCAPE_RE.sub("", result["test"][0]["output"]) @@ -119,8 +121,6 @@ def summarize_results(results): PYTEST_COUNT_NORMALIZE.get(name, name): int(num) for num, name in PYTEST_COUNT_RE.findall(match.group("summary")) } - passed = counts.get("passed", 0) + counts.get("xfailed", 0) - failed = counts.get("failed", 0) + counts.get("xpassed", 0) break except Exception: @@ -133,8 +133,12 @@ def summarize_results(results): "duration": duration, "setup_duration": setup_duration, "test_duration": test_duration, - "passed": passed, - "failed": failed, + "passed": counts.get("passed", 0), + "failed": counts.get("failed", 0), + "xfailed": counts.get("xfailed", 0), + "xpassed": counts.get("xpassed", 0), + "errors": counts.get("errors", 0), + "warnings": counts.get("warnings", 0), "runner": runner, } ) From c730f73c4dcbf460b1fe4be8f0b57d9e91c15164 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 9 Jul 2026 15:24:55 -0700 Subject: [PATCH 3/4] Tweak to fix 0 counts --- .github/scripts/tox-summary.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/scripts/tox-summary.py b/.github/scripts/tox-summary.py index f975ba0138..350e0605e2 100755 --- a/.github/scripts/tox-summary.py +++ b/.github/scripts/tox-summary.py @@ -26,9 +26,10 @@ r"(?P[a-zA-Z0-9_-]+)-(?P\d+)-(?P[a-zA-Z0-9]+)-(?P[a-zA-Z0-9_-]+)-results.json" ) ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-9;]*[a-zA-Z]") -PYTEST_SUMMARY_RE = re.compile(r"=+ (?P.+?) in (?P[\d.]+)s =+") +PYTEST_SUMMARY_RE = re.compile(r"=+ (?P.+?) in (?P[\d.]+)s(?: \([\d:]+\))? =+") PYTEST_COUNT_RE = re.compile(r"(\d+) (passed|failed|skipped|xfailed|xpassed|errors?|warnings?|deselected|rerun)") PYTEST_COUNT_NORMALIZE = {"error": "errors", "warning": "warnings"} +COUNT_KEYS = ("passed", "failed", "xfailed", "xpassed", "errors", "warnings") GITHUB_SERVER_URL = os.environ.get("GITHUB_SERVER_URL", "https://github.com") GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "newrelic/newrelic-python-agent") @@ -75,16 +76,18 @@ def main(): with GITHUB_SUMMARY.open("w") as output_fp: summary = summarize_results(results) - totals = { - f"total_{key}": sum(r[key] for r in summary) - for key in ("passed", "failed", "xfailed", "xpassed", "errors", "warnings") - } + totals = {f"total_{key}": sum(r[key] for r in summary) for key in COUNT_KEYS} # Print table header print(TABLE_HEADER.format(**totals), file=output_fp) for result in summary: + # Print "-" for counts we couldn't parse to distinguish from 0 counts + row = dict(result) + if not row["parsed"]: + for key in COUNT_KEYS: + row[key] = "-" line = "| {env_name} | {status} | {duration} | {setup_duration} | {test_duration} | {runner} | {passed} | {failed} | {xfailed} | {xpassed} | {errors} | {warnings} |".format( - **result + **row ) print(line, file=output_fp) @@ -140,6 +143,7 @@ def summarize_results(results): "errors": counts.get("errors", 0), "warnings": counts.get("warnings", 0), "runner": runner, + "parsed": bool(counts), # If counts is still empty, this failed to parse } ) From 3e34fa0d5ca92d043da164d7b5b2dbc8d62e214c Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 9 Jul 2026 17:02:17 -0700 Subject: [PATCH 4/4] Tweak implementation again --- .github/scripts/tox-summary.py | 38 ++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/scripts/tox-summary.py b/.github/scripts/tox-summary.py index 350e0605e2..1aae6ea5b7 100755 --- a/.github/scripts/tox-summary.py +++ b/.github/scripts/tox-summary.py @@ -112,22 +112,7 @@ def summarize_results(results): test_duration = f"{test_duration:.2f}" if test_duration >= 0 else "N/A" # Get test counts from test run - counts = {} - try: - # Remove ANSI color control characters - raw_output = ANSI_ESCAPE_RE.sub("", result["test"][0]["output"]) - # Read backwards through the output since the summary is at the bottom - output = reversed(list(raw_output.splitlines())) - for line in output: - if match := PYTEST_SUMMARY_RE.match(line): - counts = { - PYTEST_COUNT_NORMALIZE.get(name, name): int(num) - for num, name in PYTEST_COUNT_RE.findall(match.group("summary")) - } - break - - except Exception: - pass + counts = extract_pytest_counts(result) summary.append( { @@ -150,5 +135,26 @@ def summarize_results(results): return sorted(summary, key=lambda result: (1 if "OK" in result["status"] else 0, result["env_name"])) +def extract_pytest_counts(result): + # Trace through each tox command run backwards to find the pytest summary + for command_results in reversed(result.get("test", [])): + try: + raw_output = command_results.get("output", "") + # Remove ANSI color control characters + uncolored_output = ANSI_ESCAPE_RE.sub("", raw_output) + # Read backwards through the output since the summary is at the bottom + output = reversed(list(uncolored_output.splitlines())) + for line in output: + if match := PYTEST_SUMMARY_RE.match(line): + return { + PYTEST_COUNT_NORMALIZE.get(name, name): int(num) + for num, name in PYTEST_COUNT_RE.findall(match.group("summary")) + } + except Exception: + pass + + return {} + + if __name__ == "__main__": main()