Skip to content

Commit d05ad25

Browse files
siddharth-khatsuriyasiddharth-khatsuriya
authored
fix: Resolve discrepancies in test outcome counting (#432)
### Description Currently for test report if there are multiple blocks in the testcase tag (i.e error + failure) then it will count both and resulting summary may not align correctly. This PR updates the logic for report summary. ### Checklist - [ ] `README.md` has been updated or is not required - [ ] push trigger tests - [ ] manual release test - [ ] automated releases test - [ ] pull request trigger tests - [ ] schedule trigger tests - [ ] workflow errors/warnings reviewed and addressed ### Testing done o365: https://github.com/splunk/splunk-add-on-for-microsoft-office-365/actions/runs/16805102170?pr=858 --------- ### Description (PR description goes here) ### Checklist - [ ] `README.md` has been updated or is not required - [ ] push trigger tests - [ ] manual release test - [ ] automated releases test - [ ] pull request trigger tests - [ ] schedule trigger tests - [ ] workflow errors/warnings reviewed and addressed ### Testing done (for each selected checkbox, the corresponding test results link should be listed here)
2 parents 58c1d31 + 02fb5a5 commit d05ad25

File tree

1 file changed

+10
-10
lines changed

1 file changed

+10
-10
lines changed

‎.github/workflows/reusable-build-test-release.yml‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -485,7 +485,7 @@ jobs:
485485
if [ -n "$junit_xml_file" ]; then
486486
total_tests=$(xmllint --xpath "count(//testcase)" "$junit_xml_file")
487487
failures=$(xmllint --xpath "count(//testcase[failure])" "$junit_xml_file")
488-
errors=$(xmllint --xpath "count(//testcase[error])" "$junit_xml_file")
488+
errors=$(xmllint --xpath "count(//testcase[error and not(failure)])" "$junit_xml_file")
489489
skipped=$(xmllint --xpath "count(//testcase[skipped])" "$junit_xml_file")
490490
passed=$((total_tests - failures - errors - skipped))
491491
echo -e "| Total Tests | Passed Tests | Failed Tests | Errored Tests | Skipped Tests |\n| ----------- | ------------ | ------------ | ------------- | ------------- |\n| $total_tests | $passed | $failures | $errors | $skipped |" >> "$GITHUB_STEP_SUMMARY"
@@ -1342,7 +1342,7 @@ jobs:
13421342
if [ -n "$junit_xml_file" ]; then
13431343
total_tests=$(xmllint --xpath "count(//testcase)" "$junit_xml_file")
13441344
failures=$(xmllint --xpath "count(//testcase[failure])" "$junit_xml_file")
1345-
errors=$(xmllint --xpath "count(//testcase[error])" "$junit_xml_file")
1345+
errors=$(xmllint --xpath "count(//testcase[error and not(failure)])" "$junit_xml_file")
13461346
skipped=$(xmllint --xpath "count(//testcase[skipped])" "$junit_xml_file")
13471347
passed=$((total_tests - failures - errors - skipped))
13481348
echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} |$total_tests |$passed |$failures |$errors | $skipped |${{steps.test_report.outputs.url_html}}" > job_summary.txt
@@ -1621,7 +1621,7 @@ jobs:
16211621
if [ -n "$junit_xml_file" ]; then
16221622
total_tests=$(xmllint --xpath "count(//testcase)" "$junit_xml_file")
16231623
failures=$(xmllint --xpath "count(//testcase[failure])" "$junit_xml_file")
1624-
errors=$(xmllint --xpath "count(//testcase[error])" "$junit_xml_file")
1624+
errors=$(xmllint --xpath "count(//testcase[error and not(failure)])" "$junit_xml_file")
16251625
skipped=$(xmllint --xpath "count(//testcase[skipped])" "$junit_xml_file")
16261626
passed=$((total_tests - failures - errors - skipped))
16271627
echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.browser }} ${{ matrix.vendor-version.image }} ${{ matrix.marker }} |$total_tests |$passed |$failures |$errors |$skipped |${{steps.test_report.outputs.url_html}}" > job_summary.txt
@@ -1898,7 +1898,7 @@ jobs:
18981898
if [ -n "$junit_xml_file" ]; then
18991899
total_tests=$(xmllint --xpath "count(//testcase)" "$junit_xml_file")
19001900
failures=$(xmllint --xpath "count(//testcase[failure])" "$junit_xml_file")
1901-
errors=$(xmllint --xpath "count(//testcase[error])" "$junit_xml_file")
1901+
errors=$(xmllint --xpath "count(//testcase[error and not(failure)])" "$junit_xml_file")
19021902
skipped=$(xmllint --xpath "count(//testcase[skipped])" "$junit_xml_file")
19031903
passed=$((total_tests - failures - errors - skipped))
19041904
echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.marker }} ${{ matrix.vendor-version.image }} |$total_tests |$passed |$failures |$errors | $skipped |${{steps.test_report.outputs.url_html}}" > job_summary.txt
@@ -2174,7 +2174,7 @@ jobs:
21742174
if [ -n "$junit_xml_file" ]; then
21752175
total_tests=$(xmllint --xpath "count(//testcase)" "$junit_xml_file")
21762176
failures=$(xmllint --xpath "count(//testcase[failure])" "$junit_xml_file")
2177-
errors=$(xmllint --xpath "count(//testcase[error])" "$junit_xml_file")
2177+
errors=$(xmllint --xpath "count(//testcase[error and not(failure)])" "$junit_xml_file")
21782178
skipped=$(xmllint --xpath "count(//testcase[skipped])" "$junit_xml_file")
21792179
passed=$((total_tests - failures - errors - skipped))
21802180
echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.marker }} ${{ matrix.vendor-version.image }} |$total_tests |$passed |$failures |$errors | $skipped |${{steps.test_report.outputs.url_html}}" > job_summary.txt
@@ -2439,7 +2439,7 @@ jobs:
24392439
if [ -n "$junit_xml_file" ]; then
24402440
total_tests=$(xmllint --xpath "count(//testcase)" "$junit_xml_file")
24412441
failures=$(xmllint --xpath "count(//testcase[failure])" "$junit_xml_file")
2442-
errors=$(xmllint --xpath "count(//testcase[error])" "$junit_xml_file")
2442+
errors=$(xmllint --xpath "count(//testcase[error and not(failure)])" "$junit_xml_file")
24432443
skipped=$(xmllint --xpath "count(//testcase[skipped])" "$junit_xml_file")
24442444
passed=$((total_tests - failures - errors - skipped))
24452445
echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ matrix.ta-version-from-upgrade }} ${{ matrix.vendor-version.image }} |$total_tests |$passed |$failures |$errors | $skipped |${{steps.test_report.outputs.url_html}}" > job_summary.txt
@@ -2707,10 +2707,10 @@ jobs:
27072707
junit_xml_path="${{ needs.setup.outputs.directory-path }}/test-results"
27082708
junit_xml_file=$(find "$junit_xml_path" -name "*.xml" -type f 2>/dev/null | head -n 1)
27092709
if [ -n "$junit_xml_file" ]; then
2710-
total_tests=$(xmllint --xpath 'sum(//testsuite/@tests)' "$junit_xml_file")
2711-
failures=$(xmllint --xpath 'sum(//testsuite/@failures)' "$junit_xml_file")
2712-
errors=$(xmllint --xpath 'sum(//testsuite/@errors)' "$junit_xml_file")
2713-
skipped=$(xmllint --xpath 'sum(//testsuite/@skipped)' "$junit_xml_file")
2710+
total_tests=$(xmllint --xpath "count(//testcase)" "$junit_xml_file")
2711+
failures=$(xmllint --xpath "count(//testcase[failure])" "$junit_xml_file")
2712+
errors=$(xmllint --xpath "count(//testcase[error and not(failure)])" "$junit_xml_file")
2713+
skipped=$(xmllint --xpath "count(//testcase[skipped])" "$junit_xml_file")
27142714
passed=$((total_tests - failures - errors - skipped))
27152715
echo "splunk ${{ matrix.splunk.version }}${{ secrets.OTHER_TA_REQUIRED_CONFIGS }} ${{ steps.os-name-version.outputs.os-name }} ${{ steps.os-name-version.outputs.os-version }} |$total_tests |$passed |$failures |$errors |$skipped |${{steps.test_report.outputs.url_html}}" > job_summary.txt
27162716
else

0 commit comments

Comments
 (0)