From cb43779b9912097d7bb9329b1ea19a10762435da Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Sun, 21 Dec 2025 20:12:07 +0500 Subject: [PATCH 1/5] Automation: add timeout on test phase Tests may stuck for some reason and github will kill CI pipeline leaving us without any clue what happened. This change limits time for each step to run. When timeout raised - github will kill single step, and other steps will collect results and will provide test reports with logs. --- .github/workflows/pxf-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml index d7c9a29d..8acacf29 100644 --- a/.github/workflows/pxf-ci.yml +++ b/.github/workflows/pxf-ci.yml @@ -179,6 +179,7 @@ jobs: - name: Run Test - ${{ matrix.test_group }} id: run_test continue-on-error: true + timeout-minutes: 120 run: | docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf/automation && source ../concourse/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh && ../concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh ${{ matrix.test_group }}" From 7de57e55073771335998d49b6a2579baeb389727 Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Sun, 21 Dec 2025 20:57:46 +0500 Subject: [PATCH 2/5] Automation: Fail whole test-group when we have single failed test --- .github/workflows/pxf-ci.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml index 8acacf29..1922bce7 100644 --- a/.github/workflows/pxf-ci.yml +++ b/.github/workflows/pxf-ci.yml @@ -182,9 +182,10 @@ jobs: timeout-minutes: 120 run: | docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf/automation && source ../concourse/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh && ../concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh ${{ matrix.test_group }}" - + - name: Collect artifacts and generate stats if: always() + id: collect_artifacts run: | mkdir -p artifacts/logs TEST_GROUP="${{ matrix.test_group }}" @@ -230,6 +231,8 @@ jobs: } EOF + echo "failed_count=$FAILED" >> $GITHUB_OUTPUT + echo "skipped_count=$SKIPPED" >> $GITHUB_OUTPUT echo "Test stats for $TEST_GROUP: total=$TOTAL, passed=$PASSED, failed=$FAILED, skipped=$SKIPPED" - name: Cleanup containers @@ -250,8 +253,11 @@ jobs: - name: Check test result if: always() run: | - if [ "${{ steps.run_test.outcome }}" == "failure" ]; then - echo "Test group ${{ matrix.test_group }} failed" + FAILED_COUNT="${{ steps.collect_artifacts.outputs.failed_count || 0 }}" + SKIPPED_COUNT="${{ steps.collect_artifacts.outputs.skipped_count || 0 }}" + + if [ "${{ steps.run_test.outcome }}" == "failure" ] || [ "$FAILED_COUNT" -gt 0 ]; then + echo "Test group ${{ matrix.test_group }} failed (Failures: $FAILED_COUNT, Skipped: $SKIPPED_COUNT)" exit 1 fi @@ -280,6 +286,7 @@ jobs: OVERALL_SKIPPED=0 GROUPS_PASSED=0 GROUPS_FAILED=0 + FAILED_GROUP_NAMES="" # Collect all test stats declare -A GROUP_STATS @@ -300,10 +307,11 @@ jobs: OVERALL_FAILED=$((OVERALL_FAILED + failed)) OVERALL_SKIPPED=$((OVERALL_SKIPPED + skipped)) - if [ "$result" == "success" ]; then + if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then GROUPS_PASSED=$((GROUPS_PASSED + 1)) else GROUPS_FAILED=$((GROUPS_FAILED + 1)) + FAILED_GROUP_NAMES="${FAILED_GROUP_NAMES}${group} " fi fi done @@ -331,7 +339,7 @@ jobs: for group in $(echo "${!GROUP_STATS[@]}" | tr ' ' '\n' | sort); do IFS=',' read -r result total passed failed skipped <<< "${GROUP_STATS[$group]}" - if [ "$result" == "success" ]; then + if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then status="✅ PASS" else status="❌ FAIL" @@ -343,6 +351,6 @@ jobs: # Check if any group failed if [ $GROUPS_FAILED -gt 0 ]; then - echo "::error::${GROUPS_FAILED} test group(s) failed" + echo "::error::${GROUPS_FAILED} test group(s) failed: ${FAILED_GROUP_NAMES}" exit 1 fi From 9bff28dfb7ac62a5546c2fdd649a0e06c9c37014 Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Sun, 21 Dec 2025 23:28:04 +0500 Subject: [PATCH 3/5] Automation: save automation logs --- concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh b/concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh index 1741540d..5d5903d7 100755 --- a/concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh +++ b/concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh @@ -555,6 +555,7 @@ gpdb_test() { save_test_reports() { local group="$1" local surefire_dir="${REPO_ROOT}/automation/target/surefire-reports" + local logs_dir="${REPO_ROOT}/automation/automation_logs" local artifacts_dir="${REPO_ROOT}/automation/test_artifacts" local group_dir="${artifacts_dir}/${group}" @@ -566,6 +567,13 @@ save_test_reports() { else echo "[run_tests] No surefire reports found for $group" fi + + if [ -d "$logs_dir" ] && [ "$(ls -A "$logs_dir" 2>/dev/null)" ]; then + echo "[run_tests] Saving $group test logs to $group_dir" + cp -r "$logs_dir" "$group_dir/" 2>/dev/null || true + else + echo "[run_tests] No automation logs found for $group" + fi } # Generate test summary from surefire reports From fb3ec60f1ba265be1078bbf7d13b76d399292e38 Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Mon, 22 Dec 2025 01:27:20 +0500 Subject: [PATCH 4/5] Automation: wait for files to be copied in Hive tests --- .../greenplum/pxf/automation/features/hive/HiveBaseTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/hive/HiveBaseTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/hive/HiveBaseTest.java index b537e8f4..ff669d3d 100755 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/hive/HiveBaseTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/hive/HiveBaseTest.java @@ -384,9 +384,7 @@ void loadDataIntoHive(Hdfs hdfs, Hive hive, String fileName, HiveTable tableName hdfs.copyFromLocal(localPath, hdfsPath); // Verify file was copied to HDFS - if (!hdfs.doesFileExist(hdfsPath)) { - throw new RuntimeException("File was not copied to HDFS: " + hdfsPath); - } + hdfs.waitForFile(hdfsPath, 3); // load to hive table hive.loadData(tableName, hdfsPath, false); From 4b24daf4f0d454c779e322e1bdba7b79a3a8431a Mon Sep 17 00:00:00 2001 From: Nikolay Antonov Date: Wed, 24 Dec 2025 19:58:39 +0500 Subject: [PATCH 5/5] Automation: upgrade testng to reduce amount of SKIPPED tests Old TestNG versions has bug https://github.com/testng-team/testng/issues/739 - when `@BeforeClass(alwaysRun = true)` fails - all tests are marked as SKIPPED despite of `alwaysRun = true`. Fixed in 6.9.5+. --- automation/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/pom.xml b/automation/pom.xml index 3b190572..a3ffa806 100644 --- a/automation/pom.xml +++ b/automation/pom.xml @@ -163,7 +163,7 @@ org.testng testng - 6.8.7 + 6.14.3