Added a workflow to parallelise the E2E tests. Updated E2E tests to c…

.github/workflows/code-coverage.yml

@@ -0,0 +1,136 @@
+name: Code Coverage
+
+permissions:
+  contents: read
+
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  test-with-coverage:
+    runs-on: ubuntu-latest
+    environment: azure-prod
+    env:
+      DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
+      DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
+      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+      DATABRICKS_CATALOG: peco
+      DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
+    steps:
+      #----------------------------------------------
+      #       check-out repo and set-up python
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      #----------------------------------------------
+      #  -----  install & configure poetry  -----
+      #----------------------------------------------
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      #----------------------------------------------
+      #       load cached venv if cache exists
+      #----------------------------------------------
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+      #----------------------------------------------
+      # install dependencies if cache does not exist
+      #----------------------------------------------
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+      #----------------------------------------------
+      # install your root project, if required
+      #----------------------------------------------
+      - name: Install library
+        run: poetry install --no-interaction --all-extras
+      #----------------------------------------------
+      #    run all tests with coverage
+      #----------------------------------------------
+      - name: Run all tests with coverage 
+        continue-on-error: false
+        run: |
+          poetry run pytest tests/unit tests/e2e \
+            -n auto \
+            --cov=src \
+            --cov-report=xml \
+            --cov-report=term \
+            -v
+
+      #----------------------------------------------
+      #         check for coverage override
+      #----------------------------------------------
+      - name: Check for coverage override
+        id: override
+        run: |
+          OVERRIDE_COMMENT=$(echo "${{ github.event.pull_request.body }}" | grep -E "SKIP_COVERAGE_CHECK\s*=" || echo "")
+          if [ -n "$OVERRIDE_COMMENT" ]; then
+            echo "override=true" >> $GITHUB_OUTPUT
+            REASON=$(echo "$OVERRIDE_COMMENT" | sed -E 's/.*SKIP_COVERAGE_CHECK\s*=\s*(.+)/\1/')
+            echo "reason=$REASON" >> $GITHUB_OUTPUT
+            echo "Coverage override found in PR description: $REASON"
+          else
+            echo "override=false" >> $GITHUB_OUTPUT
+            echo "No coverage override found"
+          fi
+      #----------------------------------------------
+      #         check coverage percentage
+      #----------------------------------------------
+      - name: Check coverage percentage
+        if: steps.override.outputs.override == 'false'
+        run: |
+          COVERAGE_FILE="coverage.xml"
+          if [ ! -f "$COVERAGE_FILE" ]; then
+            echo "ERROR: Coverage file not found at $COVERAGE_FILE"
+            exit 1
+          fi
+
+          # Install xmllint if not available
+          if ! command -v xmllint &> /dev/null; then
+            sudo apt-get update && sudo apt-get install -y libxml2-utils
+          fi
+
+          COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE")
+          TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE")
+          PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))")
+
+          echo "Branch Coverage: $PERCENTAGE%"
+          echo "Required Coverage: 85%"
+
+          # Use Python to compare the coverage with 85
+          python3 -c "import sys; sys.exit(0 if float('$PERCENTAGE') >= 85 else 1)"
+          if [ $? -eq 1 ]; then
+            echo "ERROR: Coverage is $PERCENTAGE%, which is less than the required 85%"
+            exit 1
+          else
+            echo "SUCCESS: Coverage is $PERCENTAGE%, which meets the required 85%"
+          fi
+
+      #----------------------------------------------
+      #         coverage enforcement summary
+      #----------------------------------------------
+      - name: Coverage enforcement summary
+        run: |
+          if [ "${{ steps.override.outputs.override }}" == "true" ]; then
+            echo "⚠️ Coverage checks bypassed: ${{ steps.override.outputs.reason }}"
+            echo "Please ensure this override is justified and temporary"
+          else
+            echo "✅ Coverage checks enforced - minimum 85% required"
+          fi
+

pyproject.toml

@@ -39,6 +39,7 @@ pylint = ">=2.12.0"
 black = "^22.3.0"
 pytest-dotenv = "^0.5.2"
 pytest-cov = "^4.0.0"
+pytest-xdist = "^3.0.0"
 numpy = [
     { version = ">=1.16.6", python = ">=3.8,<3.11" },
     { version = ">=1.23.4", python = ">=3.11" },

tests/e2e/test_complex_types.py

@@ -1,6 +1,7 @@
 import pytest
 from numpy import ndarray
 from typing import Sequence
+from uuid import uuid4
 
 from tests.e2e.test_driver import PySQLPytestTestCase
 
@@ -10,12 +11,15 @@ class TestComplexTypes(PySQLPytestTestCase):
     def table_fixture(self, connection_details):
         self.arguments = connection_details.copy()
         """A pytest fixture that creates a table with a complex type, inserts a record, yields, and then drops the table"""
+
+        table_name = f"pysql_test_complex_types_table_{str(uuid4()).replace('-', '_')}"
+        self.table_name = table_name
 
         with self.cursor() as cursor:
             # Create the table
             cursor.execute(
-                """
-                CREATE TABLE IF NOT EXISTS pysql_test_complex_types_table (
+                f"""
+                CREATE TABLE IF NOT EXISTS {table_name} (
                     array_col ARRAY<STRING>,
                     map_col MAP<STRING, INTEGER>,
                     struct_col STRUCT<field1: STRING, field2: INTEGER>,
@@ -27,8 +31,8 @@ def table_fixture(self, connection_details):
             )
             # Insert a record
             cursor.execute(
-                """
-                INSERT INTO pysql_test_complex_types_table
+                f"""
+                INSERT INTO {table_name}
                 VALUES (
                     ARRAY('a', 'b', 'c'),
                     MAP('a', 1, 'b', 2, 'c', 3),
@@ -40,10 +44,10 @@ def table_fixture(self, connection_details):
                 """
             )
             try:
-                yield
+                yield table_name
             finally:
                 # Clean up the table after the test
-                cursor.execute("DELETE FROM pysql_test_complex_types_table")
+                cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
 
     @pytest.mark.parametrize(
         "field,expected_type",
@@ -61,7 +65,7 @@ def test_read_complex_types_as_arrow(self, field, expected_type, table_fixture):
 
         with self.cursor() as cursor:
             result = cursor.execute(
-                "SELECT * FROM pysql_test_complex_types_table LIMIT 1"
+                f"SELECT * FROM {table_fixture} LIMIT 1"
             ).fetchone()
 
         assert isinstance(result[field], expected_type)
@@ -83,7 +87,7 @@ def test_read_complex_types_as_string(self, field, table_fixture):
             extra_params={"_use_arrow_native_complex_types": False}
         ) as cursor:
             result = cursor.execute(
-                "SELECT * FROM pysql_test_complex_types_table LIMIT 1"
+                f"SELECT * FROM {table_fixture} LIMIT 1"
             ).fetchone()
 
         assert isinstance(result[field], str)