Merge v1.4-andium and fix build

.github/workflows/packaging_wheels.yml

@@ -33,11 +33,12 @@ jobs:
         python: [ cp39, cp310, cp311, cp312, cp313, cp314 ]
         platform:
           - { os: windows-2025,     arch: amd64,      cibw_system: win }
+          - { os: windows-11-arm,   arch: ARM64,      cibw_system: win } # cibw requires ARM64 to be uppercase
           - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
           - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
           - { os: macos-15,         arch: arm64,      cibw_system: macosx }
           - { os: macos-15,         arch: universal2, cibw_system: macosx }
-          - { os: macos-13,         arch: x86_64,     cibw_system: macosx }
+          - { os: macos-15-intel,   arch: x86_64,     cibw_system: macosx }
         minimal:
           - ${{ inputs.minimal }}
         exclude:
@@ -46,6 +47,8 @@ jobs:
           - { minimal: true, python: cp312 }
           - { minimal: true, python: cp313 }
           - { minimal: true, platform: { arch: universal2 } }
+          - { python: cp39,  platform: { os: windows-11-arm, arch: ARM64 } }  # too many dependency problems for win arm64
+          - { python: cp310,  platform: { os: windows-11-arm, arch: ARM64 } }  # too many dependency problems for win arm64
     runs-on: ${{ matrix.platform.os }}
     env:
       ### cibuildwheel configuration

.github/workflows/release.yml

@@ -22,6 +22,11 @@ on:
         options:
           - test
           - prod
+      nightly-stale-after-days:
+        type: string
+        description: After how many days should nightlies be considered stale
+        required: true
+        default: 3
       store-s3:
         type: boolean
         description: Also store test packages in S3 (always true for prod)
@@ -41,6 +46,17 @@ jobs:
       duckdb-sha: ${{ inputs.duckdb-sha }}
       set-version: ${{ inputs.stable-version }}
 
+  submodule_pr:
+    name: Create or update PR to bump submodule to given SHA
+    needs: build_sdist
+    uses: ./.github/workflows/submodule_auto_pr.yml
+    with:
+      duckdb-python-sha: ${{ inputs.duckdb-python-sha }}
+      duckdb-sha: ${{ inputs.duckdb-sha }}
+    secrets:
+      # reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206
+      DUCKDBLABS_BOT_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+
   workflow_state:
     name: Set state for the release workflow
     needs: build_sdist
@@ -51,23 +67,36 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - id: index_check
-        name: Check ${{ needs.build_sdist.outputs.package-version }} on PyPI
+        name: Check version on PyPI
         run: |
-          set -eu
-          # Check PyPI whether the release we're building is already present
+          set -ex
           pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org
-          pkg_version=${{ needs.build_sdist.outputs.package-version }}
-          url=https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json
-          http_status=$( curl -s -o /dev/null -w "%{http_code}" $url || echo $? )
-          if [[ $http_status == "200" ]]; then
-            echo "::warning::Package version ${pkg_version} is already present on ${pypi_hostname}"
-            pypi_state=VERSION_FOUND
-          elif [[ $http_status == 000* ]]; then
-            echo "::error::Error checking PyPI at ${url}: curl exit code ${http_status#'000'}"
-            pypi_state=UNKNOWN
-          else
-            echo "::notice::Package version ${pkg_version} not found on ${pypi_hostname} (http status: ${http_status})"
+          # install duckdb
+          curl https://install.duckdb.org | sh
+          # query pypi
+          result=$(cat <<EOF | ${HOME}/.duckdb/cli/latest/duckdb | xargs
+          ---- Output lines
+          .mode line
+          ---- Query that fetches the given version's age, if the version already exists
+          SELECT
+            today() - (file.value->>'upload_time_iso_8601')::DATE     AS age,
+          FROM read_json('https://${pypi_hostname}/pypi/duckdb/json') AS jd
+          CROSS JOIN json_each(jd.releases)                           AS rel(key, value)
+          CROSS JOIN unnest(FROM_JSON(rel.value, '["JSON"]'))         AS file(value)
+          WHERE rel.key='${{ needs.build_sdist.outputs.package-version }}'
+          LIMIT 1;
+          EOF
+          )
+          if [ -z "$result" ]; then
             pypi_state=VERSION_NOT_FOUND
+          else
+            pypi_state=VERSION_FOUND
+          fi
+          if [[ -z "${{ inputs.stable-version }}" ]]; then
+            age=${result#age = }
+            if [ "${age}" -ge "${{ inputs.nightly-stale-after-days }}" ]; then
+              echo "::warning title=Stale nightly for ${{ github.ref_name }}::Nightly is ${age} days old (max=${{ inputs.nightly-stale-after-days }})"
+            fi
           fi
           echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT
 
@@ -96,7 +125,7 @@ jobs:
             echo "::notice::S3 upload disabled in inputs, not generating S3 URL"
             exit 0
           fi
-          if [[ VERSION_FOUND == "${{ steps.index_check.outputs.pypi_state }}" ]]; then
+          if [[ VERSION_NOT_FOUND != "${{ steps.index_check.outputs.pypi_state }}" ]]; then
             echo "::warning::S3 upload disabled because package version already uploaded to PyPI"
             exit 0
           fi
@@ -110,7 +139,7 @@ jobs:
   build_wheels:
     name: Build and test releases
     needs: workflow_state
-    if: ${{ needs.workflow_state.outputs.pypi_state != 'VERSION_FOUND' }}
+    if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_NOT_FOUND' }}
     uses: ./.github/workflows/packaging_wheels.yml
     with:
       minimal: false

.github/workflows/submodule_auto_pr.yml

@@ -0,0 +1,130 @@
+name: Submodule Auto PR
+on:
+  workflow_call:
+    inputs:
+      duckdb-python-sha:
+        type: string
+        description: The commit to build against (defaults to latest commit of current ref)
+        required: false
+      duckdb-sha:
+        type: string
+        description: The DuckDB submodule commit or ref to build against
+        required: true
+      auto-land:
+        type: boolean
+        description: Immediately merge the PR (placeholder - doesn't work)
+        default: false
+    secrets:
+      DUCKDBLABS_BOT_TOKEN:
+        description: Github token of the DuckDBLabs bot
+        required: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  create_pr:
+    name: Create PR to bump duckdb submodule to given SHA
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout DuckDB Python
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.duckdb-python-sha }}
+          fetch-depth: 0
+          submodules: true
+
+      - name: Checkout or Create Needed Branch
+        run: |
+          git fetch --all
+          head_sha=${{ inputs.duckdb-python-sha }}
+          branch_name="vendoring-${{ github.ref_name }}"
+          if [[ `git rev-parse --verify ${branch_name} 2>/dev/null` ]]; then
+            # branch exists
+            git checkout ${branch_name}
+          else
+            # new branch
+            git checkout -b ${branch_name}
+          fi
+          [[ ${head_sha} ]] && git reset --hard ${head_sha} || true
+
+      - name: Checkout DuckDB at Given SHA
+        run: |
+          cd external/duckdb
+          git fetch origin
+          git checkout ${{ inputs.duckdb-sha }}
+
+      - name: Determine GH PR Command
+        id: gh_pr_command
+        env:
+          GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+        run: |
+          pr_url=$( gh pr list --head vendoring-${{ github.ref_name }} --state open --json url --jq '.[].url' )
+          if [[ $pr_url ]]; then
+            echo "::notice::Found existing pr, will edit (${pr_url})"
+            gh_command="edit ${pr_url}"
+          else
+            echo "::notice::No existing PR, will create new"
+            gh_command="create --head vendoring-${{ github.ref_name }} --base ${{ github.ref_name }}"
+          fi
+          echo "subcommand=${gh_command}" >> $GITHUB_OUTPUT
+
+      - name: Set Git User
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "DuckDB Labs GitHub Bot"
+
+      - name: Create PR to Bump DuckDB Submodule
+        env:
+          GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+        run: |
+          # No need to do anything if the submodule is already at the given sha
+          [[ `git status --porcelain -- external/duckdb` == "" ]] && exit 0
+          # We have changes. Commit and push
+          git add external/duckdb
+          git commit -m "Bump submodule"
+          git push --force origin vendoring-${{ github.ref_name }}
+          # create PR msg
+          echo "Bump duckdb submodule:" > body.txt
+          echo "- Target branch: ${{ github.ref_name }}" >> body.txt
+          echo "- Date: $( date +"%Y-%m-%d %H:%M:%S" )" >> body.txt
+          echo "- DuckDB SHA: ${{ inputs.duckdb-sha }}" >> body.txt
+          echo "- Trigger: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> body.txt
+          subcommand="${{ steps.gh_pr_command.outputs.subcommand }}"
+          gh pr ${subcommand} \
+            --title "[duckdb-labs bot] Bump DuckDB submodule" \
+            --body-file body.txt > output.txt 2>&1
+          success=$?
+          # Show summary
+          url=$( [[ $success ]] && gh pr view vendoring-${{ github.ref_name }} --json url --jq .url || true )
+          echo "## Submodule PR Summary" >> $GITHUB_STEP_SUMMARY
+          if [[ $success ]]; then
+            prefix=$( [[ $subcommand == edit* ]] && echo "Created" || echo "Updated" )
+            echo "### ${prefix} PR: [${url}](${url})" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "### Failed to create PR" >> $GITHUB_STEP_SUMMARY
+          fi
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          cat output.txt >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          [[ $success ]] || exit 1
+
+      - name: Automerge PR
+        if: ${{ inputs.auto-land }}
+        env:
+          GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+        run: |
+          # PLACEHOLDER: DUCKDBLABS_BOT_TOKEN DOES NOT HAVE PERMISSIONS TO MERGE PRS
+          set -ex
+          gh pr merge vendoring-${{ github.ref_name }} --rebase > output.txt
+          success=$?
+          # Show summary
+          if [[ $success ]]; then
+            echo "### PR merged" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "### Failed to auto-merge PR" >> $GITHUB_STEP_SUMMARY
+          fi
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          cat output.txt >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY

.github/workflows/targeted_test.yml

@@ -9,6 +9,7 @@ on:
         type: choice
         options:
           - 'windows-2025'
+          - 'windows-11-arm'
           - 'ubuntu-24.04'
           - 'ubuntu-24.04-arm'
           - 'macos-15'
@@ -36,6 +37,11 @@ on:
         description: 'Custom test path (must be in tests/ directory, overrides testsuite)'
         required: false
         type: string
+      verbose-uv:
+        description: 'Let uv generate verbose output (pytest verbosity is always on)'
+        required: false
+        type: boolean
+        default: true
 
 jobs:
   test:
@@ -83,4 +89,4 @@ jobs:
       - name: Run tests
         shell: bash
         run: |
-          uv run pytest -vv ${{ steps.test_path.outputs.test_path }}
+          uv ${{ inputs.verbose-uv && 'run -v' || 'run' }} pytest -vv ${{ steps.test_path.outputs.test_path }}

_duckdb-stubs/__init__.pyi

@@ -318,7 +318,18 @@ class DuckDBPyConnection:
     def list_type(self, type: sqltypes.DuckDBPyType) -> sqltypes.DuckDBPyType: ...
     def load_extension(self, extension: str) -> None: ...
     def map_type(self, key: sqltypes.DuckDBPyType, value: sqltypes.DuckDBPyType) -> sqltypes.DuckDBPyType: ...
-    def pl(self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: bool = False) -> polars.DataFrame: ...
+    @pytyping.overload
+    def pl(
+        self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[False] = ...
+    ) -> polars.DataFrame: ...
+    @pytyping.overload
+    def pl(
+        self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[True]
+    ) -> polars.LazyFrame: ...
+    @pytyping.overload
+    def pl(
+        self, rows_per_batch: pytyping.SupportsInt = 1000000, *, lazy: bool = False
+    ) -> pytyping.Union[polars.DataFrame, polars.LazyFrame]: ...
     def query(self, query: str, *, alias: str = "", params: object = None) -> DuckDBPyRelation: ...
     def query_progress(self) -> float: ...
     def read_csv(
@@ -596,7 +607,16 @@ class DuckDBPyRelation:
     ) -> DuckDBPyRelation: ...
     def order(self, order_expr: str) -> DuckDBPyRelation: ...
     def percent_rank(self, window_spec: str, projected_columns: str = "") -> DuckDBPyRelation: ...
-    def pl(self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: bool = False) -> polars.DataFrame: ...
+    @pytyping.overload
+    def pl(
+        self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[False] = ...
+    ) -> polars.DataFrame: ...
+    @pytyping.overload
+    def pl(self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: pytyping.Literal[True]) -> polars.LazyFrame: ...
+    @pytyping.overload
+    def pl(
+        self, batch_size: pytyping.SupportsInt = 1000000, *, lazy: bool = False
+    ) -> pytyping.Union[polars.DataFrame, polars.LazyFrame]: ...
     def product(
         self, column: str, groups: str = "", window_spec: str = "", projected_columns: str = ""
     ) -> DuckDBPyRelation: ...
@@ -700,6 +720,7 @@ class DuckDBPyRelation:
         partition_by: pytyping.List[str] | None = None,
         write_partition_columns: bool | None = None,
         append: bool | None = None,
+        filename_pattern: str | None = None,
     ) -> None: ...
     def to_table(self, table_name: str) -> None: ...
     def to_view(self, view_name: str, replace: bool = True) -> DuckDBPyRelation: ...
@@ -752,6 +773,7 @@ class DuckDBPyRelation:
         partition_by: pytyping.List[str] | None = None,
         write_partition_columns: bool | None = None,
         append: bool | None = None,
+        filename_pattern: str | None = None,
     ) -> None: ...
     @property
     def alias(self) -> str: ...
@@ -1048,7 +1070,7 @@ def commit(*, connection: DuckDBPyConnection | None = None) -> DuckDBPyConnectio
 def connect(
     database: str | pathlib.Path = ":memory:",
     read_only: bool = False,
-    config: dict[str, str] | None = None,
+    config: dict[str, str | bool | int | float | list[str]] | None = None,
 ) -> DuckDBPyConnection: ...
 def create_function(
     name: str,
@@ -1241,12 +1263,27 @@ def map_type(
 def order(
     df: pandas.DataFrame, order_expr: str, *, connection: DuckDBPyConnection | None = None
 ) -> DuckDBPyRelation: ...
+@pytyping.overload
 def pl(
     rows_per_batch: pytyping.SupportsInt = 1000000,
     *,
-    lazy: bool = False,
+    lazy: pytyping.Literal[False] = ...,
     connection: DuckDBPyConnection | None = None,
 ) -> polars.DataFrame: ...
+@pytyping.overload
+def pl(
+    rows_per_batch: pytyping.SupportsInt = 1000000,
+    *,
+    lazy: pytyping.Literal[True],
+    connection: DuckDBPyConnection | None = None,
+) -> polars.LazyFrame: ...
+@pytyping.overload
+def pl(
+    rows_per_batch: pytyping.SupportsInt = 1000000,
+    *,
+    lazy: bool = False,
+    connection: DuckDBPyConnection | None = None,
+) -> pytyping.Union[polars.DataFrame, polars.LazyFrame]: ...
 def project(
     df: pandas.DataFrame, *args: str | Expression, groups: str = "", connection: DuckDBPyConnection | None = None
 ) -> DuckDBPyRelation: ...

duckdb/experimental/__init__.py

@@ -1,3 +1,5 @@
 from . import spark  # noqa: D104
 
-__all__ = spark.__all__
+__all__ = [
+    "spark",
+]