Require a lower bound on every dependency in pyproject.toml (#71378)

An unbounded requirement lets the resolver answer with any version that has
ever been published, so what our constraints pin - and what a user ends up
installing - depends on how the resolution went rather than on what the code
needs. Four such requirements were found and fixed by hand while pinning
providers in constraints (#71324); they sat in four different dependency
tables, which is why they went unnoticed for so long.

Distributions that are members of the uv workspace are exempt - they resolve
from the checkout, so a version range would say nothing - as are direct URL
requirements, where the URL already names the exact artifact.

Author: potiuk

.pre-commit-config.yaml

@@ -1081,6 +1081,13 @@ repos:
         pass_filenames: false
         require_serial: true
         additional_dependencies: ['packaging>=25', 'pyyaml', 'tomli>=2.0.1', 'rich>=13.6.0']
+      - id: check-dependency-lower-bounds
+        name: Check that dependencies in pyproject.toml have lower bounds
+        language: python
+        entry: ./scripts/ci/prek/check_dependency_lower_bounds.py
+        files: (^|/)pyproject\.toml$
+        require_serial: true
+        additional_dependencies: ['packaging>=25', 'tomli>=2.0.1', 'rich>=13.6.0']
       - id: update-reproducible-source-date-epoch
         name: Update Source Date Epoch for reproducible builds
         language: python

clients/python/pyproject.toml

@@ -59,7 +59,7 @@ classifiers = [
 
 dependencies = [
     "pydantic >= 2.11.0",
-    "python-dateutil",
+    "python-dateutil>=2.7.0",
     "urllib3>=2.1.0,!=2.6.0",
 ]
 

contributing-docs/13_airflow_dependencies_and_extras.rst

@@ -239,6 +239,21 @@ rules to remember:
   stopped working (like in case of ``amazon``, ``fab``). You are free to modify those versions to higher
   versions if you need to, and ``prek`` will remove those comments automatically.
 
+* Every dependency we resolve from PyPI must have a lower bound. Without one the resolver is free to answer
+  with any version that has ever been published, so what our constraints pin - and what a user ends up
+  installing - depends on how the resolution went rather than on what the code needs. The
+  ``check-dependency-lower-bounds`` prek hook enforces this across ``project.dependencies``,
+  ``project.optional-dependencies``, ``dependency-groups`` and ``build-system.requires`` of every
+  ``pyproject.toml``. Use the oldest version you are willing to test against:
+
+  .. code-block:: python
+
+     "pyspark>=4.0.0",
+
+  Two kinds of requirement are exempt: distributions that are members of our ``uv`` workspace (they are
+  resolved from the checkout, so a version range would say nothing) and direct URL requirements (the URL
+  already names the exact artifact).
+
 Our CI system will do all the tests for you anyway - including running some lower-bind checks on dependencies.
 For example it will take each provider in a turn and will try to resolve lowest-possible dependencies defined
 for that provider and see if the tests are still passing, so we should be relatively protected against putting

providers/common/ai/docs/index.rst

@@ -241,11 +241,11 @@ Install them when installing from PyPI. For example:
 ==============  =======================================================================================================================================
 Extra           Dependencies
 ==============  =======================================================================================================================================
-``anthropic``   ``pydantic-ai-slim[anthropic]``
-``bedrock``     ``pydantic-ai-slim[bedrock]``
-``google``      ``pydantic-ai-slim[google]``
-``openai``      ``pydantic-ai-slim[openai]``
-``mcp``         ``pydantic-ai-slim[mcp]``
+``anthropic``   ``pydantic-ai-slim[anthropic]>=2.0.0``
+``bedrock``     ``pydantic-ai-slim[bedrock]>=2.0.0``
+``google``      ``pydantic-ai-slim[google]>=2.0.0``
+``openai``      ``pydantic-ai-slim[openai]>=2.0.0``
+``mcp``         ``pydantic-ai-slim[mcp]>=2.0.0``
 ``code-mode``   ``pydantic-ai-harness[codemode]>=0.3.0``
 ``shields``     ``pydantic-ai-shields>=0.3.4``
 ``skills``      ``apache-airflow-providers-git>=0.4.0``, ``pydantic-ai-skills>=1.2.0``

providers/common/ai/pyproject.toml

@@ -76,11 +76,11 @@ dependencies = [
 # The optional dependencies should be modified in place in the generated file
 # Any change in the dependencies is preserved when the file is regenerated
 [project.optional-dependencies]
-"anthropic" = ["pydantic-ai-slim[anthropic]"]
-"bedrock" = ["pydantic-ai-slim[bedrock]"]
-"google" = ["pydantic-ai-slim[google]"]
-"openai" = ["pydantic-ai-slim[openai]"]
-"mcp" = ["pydantic-ai-slim[mcp]"]
+"anthropic" = ["pydantic-ai-slim[anthropic]>=2.0.0"]
+"bedrock" = ["pydantic-ai-slim[bedrock]>=2.0.0"]
+"google" = ["pydantic-ai-slim[google]>=2.0.0"]
+"openai" = ["pydantic-ai-slim[openai]>=2.0.0"]
+"mcp" = ["pydantic-ai-slim[mcp]>=2.0.0"]
 # Code mode: collapse tool calls into a single `run_code` tool that the model
 # drives by writing Python, executed in the Monty sandbox (pydantic-monty).
 # Enables AgentOperator(code_mode=True). Monty is pre-1.0; pinned here as an

scripts/ci/prek/check_dependency_lower_bounds.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# /// script
+# requires-python = ">=3.10,<3.11"
+# dependencies = [
+#   "packaging>=25",
+#   "rich>=13.6.0",
+#   "tomli>=2.0.1",
+# ]
+# ///
+"""
+Validate that every external dependency declared in a ``pyproject.toml`` has a lower bound.
+
+An unbounded requirement lets the resolver answer with any version that happens to be on
+PyPI, so what a constraints file pins - and what a user ends up installing - depends on how
+the resolution went rather than on what the code needs. Naming the oldest supported version
+makes that answer deterministic and documents the floor the code is tested against.
+
+Checked locations: ``project.dependencies``, ``project.optional-dependencies``,
+``dependency-groups`` and ``build-system.requires``.
+
+Requirements resolved from the uv workspace rather than from PyPI are exempt - their source
+is the checkout, so a version range would say nothing. Direct URL requirements are exempt
+too, since the URL already names the exact artifact.
+"""
+
+from __future__ import annotations
+
+import sys
+from functools import cache
+from pathlib import Path
+
+from common_prek_utils import AIRFLOW_ROOT_PATH, console
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.utils import canonicalize_name
+from rich.markup import escape
+
+try:
+    import tomllib
+except ImportError:
+    import tomli as tomllib  # type: ignore[no-redef]
+
+# Operators that place a floor under the resolved version; anything else (``!=``, ``<``,
+# ``<=``) leaves the resolver free to reach back to the oldest release ever published.
+LOWER_BOUND_OPERATORS = {">=", ">", "==", "===", "~="}
+
+
+def _load_toml(path: Path) -> dict:
+    return tomllib.loads(path.read_text())
+
+
+@cache
+def get_workspace_distribution_names() -> frozenset[str]:
+    """Return the canonical names of the distributions uv resolves from this checkout."""
+    root_pyproject = AIRFLOW_ROOT_PATH / "pyproject.toml"
+    root_data = _load_toml(root_pyproject)
+    members = root_data.get("tool", {}).get("uv", {}).get("workspace", {}).get("members", [])
+    names = set()
+    for member in members:
+        for member_path in sorted(AIRFLOW_ROOT_PATH.glob(f"{member}/pyproject.toml")):
+            if name := _load_toml(member_path).get("project", {}).get("name"):
+                names.add(canonicalize_name(name))
+    return frozenset(names)
+
+
+def extract_requirements(data: dict) -> list[tuple[str, str]]:
+    """Return ``(section, requirement)`` pairs for every dependency table we guard."""
+    requirements: list[tuple[str, str]] = []
+    project = data.get("project") or {}
+    for dependency in project.get("dependencies") or []:
+        requirements.append(("project.dependencies", dependency))
+    for extra, dependencies in (project.get("optional-dependencies") or {}).items():
+        for dependency in dependencies:
+            requirements.append((f'project.optional-dependencies."{extra}"', dependency))
+    for group, dependencies in (data.get("dependency-groups") or {}).items():
+        for dependency in dependencies:
+            # A group may also pull in another group via ``{include-group = "..."}``.
+            if isinstance(dependency, str):
+                requirements.append((f'dependency-groups."{group}"', dependency))
+    for dependency in (data.get("build-system") or {}).get("requires") or []:
+        requirements.append(("build-system.requires", dependency))
+    return requirements
+
+
+def check_requirement(section: str, dependency: str, workspace_names: frozenset[str]) -> str | None:
+    """Return an error message when ``dependency`` needs a lower bound, otherwise ``None``."""
+    try:
+        requirement = Requirement(dependency)
+    except InvalidRequirement as error:
+        return f"[{section}] {dependency!r} is not a valid requirement: {error}"
+    if (
+        requirement.url
+        or canonicalize_name(requirement.name) in workspace_names
+        or any(specifier.operator in LOWER_BOUND_OPERATORS for specifier in requirement.specifier)
+    ):
+        return None
+    return f"[{section}] {dependency!r} has no lower bound - add one, for example {requirement.name}>=X.Y.Z"
+
+
+def check_pyproject_file(path: Path, workspace_names: frozenset[str]) -> list[str]:
+    return [
+        error
+        for section, dependency in extract_requirements(_load_toml(path))
+        if (error := check_requirement(section, dependency, workspace_names))
+    ]
+
+
+def main() -> int:
+    workspace_names = get_workspace_distribution_names()
+    failed = False
+    for file in sys.argv[1:]:
+        path = Path(file)
+        if errors := check_pyproject_file(path, workspace_names):
+            failed = True
+            console.print(f"\n[red]Missing lower bounds in {file}:[/]\n")
+            for error in errors:
+                console.print(f"  {escape(error)}")
+    if failed:
+        console.print(
+            "\n[bright_yellow]Every dependency resolved from PyPI needs a lower bound.[/]\n"
+            "Without one the resolver may pick any published version, so constraints pin\n"
+            "whatever the resolution happened to produce rather than the oldest version the\n"
+            "code supports. Use the oldest version you are willing to test against.\n"
+        )
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/tests/ci/prek/test_check_dependency_lower_bounds.py

@@ -0,0 +1,143 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import textwrap
+
+import pytest
+from check_dependency_lower_bounds import check_pyproject_file, check_requirement, extract_requirements
+
+WORKSPACE_NAMES = frozenset({"apache-airflow-core", "apache-airflow-devel-common"})
+
+
+class TestCheckRequirement:
+    @pytest.mark.parametrize(
+        "dependency",
+        [
+            pytest.param("pyspark>=4.0.0", id="greater-or-equal"),
+            pytest.param("pyspark>4.0.0", id="greater"),
+            pytest.param("hatchling==1.31.0", id="pinned"),
+            pytest.param("hatchling===1.31.0", id="arbitrary-equality"),
+            pytest.param("hatchling~=1.31", id="compatible-release"),
+            pytest.param("urllib3>=2.1.0,!=2.6.0", id="lower-bound-with-exclusion"),
+            pytest.param("pydantic-ai-slim[mcp]>=2.0.0", id="extras"),
+            pytest.param('fastavro>=1.10.0; python_version < "3.14"', id="marker"),
+        ],
+    )
+    def test_no_error_when_lower_bound_present(self, dependency):
+        assert check_requirement("project.dependencies", dependency, WORKSPACE_NAMES) is None
+
+    @pytest.mark.parametrize(
+        "dependency",
+        [
+            pytest.param("pyspark", id="bare"),
+            pytest.param("pydantic-ai-slim[mcp]", id="extras"),
+            pytest.param("pyspark<5.0.0", id="upper-bound-only"),
+            pytest.param("pyspark!=4.1.0", id="exclusion-only"),
+            pytest.param('krb5; python_version < "3.14"', id="marker-only"),
+        ],
+    )
+    def test_error_when_lower_bound_missing(self, dependency):
+        error = check_requirement("project.dependencies", dependency, WORKSPACE_NAMES)
+        assert error is not None
+        assert "has no lower bound" in error
+        assert "[project.dependencies]" in error
+
+    @pytest.mark.parametrize(
+        "dependency",
+        [
+            pytest.param("apache-airflow-core", id="plain"),
+            pytest.param("apache_airflow_core", id="non-canonical-name"),
+            pytest.param("apache-airflow-devel-common[mypy]", id="extras"),
+        ],
+    )
+    def test_no_error_for_workspace_distribution(self, dependency):
+        assert check_requirement("dependency-groups.dev", dependency, WORKSPACE_NAMES) is None
+
+    def test_no_error_for_direct_url_requirement(self):
+        dependency = (
+            "sphinx-airflow-theme@https://airflow.apache.org/sphinx-airflow-theme/"
+            "sphinx_airflow_theme-0.3.13-py3-none-any.whl"
+        )
+        assert check_requirement("project.optional-dependencies.docs", dependency, WORKSPACE_NAMES) is None
+
+    def test_error_for_invalid_requirement(self):
+        error = check_requirement("project.dependencies", "not a requirement!", WORKSPACE_NAMES)
+        assert error is not None
+        assert "is not a valid requirement" in error
+
+
+class TestExtractRequirements:
+    def test_extracts_every_guarded_table(self):
+        data = {
+            "build-system": {"requires": ["hatchling"]},
+            "project": {
+                "dependencies": ["pyspark"],
+                "optional-dependencies": {"kerberos": ["krb5"]},
+            },
+            "dependency-groups": {"dev": ["pytest", {"include-group": "docs"}]},
+        }
+        assert extract_requirements(data) == [
+            ("project.dependencies", "pyspark"),
+            ('project.optional-dependencies."kerberos"', "krb5"),
+            ('dependency-groups."dev"', "pytest"),
+            ("build-system.requires", "hatchling"),
+        ]
+
+    def test_no_requirements_when_tables_absent(self):
+        assert extract_requirements({"tool": {"uv": {"required-version": ">=0.9.0"}}}) == []
+
+
+class TestCheckPyprojectFile:
+    def _write(self, tmp_path, content):
+        path = tmp_path / "pyproject.toml"
+        path.write_text(textwrap.dedent(content))
+        return path
+
+    def test_reports_every_unbounded_dependency(self, tmp_path):
+        path = self._write(
+            tmp_path,
+            """
+            [project]
+            name = "apache-airflow-providers-samba"
+            dependencies = ["smbprotocol>=1.5.0"]
+
+            [project.optional-dependencies]
+            "kerberos" = ["krb5", "apache-airflow-core"]
+
+            [dependency-groups]
+            dev = ["pytest"]
+            """,
+        )
+        errors = check_pyproject_file(path, WORKSPACE_NAMES)
+        assert len(errors) == 2
+        assert "krb5" in errors[0]
+        assert "pytest" in errors[1]
+
+    def test_no_errors_when_all_bounded(self, tmp_path):
+        path = self._write(
+            tmp_path,
+            """
+            [project]
+            name = "apache-airflow-providers-samba"
+            dependencies = ["smbprotocol>=1.5.0"]
+
+            [dependency-groups]
+            dev = ["pytest>=9.1.1"]
+            """,
+        )
+        assert check_pyproject_file(path, WORKSPACE_NAMES) == []