Consolidate Python package management on uv

.github/copilot-instructions.md

@@ -55,7 +55,7 @@ Before formatting or linting changes, load the [formatting-and-linting skill](/.
 ### Documentation
 
 ```bash
-pip install -r doc/requirements.txt -r doc/historical_ccf_requirements.txt
+uv pip install -r doc/requirements.txt -r doc/historical_ccf_requirements.txt
 sphinx-build --fail-on-warning -b html doc doc/html
 ```
 

.github/workflows/bencher-ab.yml

@@ -14,7 +14,7 @@ on:
 permissions: read-all
 
 env:
-  PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
+  UV_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}

.github/workflows/bencher.yml

@@ -11,7 +11,7 @@ on:
 permissions: read-all
 
 env:
-  PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
+  UV_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/ci-al4.yml

@@ -13,7 +13,7 @@ concurrency:
 permissions: read-all
 
 env:
-  PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
+  UV_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
 
 jobs:
   vmss-virtual-a:

.github/workflows/ci-verification.yml

@@ -11,7 +11,7 @@ concurrency:
 permissions: read-all
 
 env:
-  PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
+  UV_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
 
 jobs:
   model-checking-consistency:

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ env:
   NPM_CONFIG_FETCH_RETRY_MINTIMEOUT: "5000"
   NPM_CONFIG_FETCH_RETRY_MAXTIMEOUT: "30000"
   NPM_CONFIG_FETCH_TIMEOUT: "60000"
-  PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
+  UV_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
 
 jobs:
   vmss-virtual-a:
@@ -72,22 +72,21 @@ jobs:
       - name: "Run tests for Python package"
         run: |
           set -ex
-          python3 -m venv env
+          python3 -m venv --without-pip env
           source env/bin/activate
-          pip install -U -q pip pytest
-          pip install -q -U -e python
+          uv pip install -q pytest
+          uv pip install -q -e python
           cd python
           pytest
 
       - name: "Check doc builds with no warnings"
         run: |
           set -ex
-          python3 -m venv env
+          python3 -m venv --without-pip env
           source env/bin/activate
-          pip install -U pip
-          pip install -U -e ./python
-          pip install -U -r doc/requirements.txt
-          pip install -U -r doc/historical_ccf_requirements.txt
+          uv pip install -e ./python
+          uv pip install -r doc/requirements.txt
+          uv pip install -r doc/historical_ccf_requirements.txt
           sphinx-build --fail-on-warning -b html doc doc/html
 
       - name: "Run Unit tests"

.github/workflows/codeql-analysis.yml

@@ -23,7 +23,7 @@ concurrency:
 permissions: read-all
 
 env:
-  PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
+  UV_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
 
 jobs:
   analyze:

.github/workflows/coverage.yml

@@ -15,7 +15,7 @@ permissions: read-all
 # scripts/coverage_summary.py, so that the two stay in sync from a single place.
 env:
   COVERAGE_HISTORY_POINTS: "9"
-  PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
+  UV_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
 
 jobs:
   coverage_virtual:

.github/workflows/doc.yml

@@ -8,6 +8,9 @@ on:
 
 permissions: read-all
 
+env:
+  UV_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
+
 jobs:
   build:
     name: Build
@@ -38,12 +41,11 @@ jobs:
       - name: Build Documentation
         run: |
           set -x
-          python3 -m venv env
+          python3 -m venv --without-pip env
           source env/bin/activate
-          pip install -U pip
-          pip install -U -e ./python
-          pip install -U -r doc/requirements.txt
-          pip install -U -r doc/historical_ccf_requirements.txt
+          uv pip install -e ./python
+          uv pip install -r doc/requirements.txt
+          uv pip install -r doc/historical_ccf_requirements.txt
           sphinx-multiversion -D smv_remote_whitelist=origin doc build/html
         shell: bash
 

.github/workflows/long-test.yml

@@ -18,7 +18,7 @@ concurrency:
 permissions: read-all
 
 env:
-  PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
+  UV_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
 
 jobs:
   long-asan:

.github/workflows/release.yml

@@ -15,7 +15,7 @@ permissions: read-all
 
 env:
   IMAGE: mcr.microsoft.com/azurelinux/base/core:3.0
-  PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
+  UV_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
 
 jobs:
   make_sbom:
@@ -261,10 +261,7 @@ jobs:
         run: |
           set -ex
           cd python
-          python3 -m venv env
-          source ./env/bin/activate
-          pip install wheel build
-          python -m build --wheel
+          uv build --wheel
           WHL=`ls dist/*.whl`
           echo "name=$WHL" >> $GITHUB_OUTPUT
         shell: bash

.github/workflows/tla-shallow.yml

@@ -14,7 +14,7 @@ concurrency:
 permissions: read-all
 
 env:
-  PIP_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
+  UV_INDEX_URL: https://packagefeedproxy.microsoft.io/pypi/simple/
 
 jobs:
   simulation:

CMakeLists.txt

@@ -533,6 +533,7 @@ install(
 install(PROGRAMS tests/sandbox/sandbox.sh DESTINATION bin)
 install(PROGRAMS tests/docker_wrap.sh DESTINATION bin)
 install(PROGRAMS scripts/ensure-snmalloc.sh DESTINATION bin)
+install(PROGRAMS scripts/install_uv.sh DESTINATION bin)
 install(FILES samples/constitutions/default/actions.js DESTINATION bin)
 install(FILES samples/constitutions/default/validate.js DESTINATION bin)
 install(FILES samples/constitutions/sandbox/resolve.js DESTINATION bin)

livehtml.sh

@@ -7,13 +7,16 @@ set -e
 echo "Setting up Python environment..."
 if [ ! -f "env/bin/activate" ]
     then
-        python3 -m venv env
+        python3 -m venv --without-pip env
 fi
 
+bash ./scripts/install_uv.sh env/bin
 source env/bin/activate
-pip install -U pip
-pip install -q -U -e ./python/
-pip install -q -U -r ./doc/requirements.txt
+if [ -n "${PIP_INDEX_URL:-}" ] && [ -z "${UV_INDEX_URL:-}" ]; then
+    export UV_INDEX_URL="$PIP_INDEX_URL"
+fi
+uv pip install -q -e ./python/
+uv pip install -q -r ./doc/requirements.txt
 echo "Python environment successfully setup"
 
 sphinx-autobuild -b html doc doc/html --host localhost --port 8080

scripts/install_uv.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the Apache 2.0 License.
+
+set -euo pipefail
+
+UV_VERSION=0.11.19
+install_dir=${1:-/usr/local/bin}
+
+if [[ -x "$install_dir/uv" && -x "$install_dir/uvx" ]] &&
+    [[ "$("$install_dir/uv" --version)" == "uv ${UV_VERSION}"* ]]; then
+    exit 0
+fi
+
+if command -v uv >/dev/null 2>&1 &&
+    command -v uvx >/dev/null 2>&1 &&
+    [[ "$(uv --version)" == "uv ${UV_VERSION}"* ]]; then
+    exit 0
+fi
+
+mkdir -p "$install_dir"
+
+curl --proto "=https" --tlsv1.2 -LsSf \
+    "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-installer.sh" |
+    env UV_INSTALL_DIR="$install_dir" UV_NO_MODIFY_PATH=1 sh
+
+"$install_dir/uv" --version

scripts/setup-ci-al4.sh

@@ -5,6 +5,7 @@
 set -exo pipefail
 
 H2SPEC_VERSION="v2.6.0"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 
 retry() {
     local description=$1
@@ -141,14 +142,11 @@ install_packaging_and_python() {
         # For packaging
         rpm-build
         # For end to end tests and scripts
-        python3-pip
+        python3
         python3-devel
     )
     dnf -y install "${packages[@]}"
-
-    if ! python3 -m pip install uv==0.11.19 --break-system-packages; then
-        python3 -m pip install uv==0.11.19
-    fi
+    bash "$SCRIPT_DIR/install_uv.sh" /usr/local/bin
 }
 
 retry "Source control dependencies" install_source_control

scripts/setup-ci.sh

@@ -5,6 +5,7 @@
 set -exo pipefail
 
 H2SPEC_VERSION="v2.6.0"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 
 TDNF_OPTIONS=(-y)
 if [[ -n ${SOURCE_DATE_EPOCH:-} ]]; then
@@ -119,10 +120,10 @@ install_packaging_and_python() {
         # For packaging
         rpm-build
         # For end to end tests and scripts
-        python3-pip
+        python3
     )
     tdnf "${TDNF_OPTIONS[@]}" install "${packages[@]}" &&
-    pip install uv==0.11.19
+    bash "$SCRIPT_DIR/install_uv.sh" /usr/local/bin
 }
 
 retry "Source control dependencies" install_source_control

scripts/setup-dev-al4.sh

@@ -4,6 +4,8 @@
 
 set -exo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
 retry() {
     local description=$1
     shift
@@ -42,7 +44,6 @@ install_dev_dependencies() {
     # is installed by setup-ci-al4.sh; here we add the developer-only tools.
     dnf -y install  \
         clang-tools-extra  \
-        python3-pip  \
         kernel-tools  \
         jq  \
         tar
@@ -55,9 +56,12 @@ install_lts_test_dependencies() {
 
 install_python_tools() {
     # Match the clang-format version used by the existing formatting checks.
-    if ! python3 -m pip install gersemi clang-format==18.1.8 --break-system-packages; then
-        python3 -m pip install gersemi clang-format==18.1.8
-    fi
+    bash "$SCRIPT_DIR/install_uv.sh" /usr/local/bin || return 1
+    uv pip install \
+        --system \
+        --break-system-packages \
+        gersemi \
+        clang-format==18.1.8
 
     local clang_format
     clang_format=$(PATH="/usr/local/bin:$PATH" command -v clang-format)

scripts/setup-dev.sh

@@ -4,6 +4,8 @@
 
 set -exo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
 retry() {
     local description=$1
     shift
@@ -40,7 +42,6 @@ retry() {
 install_dev_dependencies() {
     tdnf -y install  \
         clang-tools-extra  \
-        python-pip \
         kernel-tools \
         jq \
         tar \
@@ -53,7 +54,8 @@ install_lts_test_dependencies() {
 }
 
 install_python_tools() {
-    pip install gersemi
+    bash "$SCRIPT_DIR/install_uv.sh" /usr/local/bin || return 1
+    uv pip install --system --break-system-packages gersemi
 }
 
 retry "Development dependencies" install_dev_dependencies

scripts/setup-ubuntu-ci-checks.sh

@@ -12,6 +12,8 @@
 
 set -euo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
 log() {
   echo "-=[ $* ]=-"
 }
@@ -40,7 +42,6 @@ install_packages_ubuntu() {
     gawk \
     findutils \
     python3 \
-    python3-pip \
     npm \
     jq \
     clang-format-18
@@ -54,16 +55,8 @@ install_packages_ubuntu() {
 # uv provides the isolated Python tool runtime (uvx) used by black, ruff, mypy,
 # gersemi and openapi-spec-validator.
 install_uv() {
-  if command -v uv >/dev/null 2>&1; then
-    log "uv already installed ($(uv --version))"
-    return
-  fi
-  log "Installing uv from PyPI"
-  # Ubuntu 24 may enforce externally-managed Python environments. Try with
-  # --break-system-packages first, then fall back for distros that don't need it.
-  if ! $SUDO python3 -m pip install --upgrade uv --break-system-packages; then
-    $SUDO python3 -m pip install --upgrade uv
-  fi
+  log "Installing pinned uv release"
+  $SUDO bash "$SCRIPT_DIR/install_uv.sh" /usr/local/bin
 }
 
 install_packages_ubuntu

tests/ci/ccf_caci_ci

@@ -6,6 +6,6 @@ RUN tdnf -y install ca-certificates git
 RUN tdnf -y install wget tar dotnet-sdk-8.0
 ENV RUNNER_ALLOW_RUNASROOT=true
 
-COPY scripts/setup-ci.sh /tmp/setup-ci.sh
+COPY scripts/setup-ci.sh scripts/install_uv.sh /tmp/
 RUN chmod +x /tmp/setup-ci.sh
 RUN /tmp/setup-ci.sh

tests/recovery_benchmark.sh

@@ -38,7 +38,7 @@ while [ "$1" != "" ]; do
     shift
 done
 
-set -e
+set -eo pipefail
 
 function service_http_status()
 {
@@ -96,9 +96,10 @@ if poll_for_service_open ${network_live_time} ${sandbox_pid}; then
 fi
 
 echo "** Load service"
-python3 -m venv .recovery_bench_env
+python3 -m venv --without-pip .recovery_bench_env
+bash "${ccf_install_path}"/bin/install_uv.sh "$PWD"/.recovery_bench_env/bin
 source .recovery_bench_env/bin/activate
-python -m pip -q install locust
+uv pip install -q locust
 
 locust --headless --locustfile ../tests/infra/locust_file.py --ca ./workspace/sandbox_common/service_cert.pem --key ./workspace/sandbox_common/user0_privk.pem --cert ./workspace/sandbox_common/user0_cert.pem --spawn-rate 100 --users 100 --rate 1000000 --node-host https://127.0.0.1:8000 --host https://0.0.0.0 --run-time "${load_run_time_s}s"