diff --git a/.github/workflows/vfox-test.yaml b/.github/workflows/vfox-test.yaml index 9969e1d..062e553 100644 --- a/.github/workflows/vfox-test.yaml +++ b/.github/workflows/vfox-test.yaml @@ -63,6 +63,30 @@ jobs: exit 1 fi + - name: Install Python 3.10.20 with uv-build + if: matrix.mirror == '' + env: + VFOX_PYTHON_USE_UV_BUILD: "1" + run: | + eval "$(vfox activate bash)" + vfox install python@3.10.20 + vfox use -g python@3.10.20 + vfox current + + - name: Check uv-build python 3.10.20 + if: matrix.mirror == '' + run: | + eval "$(vfox activate bash)" + python_version=$(python -c 'import sys;print(sys.version)') + echo $python_version + + if [[ ! $python_version == 3.10.20* ]]; then + exit 1 + fi + + python -m pip --version + pip --version + test-on-windows: runs-on: ${{ matrix.os }} strategy: @@ -118,3 +142,44 @@ jobs: if ($python_version -notlike "${{ matrix.python-version }}*") { exit 1 } + + - name: Install Python 3.10.20 with uv-build + if: runner.os == 'Windows' && matrix.mirror == '' + env: + VFOX_PYTHON_USE_UV_BUILD: "1" + run: | + Invoke-Expression "$(vfox activate pwsh)" + vfox install python@3.10.20 + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + vfox use -g python@3.10.20 + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + vfox current + $python_version = $(python -c 'import sys;print(sys.version)') + if ($LASTEXITCODE -ne 0) { + Write-Error "failed to run python after uv-build install" + exit $LASTEXITCODE + } + Write-Output $python_version + + if ($python_version -notlike "3.10.20*") { + Write-Error "expected uv-build python 3.10.20, got $python_version" + exit 1 + } + + - name: Check uv-build python 3.10.20 + if: runner.os == 'Windows' && matrix.mirror == '' + run: | + Invoke-Expression "$(vfox activate pwsh)" + $python_version = $(python -c 'import sys;print(sys.version)') + Write-Output $python_version + + if ($python_version -notlike "3.10.20*") { + exit 1 + } + + python -m pip --version + pip --version diff --git a/README.md b/README.md index ae6ea4d..7710626 100644 --- a/README.md +++ b/README.md @@ -20,3 +20,18 @@ is `https://www.python.org/ftp/python/`. ```bash export VFOX_PYTHON_MIRROR=https://mirrors.huaweicloud.com/python/ ``` + +## uv-build + +Set `VFOX_PYTHON_USE_UV_BUILD=1` to install prebuilt Python archives from the +vfox vault uv-build endpoint instead of building from pyenv/python-build. + +On Linux, libc is detected automatically. Set `VFOX_PYTHON_UV_LIBC=gnu` or +`VFOX_PYTHON_UV_LIBC=musl` to override detection. + +Set `VFOX_PYTHON_UV_BUILD_MIRROR` to download uv-build archives from a mirror. +For example: + +```bash +export VFOX_PYTHON_UV_BUILD_MIRROR=https://registry.npmmirror.com/-/binary/python-build-standalone/ +``` diff --git a/hooks/available.lua b/hooks/available.lua index 2d8f2b2..7806bab 100644 --- a/hooks/available.lua +++ b/hooks/available.lua @@ -1,8 +1,12 @@ require("util") function PLUGIN:Available(ctx) + if useUvBuild() then + return parseVersionFromUvBuild() + end + if OS_TYPE == "windows" then return parseVersion() else return parseVersionFromPyenv() end -end \ No newline at end of file +end diff --git a/hooks/post_install.lua b/hooks/post_install.lua index 17c3b56..10d24bc 100644 --- a/hooks/post_install.lua +++ b/hooks/post_install.lua @@ -1,5 +1,9 @@ require("util") function PLUGIN:PostInstall(ctx) + if useUvBuild() then + return uvBuildInstall(ctx) + end + if OS_TYPE == "windows" then return windowsInstall(ctx) else diff --git a/hooks/pre_install.lua b/hooks/pre_install.lua index c08dd30..6bf1514 100644 --- a/hooks/pre_install.lua +++ b/hooks/pre_install.lua @@ -6,6 +6,10 @@ function PLUGIN:PreInstall(ctx) version = self:Available({})[1].version end + if useUvBuild() then + return uvBuildPreInstall(version) + end + if OS_TYPE == "windows" and not checkIsReleaseVersion(version) then error("The current version is not released") return diff --git a/hooks/pre_use.lua b/hooks/pre_use.lua index 075aafc..cd5f7ec 100644 --- a/hooks/pre_use.lua +++ b/hooks/pre_use.lua @@ -34,7 +34,7 @@ function PLUGIN:PreUse(ctx) } end - local installPath = targetVersion.path + local installPath = resolvePythonInstallPath(targetVersion.path, version) print("Checking Python installation at: " .. installPath) -- Perform health check diff --git a/lib/util.lua b/lib/util.lua index c8ce0b9..c298612 100644 --- a/lib/util.lua +++ b/lib/util.lua @@ -12,12 +12,22 @@ if VFOX_PYTHON_MIRROR then end local version_vault_url = "https://vault.vfox.dev/python/pyenv" +local uv_build_vault_url = "https://vault.vfox.dev/python/uv-build" +local UV_BUILD_GITHUB_RELEASE_PATTERN = "/releases/download/([^/]+)/([^/]+)$" +local URL_ENCODED_DOT = "%%2[eE]" -- request headers local REQUEST_HEADERS = { ["User-Agent"] = "vfox" } +-- pip.cmd lives under Scripts, so %~dp0..\python.exe resolves to the install root's python.exe. +-- Windows command scripts conventionally use CRLF line endings. +local WINDOWS_PIP_SHIM_CONTENT = "@echo off\r\n\"%~dp0..\\python.exe\" -m pip %*\r\n" + +local UV_BUILD_ENV = "VFOX_PYTHON_USE_UV_BUILD" +local UV_BUILD_MIRROR_ENV = "VFOX_PYTHON_UV_BUILD_MIRROR" + -- download source local DOWNLOAD_SOURCE = { MSI = PYTHON_URL .. "%s/python-%s%s.msi", @@ -213,6 +223,488 @@ end local pyenvBranch = "" +function useUvBuild() + local value = os.getenv(UV_BUILD_ENV) + if value == nil then + return false + end + value = string.lower(value) + return value == "1" or value == "true" or value == "yes" or value == "on" +end + +local function containsTraversalSegment(value) + local normalizedValue = string.gsub(value, "\\", "/") + if string.find(normalizedValue, URL_ENCODED_DOT) then + return true + end + for segment in string.gmatch(normalizedValue, "[^/]+") do + if segment == ".." then + return true + end + end + return false +end + +local function findUnsupportedControlCharacter(value) + local firstControlCharPos = nil + for _, char in ipairs({ "\r", "\n", string.char(0) }) do + local position = string.find(value, char, 1, true) + if position and (firstControlCharPos == nil or position < firstControlCharPos) then + firstControlCharPos = position + end + end + return firstControlCharPos +end + +local function shellQuote(value) + local controlCharStart = findUnsupportedControlCharacter(value) + if controlCharStart then + error("Path contains unsupported control character at position " .. controlCharStart) + end + if containsTraversalSegment(value) then + error("Path contains unsupported traversal segment: " .. value) + end + + if RUNTIME.osType == "windows" or OS_TYPE == "windows" then + if string.find(value, '"', 1, true) then + error("Path contains unsupported quote character: " .. value) + end + return '"' .. value .. '"' + end + + return "'" .. string.gsub(value, "'", "'\\''") .. "'" +end + +local function powerShellQuote(value) + local controlCharStart = findUnsupportedControlCharacter(value) + if controlCharStart then + error("PowerShell argument contains unsupported control character at position " .. controlCharStart) + end + if containsTraversalSegment(value) then + error("PowerShell argument contains unsupported traversal segment: " .. value) + end + -- The generated script is passed to powershell through cmd as a double-quoted -Command argument. + if string.find(value, '"', 1, true) then + error("PowerShell argument contains double quote which conflicts with -Command wrapper: " .. value) + end + -- PowerShell single-quoted strings escape embedded single quotes by doubling them. + return "'" .. string.gsub(value, "'", "''") .. "'" +end + +local function powerShellCommand(script) + -- Windows PowerShell is available by default on supported Windows targets. + return "powershell -NoProfile -NonInteractive -ExecutionPolicy RemoteSigned -Command " .. shellQuote(script) +end + +local function powerShellPythonCommand(pythonExe, pythonArgs) + local scriptParts = { "&", powerShellQuote(pythonExe) } + for _, arg in ipairs(pythonArgs) do + table.insert(scriptParts, powerShellQuote(arg)) + end + return powerShellCommand(table.concat(scriptParts, " ")) +end + +local function startsWith(value, prefix) + return string.sub(value, 1, string.len(prefix)) == prefix +end + +local function trimTrailingSlash(value) + return string.gsub(value, "/+$", "") +end + +local function isNilOrEmpty(value) + return value == nil or value == "" +end + +local function isHttpsUrl(value) + return type(value) == "string" and startsWith(value, "https://") and not string.find(value, "[\r\n%z]") +end + +local function commandSucceeded(status) + return status == 0 or status == true +end + +local function startsWithPath(value, prefix) + if value == prefix then + return true + end + local nextChar = string.sub(value, string.len(prefix) + 1, string.len(prefix) + 1) + return startsWith(value, prefix) and (nextChar == "/" or nextChar == "\\") +end + +local function runtimeOs() + local osType = RUNTIME.osType or OS_TYPE + osType = string.lower(osType or "") + if osType == "darwin" or osType == "macos" then + return "darwin" + elseif osType == "windows" then + return "windows" + elseif osType == "linux" then + return "linux" + end + return osType +end + +local function runtimeArch() + local archType = string.lower(RUNTIME.archType or "") + if archType == "amd64" or archType == "x64" or archType == "x86_64" then + return "x86_64" + elseif archType == "arm64" or archType == "aarch64" then + return "aarch64" + elseif archType == "armv7" or archType == "armv7l" then + return "armv7" + elseif archType == "386" or archType == "i386" or archType == "x86" then + return "x86" + end + return archType +end + +local function runtimeLibc(osType) + local configuredLibc = os.getenv("VFOX_PYTHON_UV_LIBC") + if configuredLibc and configuredLibc ~= "" then + return configuredLibc + end + + if osType ~= "linux" then + return "none" + end + + local handle = io.popen("ldd --version 2>&1") + if handle then + local output = handle:read("*a") or "" + handle:close() + if string.find(string.lower(output), "musl", 1, true) then + return "musl" + end + else + print("Warning: Could not run ldd while detecting libc") + end + + local muslLibs = { + "/lib/ld-musl-x86_64.so.1", + "/lib/ld-musl-aarch64.so.1", + "/lib/ld-musl-armhf.so.1", + "/usr/lib/libc.musl-x86_64.so.1", + "/usr/lib/libc.musl-aarch64.so.1", + "/usr/lib/libc.musl-armhf.so.1" + } + for _, muslLib in ipairs(muslLibs) do + local file = io.open(muslLib, "r") + if file then + file:close() + return "musl" + end + end + + local gnuCheck = io.popen("getconf GNU_LIBC_VERSION 2>/dev/null") + if gnuCheck then + local output = gnuCheck:read("*a") or "" + gnuCheck:close() + if output ~= "" then + return "gnu" + end + end + + print("Warning: Could not detect libc, using gnu as default. Set VFOX_PYTHON_UV_LIBC to override.") + return "gnu" +end + +local function getUvBuildPlatform() + local osType = runtimeOs() + return osType, runtimeArch(), runtimeLibc(osType) +end + +local function buildUvBuildUrl(osType, archType, libc) + if osType == nil or archType == nil or libc == nil then + osType, archType, libc = getUvBuildPlatform() + end + local query = "?os=" .. osType .. "&arch=" .. archType + if libc ~= nil and libc ~= "" and libc ~= "none" then + query = query .. "&libc=" .. libc + end + return uv_build_vault_url .. query +end + +local function uvBuildVersion(build) + if build.display_version ~= nil then + return build.display_version + end + if build.version == nil then + return nil + end + if build.variant == "freethreaded" then + return build.version .. "t" + end + return build.version +end + +local function isSupportedUvBuild(build) + local implementation = build.implementation or build.name + if implementation ~= "cpython" then + return false + end + if build.version == nil then + return false + end + if build.arch and build.arch.variant ~= nil then + return false + end + if build.variant ~= nil and build.variant ~= "default" and build.variant ~= "freethreaded" then + return false + end + if not isHttpsUrl(build.url) then + return false + end + return true +end + +local function uvBuildSha256(build) + if type(build.asset) ~= "table" or type(build.asset.sha256) ~= "string" then + return nil + end + + local sha256 = string.lower(build.asset.sha256) + if string.match(sha256, "^[0-9a-f]+$") and string.len(sha256) == 64 then + return sha256 + end + + error("Invalid uv-build sha256 for " .. build.url) +end + +local function uvBuildAssetName(build) + if type(build.filename) == "string" then + return string.lower(build.filename) + end + if type(build.url) == "string" then + return string.lower(build.url) + end + return "" +end + +local function isSupportedArchiveName(name) + return string.sub(name, -7) == ".tar.gz" or + string.sub(name, -4) == ".tgz" or + string.sub(name, -7) == ".tar.xz" or + string.sub(name, -8) == ".tar.bz2" or + string.sub(name, -4) == ".zip" or + string.sub(name, -3) == ".7z" +end + +local function uvBuildAssetPriority(build) + local name = uvBuildAssetName(build) + if not isSupportedArchiveName(name) then + return 90 + end + if string.find(name, "install_only_stripped", 1, true) then + return 10 + end + if string.find(name, "install_only", 1, true) then + return 20 + end + if string.find(name, "pgo+lto-full", 1, true) or string.find(name, "pgo%2blto-full", 1, true) then + return 30 + end + if not string.find(name, "debug", 1, true) then + return 40 + end + return 90 +end + +local function uvBuildDownloadUrl(build) + if not isHttpsUrl(build.url) then + error("Invalid uv-build download URL") + end + + local mirror = os.getenv(UV_BUILD_MIRROR_ENV) + if mirror == nil or mirror == "" then + return build.url + end + if not isHttpsUrl(mirror) then + error(UV_BUILD_MIRROR_ENV .. " must be an https URL") + end + + local release, filename = string.match(build.url, UV_BUILD_GITHUB_RELEASE_PATTERN) + if release == nil or filename == nil then + error("Unable to rewrite uv-build download URL for mirror; expected a GitHub release download URL: " .. build.url) + end + + return trimTrailingSlash(mirror) .. "/" .. release .. "/" .. filename +end + +local function getUvBuilds(osType, archType, libc) + fixHeaders() + local resp, err = http.get({ + url = buildUvBuildUrl(osType, archType, libc), + headers = REQUEST_HEADERS + }) + if err ~= nil or resp.status_code ~= 200 then + local statusCode = resp and resp.status_code or "none" + error("parsing uv-build release info failed. Status: " .. statusCode .. ", Error: " .. (err or "none")) + end + + local jsonObj = json.decode(resp.body) + return jsonObj.items or jsonObj.versions or {} +end + +local function findUvBuild(version) + local osType, archType, libc = getUvBuildPlatform() + local selectedBuild = nil + local selectedPriority = nil + for _, build in ipairs(getUvBuilds(osType, archType, libc)) do + if isSupportedUvBuild(build) and uvBuildVersion(build) == version then + local priority = uvBuildAssetPriority(build) + if selectedBuild == nil or priority < selectedPriority then + selectedBuild = build + selectedPriority = priority + end + end + end + if selectedBuild ~= nil then + return selectedBuild + end + error("No uv-build prebuilt Python found for version " .. version .. " on " .. osType .. "/" .. archType .. "/" .. libc) +end + +local function pathExists(path) + local file = io.open(path, "r") + if file then + file:close() + return true + end + return false +end + +local function ensureWindowsDirectory(path) + local command = powerShellCommand("New-Item -ItemType Directory -Force -Path " .. powerShellQuote(path) .. " | Out-Null") + local exitCode = os.execute(command) + if not commandSucceeded(exitCode) then + error("Failed to create directory: " .. path .. ". Exit code: " .. tostring(exitCode)) + end +end + +local function writeWindowsFile(path, content) + local file, err = io.open(path, "w") + if not file then + error("Failed to write file: " .. path .. ". Error: " .. (err or "unknown")) + end + local ok, writeErr = pcall(function() + file:write(content) + end) + file:close() + if not ok then + error("Failed to write file: " .. path .. ". Error: " .. tostring(writeErr)) + end +end + +local function createWindowsPipShim(scriptsPath) + ensureWindowsDirectory(scriptsPath) + + -- pip and pip3 cover the command names exposed by vfox CI and common Windows usage. + local shims = { "pip.cmd", "pip3.cmd" } + for _, shim in ipairs(shims) do + writeWindowsFile(scriptsPath .. "\\" .. shim, WINDOWS_PIP_SHIM_CONTENT) + end +end + +local function isPipCommandAvailable(scriptsPath) + return pathExists(scriptsPath .. "\\pip.exe") or pathExists(scriptsPath .. "\\pip.cmd") +end + +local function ensureWindowsUvBuildPip(path) + if runtimeOs() ~= "windows" then + return + end + + local pythonExe = path .. "\\python.exe" + local scriptsPath = path .. "\\Scripts" + if not pathExists(pythonExe) then + error("Cannot install pip: python.exe was not found at " .. pythonExe) + end + -- If Scripts does not exist yet, pathExists returns false and setup continues. + if isPipCommandAvailable(scriptsPath) then + return + end + + if not pathExists(path .. "\\Lib\\ensurepip\\__init__.py") then + print("Warning: uv-build Python does not include ensurepip; pip will not be available.") + return + end + + print("Installing pip for uv-build Python on Windows...") + local command = powerShellPythonCommand(pythonExe, { "-E", "-s", "-m", "ensurepip", "-U", "--default-pip" }) + local exitCode = os.execute(command) + if not commandSucceeded(exitCode) then + error("ensurepip failed while installing pip. Exit code: " .. tostring(exitCode)) + end + + if isPipCommandAvailable(scriptsPath) then + return + end + + local windowsBundledPath = path .. "\\Lib\\ensurepip\\_bundled" + local reinstallCommand = powerShellPythonCommand(pythonExe, { + "-E", "-s", "-m", "pip", "install", "--force-reinstall", "--no-index", + "--find-links", windowsBundledPath, "pip" + }) + local reinstallExitCode = os.execute(reinstallCommand) + if not commandSucceeded(reinstallExitCode) then + error("pip force-reinstall step failed before shim creation. Exit code: " .. tostring(reinstallExitCode)) + end + + local verifyCommand = powerShellPythonCommand(pythonExe, { "-E", "-s", "-m", "pip", "--version" }) + local verifyExitCode = os.execute(verifyCommand) + if not commandSucceeded(verifyExitCode) then + error("pip module is not available after installation attempts. Exit code: " .. tostring(verifyExitCode)) + end + + if not isPipCommandAvailable(scriptsPath) then + createWindowsPipShim(scriptsPath) + end +end + +function resolvePythonInstallPath(installPath, version) + if pathExists(installPath .. "/bin") or pathExists(installPath .. "\\python.exe") then + return installPath + end + + -- vfox stores SDK payloads under python-; uv-build archives unpack into that payload directory. + local uvBuildPath = installPath .. "/python-" .. version + if pathExists(uvBuildPath .. "/bin") or pathExists(uvBuildPath .. "\\python.exe") then + return uvBuildPath + end + + return installPath +end + +function uvBuildPreInstall(version) + local ok, value = pcall(function() + local build = findUvBuild(version) + -- Return url/sha256 so vfox core performs download, checksum verification, and archive extraction. + return { + version = version, + url = uvBuildDownloadUrl(build), + headers = REQUEST_HEADERS, + sha256 = uvBuildSha256(build), + note = "uv-build", + } + end) + if not ok then + local err = value + error("uv-build PreInstall failed: " .. tostring(err)) + end + local uvBuildPackage = value + if uvBuildPackage == nil then + error("uv-build PreInstall did not provide install metadata") + end + if isNilOrEmpty(uvBuildPackage.url) then + error("uv-build PreInstall failed: url is required") + end + if isNilOrEmpty(uvBuildPackage.sha256) then + error("uv-build PreInstall failed: sha256 is required") + end + return uvBuildPackage +end + function linuxCompile(ctx) local sdkInfo = ctx.sdkInfo['python'] local path = sdkInfo.path @@ -245,6 +737,32 @@ function linuxCompile(ctx) end end +function uvBuildInstall(ctx) + local sdkInfo = ctx.sdkInfo['python'] + local path = sdkInfo.path + local version = sdkInfo.version + + if not ctx.rootPath or ctx.rootPath == "" then + error("vfox root path is required for uv-build installation") + end + if not startsWithPath(path, ctx.rootPath) then + error("Install path is outside the expected vfox root path: " .. path) + end + + local extractedPath = resolvePythonInstallPath(path, version) + if not pathExists(extractedPath .. "/bin/python") and not pathExists(extractedPath .. "\\python.exe") then + error("Extracted uv-build archive does not contain a Python executable at expected location: " .. extractedPath) + end + + if OS_TYPE ~= "windows" then + fixShebangLines(extractedPath) + else + ensureWindowsUvBuildPip(extractedPath) + end + + print("Install Python uv-build success!") +end + function getReleaseForWindows(version) local archType = RUNTIME.archType local exeArchSuffix = "" @@ -398,6 +916,24 @@ function parseVersionFromPyenv() return result end +function parseVersionFromUvBuild() + local result = {} + local seen = {} + for _, build in ipairs(getUvBuilds()) do + if isSupportedUvBuild(build) then + local version = uvBuildVersion(build) + if not seen[version] then + table.insert(result, { + version = version, + note = "" + }) + seen[version] = true + end + end + end + return result +end + -- Fix shebang lines in Python scripts that point to temporary directories function fixShebangLines(installPath) return fixShebangForVersion(installPath, nil) diff --git a/metadata.lua b/metadata.lua index e06cfb9..3f9d06b 100644 --- a/metadata.lua +++ b/metadata.lua @@ -27,5 +27,8 @@ PLUGIN.notes = { "Mirror Setting:", "You can use VFOX_PYTHON_MIRROR environment variable to set mirror.", "eg: `export VFOX_PYTHON_MIRROR=https://mirrors.huaweicloud.com/python/`", + "Set `VFOX_PYTHON_USE_UV_BUILD=1` to use prebuilt Python archives from uv-build.", + "Set `VFOX_PYTHON_UV_LIBC=gnu|musl` to override Linux libc detection.", + "Set `VFOX_PYTHON_UV_BUILD_MIRROR` to mirror uv-build archive downloads.", " ", }