diff --git a/Build/PreBuild.ps1 b/Build/PreBuild.ps1 index fb046b57d3..13cfc5ec05 100644 --- a/Build/PreBuild.ps1 +++ b/Build/PreBuild.ps1 @@ -37,7 +37,12 @@ param ( # Run in interactive mode for azure feed authentication, defaults to false [Parameter()] - [switch] $interactive + [switch] $interactive, + + # Preserve the repo's node_modules folder (skip deletion before npm install). + # Intended for CI where node_modules may be restored from a pipeline cache. + [Parameter()] + [switch] $preserveNodeModules ) $ErrorActionPreference = "Stop" @@ -167,9 +172,13 @@ try { # delete pylance install folder to blow away local changes $nodeModulesPath = Join-Path $buildroot "node_modules" - if (Test-Path -Path $nodeModulesPath) { - Remove-Item -Recurse -Force $nodeModulesPath - } + if (-not $preserveNodeModules) { + if (Test-Path -Path $nodeModulesPath) { + Remove-Item -Recurse -Force $nodeModulesPath + } + } else { + "Preserving node_modules" + } # Install pylance version specified in package.json npm install diff --git a/Build/install_pypi_package.py b/Build/install_pypi_package.py index 6f51fb6084..1fc1bd1e07 100644 --- a/Build/install_pypi_package.py +++ b/Build/install_pypi_package.py @@ -11,6 +11,8 @@ import os import urllib.request as url_lib import zipfile +import urllib.parse +import hashlib import ssl import certifi @@ -55,17 +57,44 @@ def download_and_extract(installDir, url, version): if (installDir is None or installDir == "."): installDir = os.getcwd() - context = ssl.create_default_context(cafile=certifi.where()) - with url_lib.urlopen(url, context=context) as response: - data = response.read() - with zipfile.ZipFile(io.BytesIO(data), "r") as wheel: - for zip_info in wheel.infolist(): - - # Ignore dist info since we are merging multiple wheels - if ".dist-info/" in zip_info.filename: - continue - - wheel.extract(zip_info.filename, installDir) + cache_dir = os.environ.get("PTVS_PYPI_WHEEL_CACHE_DIR") + cache_path = None + if cache_dir: + try: + os.makedirs(cache_dir, exist_ok=True) + url_path = urllib.parse.urlparse(url).path + filename = os.path.basename(url_path) or hashlib.sha256(url.encode("utf-8")).hexdigest() + cache_path = os.path.join(cache_dir, filename) + except Exception: + cache_path = None + + data = None + if cache_path and os.path.exists(cache_path): + try: + with open(cache_path, "rb") as f: + data = f.read() + except Exception: + data = None + + if data is None: + context = ssl.create_default_context(cafile=certifi.where()) + with url_lib.urlopen(url, context=context) as response: + data = response.read() + if cache_path: + try: + with open(cache_path, "wb") as f: + f.write(data) + except Exception: + pass + + with zipfile.ZipFile(io.BytesIO(data), "r") as wheel: + for zip_info in wheel.infolist(): + + # Ignore dist info since we are merging multiple wheels + if ".dist-info/" in zip_info.filename: + continue + + wheel.extract(zip_info.filename, installDir) # parse the command line args and return them diff --git a/Build/templates/restore_packages.yml b/Build/templates/restore_packages.yml index cbd5cf16a2..193f184ff0 100644 --- a/Build/templates/restore_packages.yml +++ b/Build/templates/restore_packages.yml @@ -17,19 +17,102 @@ parameters: steps: + # Point NuGet HTTP cache to a pipeline-controlled location so we can cache it between runs. + # packages.config restores don't use the global-packages folder, but they do use the HTTP cache + # to avoid re-downloading packages from the feed. + - powershell: | + $cachePath = Join-Path "$(Pipeline.Workspace)" "nuget-http-cache" + Write-Host "##vso[task.setvariable variable=NUGET_HTTP_CACHE_PATH]$cachePath" + displayName: 'Set NuGet cache path' + + # Restore NuGet HTTP cache from previous runs (key = packages.config content hash) + - task: Cache@2 + displayName: 'Cache NuGet HTTP cache' + inputs: + key: 'nuget | "$(Agent.OS)" | Build/$(VSTarget)/packages.config' + restoreKeys: | + nuget | "$(Agent.OS)" + path: '$(Pipeline.Workspace)/nuget-http-cache' + # nuget authenticate so we can restore from azure artifacts - task: NuGetAuthenticate@1 displayName: 'NuGet Authenticate' + # Warm the packages folder using the cached NuGet HTTP cache so the subsequent + # restore in PreBuild.ps1 is largely a no-op. + - task: PowerShell@2 + displayName: 'NuGet restore (warm packages)' + env: + NUGET_HTTP_CACHE_PATH: $(NUGET_HTTP_CACHE_PATH) + inputs: + targetType: inline + workingDirectory: '$(Build.SourcesDirectory)/Build' + script: | + Write-Host "NuGet version:" + .\nuget.exe help | Select-Object -First 1 + + Write-Host "NuGet HTTP cache path: $env:NUGET_HTTP_CACHE_PATH" + try { + .\nuget.exe locals http-cache -list + } catch { + Write-Host "(Unable to query NuGet http-cache location: $($_.Exception.Message))" + } + + .\nuget.exe restore "$(VSTarget)\packages.config" -OutputDirectory "$(Build.BinariesDirectory)" -Config nuget.config -NonInteractive + # npm authenticate so we can restore from azure artifacts - task: npmAuthenticate@0 displayName: 'npm Authenticate' inputs: workingFile: .npmrc + # Cache node_modules to avoid re-installing Pylance dependencies every run. + # PreBuild.ps1 will be instructed to preserve node_modules when running in CI. + - task: Cache@2 + displayName: 'Cache node_modules' + inputs: + key: 'node_modules | "$(Agent.OS)" | package.json | ${{ parameters.pylanceVersion }} | ${{ parameters.pylanceReleaseType }}' + restoreKeys: | + node_modules | "$(Agent.OS)" | package.json + node_modules | "$(Agent.OS)" + path: '$(Build.SourcesDirectory)\\node_modules' + + # Cache npm downloads to speed up Pylance install (PreBuild.ps1 runs `npm install`) + - task: Cache@2 + displayName: 'Cache npm' + inputs: + key: 'npm | "$(Agent.OS)" | package.json | ${{ parameters.pylanceVersion }} | ${{ parameters.pylanceReleaseType }}' + restoreKeys: | + npm | "$(Agent.OS)" | package.json + npm | "$(Agent.OS)" + path: '$(Pipeline.Workspace)\\npm-cache' + + # Cache PyPI wheel downloads used by install_pypi_package.py (debugpy/etwtrace) + - task: Cache@2 + displayName: 'Cache PyPI wheels' + inputs: + key: 'pypi-wheels | "$(Agent.OS)" | ${{ parameters.debugpyVersion }}' + restoreKeys: | + pypi-wheels | "$(Agent.OS)" + path: '$(Pipeline.Workspace)\\pypi-wheel-cache' + # Restore packages and install dependencies (pylance, debugpy) - task: PowerShell@1 displayName: 'Restore packages' + env: + # Ensure nuget.exe (spawned by PreBuild.ps1) inherits the pipeline-controlled HTTP cache path. + # This makes cache usage explicit even if task variable->env propagation changes. + NUGET_HTTP_CACHE_PATH: $(NUGET_HTTP_CACHE_PATH) + + # Speed up npm install for Pylance by reusing cached downloads. + npm_config_cache: $(Pipeline.Workspace)\\npm-cache + npm_config_prefer_offline: 'true' + npm_config_audit: 'false' + npm_config_fund: 'false' + npm_config_progress: 'false' + + # Speed up debugpy/etwtrace installs by caching downloaded wheels. + PTVS_PYPI_WHEEL_CACHE_DIR: $(Pipeline.Workspace)\\pypi-wheel-cache inputs: scriptName: Build/PreBuild.ps1 - arguments: '-vstarget $(VSTarget) -pylanceVersion ${{ parameters.pylanceVersion }} -pylanceReleaseType ${{ parameters.pylanceReleaseType }} -debugpyVersion ${{ parameters.debugpyVersion }}' \ No newline at end of file + arguments: '-vstarget $(VSTarget) -pylanceVersion ${{ parameters.pylanceVersion }} -pylanceReleaseType ${{ parameters.pylanceReleaseType }} -debugpyVersion ${{ parameters.debugpyVersion }} -preserveNodeModules' \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 73880380c5..4d7395a4ec 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -178,12 +178,6 @@ extends: targetPath: '$(Build.BinariesDirectory)/raw/binaries' artifactName: Binaries sbomEnabled: false - - output: pipelineArtifact - displayName: 'Publish build artifact: raw' - condition: failed() - targetPath: '$(Build.BinariesDirectory)\raw' - artifactName: raw - sbomEnabled: false - output: pipelineArtifact displayName: 'Publish build artifact: logs' condition: succeededOrFailed() @@ -200,11 +194,6 @@ extends: targetPath: '$(Build.SourcesDirectory)/Python/Tests/TestData/default.runsettings' artifactName: 'RunSettings' sbomEnabled: false - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - - output: pipelineArtifact - displayName: 'Publish build artifact: BoM' - targetPath: '$(Build.BinariesDirectory)\layout' - artifactName: SBOM steps: @@ -235,47 +224,8 @@ extends: # publish symbols - template: /Build/templates/publish_symbols.yml@self - # MicroBuild cleanup - - task: MicroBuildCleanup@1 - displayName: 'Execute cleanup tasks' - condition: succeededOrFailed() - - # Publish test data - - template: /Build/templates/publish_test_data.yml@self - - # Installer job: builds installer, uploads vsts drop, and creates bootstrapper - # Runs in parallel with the test job after the build job completes - - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.buildInstaller, true)) }}: - - job: installer - displayName: Installer & Bootstrapper - dependsOn: build - steps: - - checkout: self - clean: true - fetchDepth: 1 - - # install microbuild plugins needed for swixproj/vsmanproj - - template: /Build/templates/install_microbuild_plugins.yml@self - - # Restore packages - - template: /Build/templates/restore_packages.yml@self - parameters: - pylanceVersion: ${{ parameters.pylanceVersion }} - pylanceReleaseType: ${{ variables.pylanceReleaseTypeVar }} - debugpyVersion: ${{ parameters.debugpyVersion }} - - # Download binaries from the build job - - download: current - artifact: Binaries - displayName: 'Download build binaries' - - # Copy binaries to expected location - - powershell: | - $source = '$(Pipeline.Workspace)\Binaries' - $dest = '$(Build.BinariesDirectory)\raw\binaries' - New-Item -ItemType Directory -Force -Path $dest | Out-Null - Copy-Item -Path "$source\*" -Destination $dest -Recurse -Force - displayName: 'Copy binaries to expected location' + # Installer & Bootstrapper (non-PR builds, or when buildInstaller parameter is set) + - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.buildInstaller, true)) }}: # Generate the bill of materials - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 @@ -286,7 +236,7 @@ extends: PackageVersion: '$(VSTarget)' continueOnError: true - # Build the installer (product already built, only setup projects) + # Build the installer - task: MSBuild@1 displayName: 'Build installer' inputs: @@ -314,10 +264,27 @@ extends: # Create VS bootstrapper for testing - template: Build/templates/create_vs_bootstrapper.yml@self - # MicroBuild cleanup - - task: MicroBuildCleanup@1 - displayName: 'Execute cleanup tasks' - condition: succeededOrFailed() + # Build, sign, and pack NuGet package (non-PR builds, or when PublishNugetPackageAsBuildArtifact is set) + - ${{ if or(notin(variables['Build.Reason'], 'PullRequest'), eq(variables['PublishNugetPackageAsBuildArtifact'], true)) }}: + - template: /Build/templates/build_nuget_package.yml@self + parameters: + ptvsPackageVersion: ${{ variables.ptvsPackageVersionVar }} + + # Publish NuGet package as build artifact + - task: 1ES.PublishBuildArtifacts@1 + displayName: 'Publish Artifact: pkg' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)\pkg' + ArtifactName: pkg + sbomEnabled: false + + # MicroBuild cleanup + - task: MicroBuildCleanup@1 + displayName: 'Execute cleanup tasks' + condition: succeededOrFailed() + + # Publish test data + - template: /Build/templates/publish_test_data.yml@self # Run tests on mixed mode debugger - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['SkipGlassCache'], true)) }}: @@ -336,48 +303,5 @@ extends: parameters: skipGlassCache: ${{ parameters.skipGlassCache }} - # Build, sign, and publish NuGet package - - ${{ if or(notin(variables['Build.Reason'], 'PullRequest'), eq(variables['PublishNugetPackageAsBuildArtifact'], true)) }}: - - job: nuget - displayName: NuGet Package - dependsOn: build - steps: - - checkout: self - clean: true - fetchDepth: 1 - - # install microbuild plugins needed for signing - - template: /Build/templates/install_microbuild_plugins.yml@self - - # Download binaries from the build job - - download: current - artifact: Binaries - displayName: 'Download build binaries' - - # Copy binaries to expected location - - powershell: | - $source = '$(Pipeline.Workspace)\Binaries' - $dest = '$(Build.BinariesDirectory)\raw\binaries' - New-Item -ItemType Directory -Force -Path $dest | Out-Null - Copy-Item -Path "$source\*" -Destination $dest -Recurse -Force - displayName: 'Copy binaries to expected location' - - # Build, sign, and pack nuget package - - template: /Build/templates/build_nuget_package.yml@self - parameters: - ptvsPackageVersion: ${{ variables.ptvsPackageVersionVar }} - - # Publish NuGet package as build artifact - - task: 1ES.PublishBuildArtifacts@1 - displayName: 'Publish Artifact: pkg' - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\pkg' - ArtifactName: pkg - sbomEnabled: false - - # MicroBuild cleanup - - task: MicroBuildCleanup@1 - displayName: 'Execute cleanup tasks' - condition: succeededOrFailed()