Consolidate CI pipeline jobs and add NuGet caching

Build/PreBuild.ps1

@@ -37,7 +37,12 @@ param (
 
     # Run in interactive mode for azure feed authentication, defaults to false
     [Parameter()]
-    [switch] $interactive
+    [switch] $interactive,
+
+    # Preserve the repo's node_modules folder (skip deletion before npm install).
+    # Intended for CI where node_modules may be restored from a pipeline cache.
+    [Parameter()]
+    [switch] $preserveNodeModules
 )
 
 $ErrorActionPreference = "Stop"
@@ -167,9 +172,13 @@ try {
 
     # delete pylance install folder to blow away local changes
     $nodeModulesPath = Join-Path $buildroot "node_modules"
-    if (Test-Path -Path $nodeModulesPath) {
-        Remove-Item -Recurse -Force $nodeModulesPath
-    }  
+    if (-not $preserveNodeModules) {
+        if (Test-Path -Path $nodeModulesPath) {
+            Remove-Item -Recurse -Force $nodeModulesPath
+        }
+    } else {
+        "Preserving node_modules"
+    }
 
     # Install pylance version specified in package.json
     npm install

Build/install_pypi_package.py

@@ -11,6 +11,8 @@
 import os
 import urllib.request as url_lib
 import zipfile
+import urllib.parse
+import hashlib
 import ssl
 import certifi
 
@@ -55,17 +57,44 @@ def download_and_extract(installDir, url, version):
     if (installDir is None or installDir == "."):
         installDir = os.getcwd()
 
-    context = ssl.create_default_context(cafile=certifi.where())
-    with url_lib.urlopen(url, context=context) as response:
-        data = response.read()
-        with zipfile.ZipFile(io.BytesIO(data), "r") as wheel:
-            for zip_info in wheel.infolist():
-
-                # Ignore dist info since we are merging multiple wheels
-                if ".dist-info/" in zip_info.filename:
-                    continue
-
-                wheel.extract(zip_info.filename, installDir)
+    cache_dir = os.environ.get("PTVS_PYPI_WHEEL_CACHE_DIR")
+    cache_path = None
+    if cache_dir:
+        try:
+            os.makedirs(cache_dir, exist_ok=True)
+            url_path = urllib.parse.urlparse(url).path
+            filename = os.path.basename(url_path) or hashlib.sha256(url.encode("utf-8")).hexdigest()
+            cache_path = os.path.join(cache_dir, filename)
+        except Exception:
+            cache_path = None
+
+    data = None
+    if cache_path and os.path.exists(cache_path):
+        try:
+            with open(cache_path, "rb") as f:
+                data = f.read()
+        except Exception:
+            data = None
+
+    if data is None:
+        context = ssl.create_default_context(cafile=certifi.where())
+        with url_lib.urlopen(url, context=context) as response:
+            data = response.read()
+        if cache_path:
+            try:
+                with open(cache_path, "wb") as f:
+                    f.write(data)
+            except Exception:
+                pass
+
+    with zipfile.ZipFile(io.BytesIO(data), "r") as wheel:
+        for zip_info in wheel.infolist():
+
+            # Ignore dist info since we are merging multiple wheels
+            if ".dist-info/" in zip_info.filename:
+                continue
+
+            wheel.extract(zip_info.filename, installDir)
 
 
 # parse the command line args and return them

Build/templates/restore_packages.yml

@@ -17,19 +17,102 @@ parameters:
 
 steps:
 
+  # Point NuGet HTTP cache to a pipeline-controlled location so we can cache it between runs.
+  # packages.config restores don't use the global-packages folder, but they do use the HTTP cache
+  # to avoid re-downloading packages from the feed.
+  - powershell: |
+      $cachePath = Join-Path "$(Pipeline.Workspace)" "nuget-http-cache"
+      Write-Host "##vso[task.setvariable variable=NUGET_HTTP_CACHE_PATH]$cachePath"
+    displayName: 'Set NuGet cache path'
+
+  # Restore NuGet HTTP cache from previous runs (key = packages.config content hash)
+  - task: Cache@2
+    displayName: 'Cache NuGet HTTP cache'
+    inputs:
+      key: 'nuget | "$(Agent.OS)" | Build/$(VSTarget)/packages.config'
+      restoreKeys: |
+        nuget | "$(Agent.OS)"
+      path: '$(Pipeline.Workspace)/nuget-http-cache'
+
   # nuget authenticate so we can restore from azure artifacts
   - task: NuGetAuthenticate@1
     displayName: 'NuGet Authenticate'
 
+  # Warm the packages folder using the cached NuGet HTTP cache so the subsequent
+  # restore in PreBuild.ps1 is largely a no-op.
+  - task: PowerShell@2
+    displayName: 'NuGet restore (warm packages)'
+    env:
+      NUGET_HTTP_CACHE_PATH: $(NUGET_HTTP_CACHE_PATH)
+    inputs:
+      targetType: inline
+      workingDirectory: '$(Build.SourcesDirectory)/Build'
+      script: |
+        Write-Host "NuGet version:"
+        .\nuget.exe help | Select-Object -First 1
+
+        Write-Host "NuGet HTTP cache path: $env:NUGET_HTTP_CACHE_PATH"
+        try {
+          .\nuget.exe locals http-cache -list
+        } catch {
+          Write-Host "(Unable to query NuGet http-cache location: $($_.Exception.Message))"
+        }
+
+        .\nuget.exe restore "$(VSTarget)\packages.config" -OutputDirectory "$(Build.BinariesDirectory)" -Config nuget.config -NonInteractive
+
   # npm authenticate so we can restore from azure artifacts
   - task: npmAuthenticate@0
     displayName: 'npm Authenticate'
     inputs:
       workingFile: .npmrc
 
+  # Cache node_modules to avoid re-installing Pylance dependencies every run.
+  # PreBuild.ps1 will be instructed to preserve node_modules when running in CI.
+  - task: Cache@2
+    displayName: 'Cache node_modules'
+    inputs:
+      key: 'node_modules | "$(Agent.OS)" | package.json | ${{ parameters.pylanceVersion }} | ${{ parameters.pylanceReleaseType }}'
+      restoreKeys: |
+        node_modules | "$(Agent.OS)" | package.json
+        node_modules | "$(Agent.OS)"
+      path: '$(Build.SourcesDirectory)\\node_modules'
+
+  # Cache npm downloads to speed up Pylance install (PreBuild.ps1 runs `npm install`)
+  - task: Cache@2
+    displayName: 'Cache npm'
+    inputs:
+      key: 'npm | "$(Agent.OS)" | package.json | ${{ parameters.pylanceVersion }} | ${{ parameters.pylanceReleaseType }}'
+      restoreKeys: |
+        npm | "$(Agent.OS)" | package.json
+        npm | "$(Agent.OS)"
+      path: '$(Pipeline.Workspace)\\npm-cache'
+
+  # Cache PyPI wheel downloads used by install_pypi_package.py (debugpy/etwtrace)
+  - task: Cache@2
+    displayName: 'Cache PyPI wheels'
+    inputs:
+      key: 'pypi-wheels | "$(Agent.OS)" | ${{ parameters.debugpyVersion }}'
+      restoreKeys: |
+        pypi-wheels | "$(Agent.OS)"
+      path: '$(Pipeline.Workspace)\\pypi-wheel-cache'
+
   # Restore packages and install dependencies (pylance, debugpy)
   - task: PowerShell@1
     displayName: 'Restore packages'
+    env:
+      # Ensure nuget.exe (spawned by PreBuild.ps1) inherits the pipeline-controlled HTTP cache path.
+      # This makes cache usage explicit even if task variable->env propagation changes.
+      NUGET_HTTP_CACHE_PATH: $(NUGET_HTTP_CACHE_PATH)
+
+      # Speed up npm install for Pylance by reusing cached downloads.
+      npm_config_cache: $(Pipeline.Workspace)\\npm-cache
+      npm_config_prefer_offline: 'true'
+      npm_config_audit: 'false'
+      npm_config_fund: 'false'
+      npm_config_progress: 'false'
+
+      # Speed up debugpy/etwtrace installs by caching downloaded wheels.
+      PTVS_PYPI_WHEEL_CACHE_DIR: $(Pipeline.Workspace)\\pypi-wheel-cache
     inputs:
       scriptName: Build/PreBuild.ps1
-      arguments: '-vstarget $(VSTarget) -pylanceVersion ${{ parameters.pylanceVersion }} -pylanceReleaseType ${{ parameters.pylanceReleaseType }} -debugpyVersion ${{ parameters.debugpyVersion }}'
+      arguments: '-vstarget $(VSTarget) -pylanceVersion ${{ parameters.pylanceVersion }} -pylanceReleaseType ${{ parameters.pylanceReleaseType }} -debugpyVersion ${{ parameters.debugpyVersion }} -preserveNodeModules'

azure-pipelines.yml

@@ -26,7 +26,7 @@ parameters:
 - name: PublishNugetPackageAsBuildArtifact
   displayName: Publish Nuget Package As Build Artifact
   type: boolean
-  default: false
+  default: true  # temporarily enabled for PR builds to debug nuget package failures
 
 # This is the version that the PTVS package currently being built will have.
 # If this value is set to 'currentBuildNumber' will default to $(Build.BuildNumber).
@@ -178,12 +178,6 @@ extends:
             targetPath: '$(Build.BinariesDirectory)/raw/binaries'
             artifactName: Binaries
             sbomEnabled: false
-          - output: pipelineArtifact
-            displayName: 'Publish build artifact: raw'
-            condition: failed()
-            targetPath: '$(Build.BinariesDirectory)\raw'
-            artifactName: raw
-            sbomEnabled: false
           - output: pipelineArtifact
             displayName: 'Publish build artifact: logs'
             condition: succeededOrFailed()
@@ -200,11 +194,6 @@ extends:
             targetPath: '$(Build.SourcesDirectory)/Python/Tests/TestData/default.runsettings'
             artifactName: 'RunSettings'
             sbomEnabled: false
-          - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}:
-            - output: pipelineArtifact
-              displayName: 'Publish build artifact: BoM'
-              targetPath: '$(Build.BinariesDirectory)\layout'
-              artifactName: SBOM
 
         steps:
 
@@ -235,47 +224,8 @@ extends:
         # publish symbols
         - template: /Build/templates/publish_symbols.yml@self
 
-        # MicroBuild cleanup
-        - task: MicroBuildCleanup@1
-          displayName: 'Execute cleanup tasks'
-          condition: succeededOrFailed()
-
-        # Publish test data
-        - template: /Build/templates/publish_test_data.yml@self
-
-        # Installer job: builds installer, uploads vsts drop, and creates bootstrapper
-        # Runs in parallel with the test job after the build job completes
-      - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.buildInstaller, true)) }}:
-        - job: installer
-          displayName: Installer & Bootstrapper
-          dependsOn: build
-          steps:
-          - checkout: self
-            clean: true
-            fetchDepth: 1
-
-          # install microbuild plugins needed for swixproj/vsmanproj
-          - template: /Build/templates/install_microbuild_plugins.yml@self
-
-          # Restore packages
-          - template: /Build/templates/restore_packages.yml@self
-            parameters:
-              pylanceVersion: ${{ parameters.pylanceVersion }}
-              pylanceReleaseType: ${{ variables.pylanceReleaseTypeVar }}
-              debugpyVersion: ${{ parameters.debugpyVersion }}
-
-          # Download binaries from the build job
-          - download: current
-            artifact: Binaries
-            displayName: 'Download build binaries'
-
-          # Copy binaries to expected location
-          - powershell: |
-              $source = '$(Pipeline.Workspace)\Binaries'
-              $dest = '$(Build.BinariesDirectory)\raw\binaries'
-              New-Item -ItemType Directory -Force -Path $dest | Out-Null
-              Copy-Item -Path "$source\*" -Destination $dest -Recurse -Force
-            displayName: 'Copy binaries to expected location'
+        # Installer & Bootstrapper (non-PR builds, or when buildInstaller parameter is set)
+        - ${{ if or(ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.buildInstaller, true)) }}:
 
           # Generate the bill of materials
           - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
@@ -286,7 +236,7 @@ extends:
               PackageVersion: '$(VSTarget)'
             continueOnError: true
 
-          # Build the installer (product already built, only setup projects)
+          # Build the installer
           - task: MSBuild@1
             displayName: 'Build installer'
             inputs:
@@ -314,10 +264,27 @@ extends:
           # Create VS bootstrapper for testing
           - template: Build/templates/create_vs_bootstrapper.yml@self
 
-          # MicroBuild cleanup
-          - task: MicroBuildCleanup@1
-            displayName: 'Execute cleanup tasks'
-            condition: succeededOrFailed()
+        # Build, sign, and pack NuGet package (non-PR builds, or when PublishNugetPackageAsBuildArtifact is set)
+        - ${{ if or(notin(variables['Build.Reason'], 'PullRequest'), eq(variables['PublishNugetPackageAsBuildArtifact'], true)) }}:
+          - template: /Build/templates/build_nuget_package.yml@self
+            parameters:
+              ptvsPackageVersion: ${{ variables.ptvsPackageVersionVar }}
+
+          # Publish NuGet package as build artifact
+          - task: 1ES.PublishBuildArtifacts@1
+            displayName: 'Publish Artifact: pkg'
+            inputs:
+              PathtoPublish: '$(Build.ArtifactStagingDirectory)\pkg'
+              ArtifactName: pkg
+              sbomEnabled: false
+
+        # MicroBuild cleanup
+        - task: MicroBuildCleanup@1
+          displayName: 'Execute cleanup tasks'
+          condition: succeededOrFailed()
+
+        # Publish test data
+        - template: /Build/templates/publish_test_data.yml@self
 
         # Run tests on mixed mode debugger
       - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['SkipGlassCache'], true)) }}:
@@ -336,48 +303,5 @@ extends:
             parameters:
               skipGlassCache: ${{ parameters.skipGlassCache }}
 
-        # Build, sign, and publish NuGet package
-      - ${{ if or(notin(variables['Build.Reason'], 'PullRequest'), eq(variables['PublishNugetPackageAsBuildArtifact'], true)) }}:
-        - job: nuget
-          displayName: NuGet Package
-          dependsOn: build
-          steps:
-          - checkout: self
-            clean: true
-            fetchDepth: 1
-
-          # install microbuild plugins needed for signing
-          - template: /Build/templates/install_microbuild_plugins.yml@self
-
-          # Download binaries from the build job
-          - download: current
-            artifact: Binaries
-            displayName: 'Download build binaries'
-
-          # Copy binaries to expected location
-          - powershell: |
-              $source = '$(Pipeline.Workspace)\Binaries'
-              $dest = '$(Build.BinariesDirectory)\raw\binaries'
-              New-Item -ItemType Directory -Force -Path $dest | Out-Null
-              Copy-Item -Path "$source\*" -Destination $dest -Recurse -Force
-            displayName: 'Copy binaries to expected location'
-
-          # Build, sign, and pack nuget package
-          - template: /Build/templates/build_nuget_package.yml@self
-            parameters:
-              ptvsPackageVersion: ${{ variables.ptvsPackageVersionVar }}
-
-          # Publish NuGet package as build artifact
-          - task: 1ES.PublishBuildArtifacts@1
-            displayName: 'Publish Artifact: pkg'
-            inputs:
-              PathtoPublish: '$(Build.ArtifactStagingDirectory)\pkg'
-              ArtifactName: pkg
-              sbomEnabled: false
-
-          # MicroBuild cleanup
-          - task: MicroBuildCleanup@1
-            displayName: 'Execute cleanup tasks'
-            condition: succeededOrFailed()