From ece71a84f1b5fbd6cabba8a7f60a35d9a5eca33a Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Wed, 24 Sep 2025 08:41:18 -0700 Subject: [PATCH 1/2] buildscripts: Use Java 17 to build GAE interop --- buildscripts/kokoro/gae-interop.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/buildscripts/kokoro/gae-interop.sh b/buildscripts/kokoro/gae-interop.sh index c4ce56cac52..0113708d960 100755 --- a/buildscripts/kokoro/gae-interop.sh +++ b/buildscripts/kokoro/gae-interop.sh @@ -5,6 +5,12 @@ if [[ -f /VERSION ]]; then cat /VERSION fi +update-alternatives --list java +update-alternatives --query java +java -version +export PATH="/usr/lib/jvm/java-21-openjdk-amd64/bin:$PATH" +java -version + KOKORO_GAE_SERVICE="java-gae-interop-test" # We deploy as different versions of a single service, this way any stale From 5ac1cad0b4538207e0b98e0ba6fb0256d1d06698 Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Thu, 16 Oct 2025 16:07:21 -0700 Subject: [PATCH 2/2] buildscripts: Convert GAE CI to Cloud Build The Google App Engine build now requires Java 17, because the App Engine libraries are now using Java 17 bytecode. The Kokoro environment doesn't include Java 17, and while we could make some custom pools to resolve it, it is easier to swap to Cloud Build than to fight and maintain the Kokoro images. With Cloud Build we can also restrict permissions easier, as the same workers aren't used for multiple tasks. However, the Gradle App Engine plugin doesn't support choosing a service account for GAE, so I swapped to using gcloud app deploy. Although we'll be using restricted service accounts, we'll configure Cloud Build to require a "/gcbrun" GitHub comment except for owners and collaborators of the repository, similar to the "kokoro:run" label today. I swapped the Gradle code to use project properties instead of system properties, as we really should have been using project properties to begin with and I didn't want to add new system properties. The sleep has probably been unnecessary since the turndown of GAE Java 7, when the architecture of GAE changed considerably. But today it is very possible a new instance is spun up for that request and GAE does a warmup request, so the delay seems unlikely to help anything and was excessive at 20 seconds. The Cloud Build file _doesn't_ include GAE in its name because it can do more than GAE testing and it is easy to run things in parallel in Cloud Build (although they share the worker). In particular, some of the Android tests may make sense to migrate away from Kokoro. We're using e2-standard-16 for Kokoro and it takes about 10 minutes. With the default Cloud Build worker e2-standard-2 it takes 20 minutes, and with e2-highcpu-8 it takes 10 minutes with 4 minutes spent on app deploy. The expectation is to run this with a Java-CI-specific service account, so we have configure logging ourselves. I chose CLOUD_LOGGING_ONLY because it was easy, but we'll want to configure GCS in the future to allow external contributors to see the logs. --- buildscripts/cloudbuild-testing.yaml | 64 +++++++++++++++++++++++ buildscripts/gae-build/Dockerfile | 10 ++++ buildscripts/kokoro/gae-interop.sh | 61 --------------------- gae-interop-testing/gae-jdk8/build.gradle | 20 +++---- 4 files changed, 85 insertions(+), 70 deletions(-) create mode 100644 buildscripts/cloudbuild-testing.yaml create mode 100644 buildscripts/gae-build/Dockerfile delete mode 100755 buildscripts/kokoro/gae-interop.sh diff --git a/buildscripts/cloudbuild-testing.yaml b/buildscripts/cloudbuild-testing.yaml new file mode 100644 index 00000000000..623b85b6882 --- /dev/null +++ b/buildscripts/cloudbuild-testing.yaml @@ -0,0 +1,64 @@ +substitutions: + _GAE_SERVICE_ACCOUNT: appengine-testing-java@grpc-testing.iam.gserviceaccount.com +options: + env: + - BUILD_ID=$BUILD_ID + - KOKORO_GAE_SERVICE=java-gae-interop-test + - DUMMY_DEFAULT_VERSION=dummy-default + - GRADLE_OPTS=-Dorg.gradle.jvmargs='-Xmx1g' + - GRADLE_FLAGS=-PskipCodegen=true -PskipAndroid=true + logging: CLOUD_LOGGING_ONLY + machineType: E2_HIGHCPU_8 + +steps: +- id: clean-stale-deploys + name: gcr.io/cloud-builders/gcloud + allowFailure: true + script: | + #!/usr/bin/env bash + set -e + echo "Cleaning out stale deploys from previous runs, it is ok if this part fails" + # If the test fails, the deployment is leaked. + # Delete all versions whose name is not 'dummy-default' and is older than 1 hour. + # This expression is an ISO8601 relative date: + # https://cloud.google.com/sdk/gcloud/reference/topic/datetimes + (gcloud app versions list --format="get(version.id)" \ + --filter="service=$KOKORO_GAE_SERVICE AND NOT version : '$DUMMY_DEFAULT_VERSION' AND version.createTime<'-p1h'" \ + | xargs -i gcloud app services delete "$KOKORO_GAE_SERVICE" --version {} --quiet) || true + +- name: gcr.io/cloud-builders/docker + args: ['build', '-t', 'gae-build', 'buildscripts/gae-build/'] + +- id: build + name: gae-build + script: | + #!/usr/bin/env bash + exec ./gradlew $GRADLE_FLAGS :grpc-gae-interop-testing-jdk8:appengineStage + +- id: deploy + name: gcr.io/cloud-builders/gcloud + args: + - app + - deploy + - gae-interop-testing/gae-jdk8/build/staged-app/app.yaml + - --service-account=$_GAE_SERVICE_ACCOUNT + - --no-promote + - --no-stop-previous-version + - --version=cb-$BUILD_ID + +- id: runInteropTestRemote + name: eclipse-temurin:17-jdk + env: + - PROJECT_ID=$PROJECT_ID + script: | + #!/usr/bin/env bash + exec ./gradlew $GRADLE_FLAGS --stacktrace -PgaeDeployVersion="cb-$BUILD_ID" \ + -PgaeProjectId="$PROJECT_ID" :grpc-gae-interop-testing-jdk8:runInteropTestRemote + +- id: cleanup + name: gcr.io/cloud-builders/gcloud + script: | + #!/usr/bin/env bash + set -e + echo "Performing cleanup now." + gcloud app services delete "$KOKORO_GAE_SERVICE" --version "cb-$BUILD_ID" --quiet diff --git a/buildscripts/gae-build/Dockerfile b/buildscripts/gae-build/Dockerfile new file mode 100644 index 00000000000..7e68b270801 --- /dev/null +++ b/buildscripts/gae-build/Dockerfile @@ -0,0 +1,10 @@ +FROM eclipse-temurin:17-jdk + +# The AppEngine Gradle plugin downloads and runs its own gcloud to get the .jar +# to link against, so we need Python even if we use gcloud deploy directly +# instead of using the plugin. +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends python3 && \ + rm -rf /var/lib/apt/lists/* diff --git a/buildscripts/kokoro/gae-interop.sh b/buildscripts/kokoro/gae-interop.sh deleted file mode 100755 index 0113708d960..00000000000 --- a/buildscripts/kokoro/gae-interop.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -set -exu -o pipefail -if [[ -f /VERSION ]]; then - cat /VERSION -fi - -update-alternatives --list java -update-alternatives --query java -java -version -export PATH="/usr/lib/jvm/java-21-openjdk-amd64/bin:$PATH" -java -version - -KOKORO_GAE_SERVICE="java-gae-interop-test" - -# We deploy as different versions of a single service, this way any stale -# lingering deploys can be easily cleaned up by purging all running versions -# of this service. -KOKORO_GAE_APP_VERSION=$(hostname) - -# A dummy version that can be the recipient of all traffic, so that the kokoro test version can be -# set to 0 traffic. This is a requirement in order to delete it. -DUMMY_DEFAULT_VERSION='dummy-default' - -function cleanup() { - echo "Performing cleanup now." - gcloud app services delete $KOKORO_GAE_SERVICE --version $KOKORO_GAE_APP_VERSION --quiet -} -trap cleanup SIGHUP SIGINT SIGTERM EXIT - -readonly GRPC_JAVA_DIR="$(cd "$(dirname "$0")"/../.. && pwd)" -cd "$GRPC_JAVA_DIR" - -## -## Deploy the dummy 'default' version of the service -## -GRADLE_FLAGS="--stacktrace -DgaeStopPreviousVersion=false -PskipCodegen=true -PskipAndroid=true" -export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" - -# Deploy the dummy 'default' version. We only require that it exists when cleanup() is called. -# It ok if we race with another run and fail here, because the end result is idempotent. -set +e -if ! gcloud app versions describe "$DUMMY_DEFAULT_VERSION" --service="$KOKORO_GAE_SERVICE"; then - ./gradlew $GRADLE_FLAGS -DgaeDeployVersion="$DUMMY_DEFAULT_VERSION" -DgaePromote=true :grpc-gae-interop-testing-jdk8:appengineDeploy -else - echo "default version already exists: $DUMMY_DEFAULT_VERSION" -fi -set -e - -# Deploy and test the real app (jdk8) -./gradlew $GRADLE_FLAGS -DgaeDeployVersion="$KOKORO_GAE_APP_VERSION" :grpc-gae-interop-testing-jdk8:runInteropTestRemote - -set +e -echo "Cleaning out stale deploys from previous runs, it is ok if this part fails" - -# Sometimes the trap based cleanup fails. -# Delete all versions whose name is not 'dummy-default' and is older than 1 hour. -# This expression is an ISO8601 relative date: -# https://cloud.google.com/sdk/gcloud/reference/topic/datetimes -gcloud app versions list --format="get(version.id)" --filter="service=$KOKORO_GAE_SERVICE AND NOT version : 'dummy-default' AND version.createTime<'-p1h'" | xargs -i gcloud app services delete "$KOKORO_GAE_SERVICE" --version {} --quiet -exit 0 diff --git a/gae-interop-testing/gae-jdk8/build.gradle b/gae-interop-testing/gae-jdk8/build.gradle index 14abbc05a9b..07033f403de 100644 --- a/gae-interop-testing/gae-jdk8/build.gradle +++ b/gae-interop-testing/gae-jdk8/build.gradle @@ -58,6 +58,7 @@ def createDefaultVersion() { return new java.text.SimpleDateFormat("yyyyMMdd't'HHmmss").format(new Date()) } +def nonShadowedProject = project // [START model] appengine { // App Engine tasks configuration @@ -67,13 +68,13 @@ appengine { deploy { // deploy configuration - projectId = 'GCLOUD_CONFIG' + projectId = nonShadowedProject.findProperty('gaeProjectId') ?: 'GCLOUD_CONFIG' // default - stop the current version - stopPreviousVersion = System.getProperty('gaeStopPreviousVersion') ?: true + stopPreviousVersion = nonShadowedProject.findProperty('gaeStopPreviousVersion') ?: true // default - do not make this the promoted version - promote = System.getProperty('gaePromote') ?: false - // Use -DgaeDeployVersion if set, otherwise the version is null and the plugin will generate it - version = System.getProperty('gaeDeployVersion', createDefaultVersion()) + promote = nonShadowedProject.findProperty('gaePromote') ?: false + // Use -PgaeDeployVersion if set, otherwise the version is null and the plugin will generate it + version = nonShadowedProject.findProperty('gaeDeployVersion') ?: createDefaultVersion() } } // [END model] @@ -83,6 +84,10 @@ version = '1.0-SNAPSHOT' // Version in generated output /** Returns the service name. */ String getGaeProject() { + def configuredProjectId = appengine.deploy.projectId + if (!"GCLOUD_CONFIG".equals(configuredProjectId)) { + return configuredProjectId + } def stream = new ByteArrayOutputStream() exec { executable 'gcloud' @@ -110,11 +115,8 @@ String getAppUrl(String project, String service, String version) { } tasks.register("runInteropTestRemote") { - dependsOn appengineDeploy + mustRunAfter appengineDeploy doLast { - // give remote app some time to settle down - sleep(20000) - def appUrl = getAppUrl( getGaeProject(), getService(project.getProjectDir().toPath()),