From 9318bd3480b3f069ad83d3df8f7a8337439ea975 Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Thu, 7 Aug 2025 23:27:36 +0200 Subject: [PATCH 1/3] KAFKA-19591 solving issues with `gradlew` content: an old template version (i.e. file `unixStartScript.txt`) was being referenced --- gradlew | 38 ++++++++++++++++++++------------------ wrapper.gradle | 4 +++- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/gradlew b/gradlew index f4bb3360e17ee..39803762879c4 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -84,7 +86,7 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -112,20 +114,7 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac - -# Loop in case we encounter an error. -for attempt in 1 2 3; do - if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then - if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v8.14.3/gradle/wrapper/gradle-wrapper.jar"; then - rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" - # Pause for a bit before looping in case the server throttled us. - sleep 5 - continue - fi - fi -done - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar +CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. @@ -212,11 +201,24 @@ if "$cygwin" || "$msys" ; then fi + +# Loop in case we encounter an error. +for attempt in 1 2 3; do + if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then + if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v8.14.1/gradle/wrapper/gradle-wrapper.jar"; then + rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" + # Pause for a bit before looping in case the server throttled us. + sleep 5 + continue + fi + fi +done + # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: -# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line. @@ -224,7 +226,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" # Stop when "xargs" is not available. diff --git a/wrapper.gradle b/wrapper.gradle index d64aeab4de942..0e0d6da201123 100644 --- a/wrapper.gradle +++ b/wrapper.gradle @@ -59,13 +59,15 @@ task bootstrapWrapper() { done """.stripIndent() + String putBootstrapStringAbove = "# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script." + def wrapperScript = wrapper.scriptFile def wrapperLines = wrapperScript.readLines() wrapperScript.withPrintWriter { out -> def bootstrapWritten = false wrapperLines.each { line -> // Print the wrapper bootstrap before the first usage of the wrapper jar. - if (!bootstrapWritten && line.contains("gradle-wrapper.jar")) { + if (!bootstrapWritten && line.contains(putBootstrapStringAbove)) { out.println(bootstrapString) bootstrapWritten = true } From 4278e5621eec92eb663cb9f369c11b821622c9aa Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Thu, 21 Aug 2025 21:46:47 +0200 Subject: [PATCH 2/3] KAFKA-19174 Gradle version upgrade (8 -->> 9); build logic is refactored details: - gradle: 8.14.3 -->> 9.1.0 - https://gradle.org/whats-new/gradle-9 - https://docs.gradle.org/9.1.0/userguide/upgrading_major_version_9.html - https://github.com/gradle/gradle/releases/tag/v9.0.0 - https://github.com/gradle/gradle/releases/tag/v9.1.0 - shadow plugin: 8.3.6 -->> 8.3.9 other version upgrades: - `setup-gradle` GitHub Action: 4.3.0 -->> 4.4.4 (note: this version was tested against Gradle 9.1.0) - gradle versions plugin: 0.48.0 -->> 0.52.0 - owasp dependency check : 8.2.1 -->> 12.1.3 - spotbugs: 6.2.3 -->> 6.2.5 - spotless: 6.25.0 -->> 7.2.1 - Zinc Scala compiler: 1.9.2 -->> 1.10.8 refactoring (performed in order to accomodate Gradle 9 breaking changes): - `:core:releaseTarGz` task changes: - `preserveFileTimestamps` Tar taks property value explicitly set to `true` (default values: Gradle 8: `true`, Gradle 9: 'false') - files and folders permissions are introduced - redundant 'gradleVersion' ext property is removed (in favor of 'versions.gradle' usage) - do not fail Gradle 9 build when no test are discovered: https://docs.gradle.org/9.1.0/userguide/upgrading_major_version_9.html#test_task_fails_when_no_tests_are_discovered - `wrapper.gradle` changes: no need to check number of dots for Gradle versions >=9 anymore --- .github/actions/setup-gradle/action.yml | 2 +- build.gradle | 39 +++++++++++++++++++----- gradle/dependencies.gradle | 4 +-- gradle/wrapper/gradle-wrapper.properties | 4 +-- gradlew | 7 ++--- wrapper.gradle | 6 ++-- 6 files changed, 41 insertions(+), 21 deletions(-) diff --git a/.github/actions/setup-gradle/action.yml b/.github/actions/setup-gradle/action.yml index fe456568066e6..47a10469cb46a 100644 --- a/.github/actions/setup-gradle/action.yml +++ b/.github/actions/setup-gradle/action.yml @@ -42,7 +42,7 @@ runs: distribution: temurin java-version: ${{ inputs.java-version }} - name: Setup Gradle - uses: gradle/actions/setup-gradle@94baf225fe0a508e581a564467443d0e2379123b # v4.3.0 + uses: gradle/actions/setup-gradle@748248ddd2a24f49513d8f472f81c3a07d4d50e1 # v4.4.4 env: GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true with: diff --git a/build.gradle b/build.gradle index b472f03bcb4ee..c3cc954e4bb7a 100644 --- a/build.gradle +++ b/build.gradle @@ -29,22 +29,21 @@ buildscript { } plugins { - id 'com.github.ben-manes.versions' version '0.48.0' + id 'com.github.ben-manes.versions' version '0.52.0' id 'idea' id 'jacoco' id 'java-library' - id 'org.owasp.dependencycheck' version '8.2.1' + id 'org.owasp.dependencycheck' version '12.1.3' id 'org.nosphere.apache.rat' version "0.8.1" id "io.swagger.core.v3.swagger-gradle-plugin" version "${swaggerVersion}" - id "com.github.spotbugs" version '6.2.3' apply false + id "com.github.spotbugs" version '6.2.5' apply false id 'org.scoverage' version '8.0.3' apply false - id 'com.gradleup.shadow' version '8.3.6' apply false - id 'com.diffplug.spotless' version "6.25.0" + id 'com.gradleup.shadow' version '8.3.9' apply false + id 'com.diffplug.spotless' version "7.2.1" } ext { - gradleVersion = versions.gradle minClientJavaVersion = 11 minNonClientJavaVersion = 17 modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams:examples", ":streams-scala", ":test-common:test-common-util"] @@ -297,7 +296,7 @@ if (repo != null) { } else { rat.enabled = false } -println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $gradleVersion, Java ${JavaVersion.current()} and Scala ${versions.scala}") +println("Starting build with version $version (commit id ${commitId == null ? "null" : commitId.take(8)}) using Gradle $versions.gradle, Java ${JavaVersion.current()} and Scala ${versions.scala}") println("Build properties: ignoreFailures=$userIgnoreFailures, maxParallelForks=$maxTestForks, maxScalacThreads=$maxScalacThreads, maxTestRetries=$userMaxTestRetries") subprojects { @@ -328,6 +327,10 @@ subprojects { tasks.register('uploadArchives').configure { dependsOn(publish) } } + tasks.withType(AbstractTestTask).configureEach { + failOnNoDiscoveredTests = false + } + // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder. if (!project.name.equals('connect')) { apply plugin: 'eclipse' @@ -1219,6 +1222,7 @@ project(':core') { from project.file("$rootDir/docs") into 'site-docs' duplicatesStrategy 'exclude' + preserveFileTimestamps = true } tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) { @@ -1264,6 +1268,27 @@ project(':core') { from(project(':streams:examples').jar) { into("libs/") } from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") } duplicatesStrategy 'exclude' + preserveFileTimestamps = true + filePermissions { + user { + read = true + write = true + } + group.read = true + other.read = true + } + dirPermissions { + unix('rwxr-xr-x') + } + eachFile { + if (name.endsWith( ".sh")) { + permissions { + user.execute = true + group.execute = true + other.execute = true + } + } + } } jar { diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 31adcd2c26887..9b1a42e53494a 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -60,7 +60,7 @@ versions += [ commonsLang: "3.18.0", commonsValidator: "1.10.0", classgraph: "4.8.179", - gradle: "8.14.3", + gradle: "9.1.0", grgit: "4.1.1", httpclient: "4.5.14", jackson: "2.19.0", @@ -125,7 +125,7 @@ versions += [ snappy: "1.1.10.7", spotbugs: "4.9.4", mockOAuth2Server: "2.2.1", - zinc: "1.9.2", + zinc: "1.10.8", // When updating the zstd version, please do as well in docker/native/native-image-configs/resource-config.json // Also make sure the compression levels in org.apache.kafka.common.record.CompressionType are still valid zstd: "1.5.6-10", diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 78cb6e16a49f9..6a38a8cea6117 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=bd71102213493060956ec229d946beee57158dbd89d0e62b91bca0fa2c5f3531 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +distributionSha256Sum=a17ddd85a26b6a7f5ddb71ff8b05fc5104c0202c6e64782429790c933686c806 +distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 39803762879c4..98a05d787ee04 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ #!/bin/sh # -# Copyright © 2015-2021 the original authors. +# Copyright © 2015 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -114,7 +114,6 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. @@ -172,7 +171,6 @@ fi # For Cygwin or MSYS, switch paths to Windows format before running java if "$cygwin" || "$msys" ; then APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) JAVACMD=$( cygpath --unix "$JAVACMD" ) @@ -205,7 +203,7 @@ fi # Loop in case we encounter an error. for attempt in 1 2 3; do if [ ! -e "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" ]; then - if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v8.14.1/gradle/wrapper/gradle-wrapper.jar"; then + if ! curl -s -S --retry 3 -L -o "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" "https://raw.githubusercontent.com/gradle/gradle/v9.1.0/gradle/wrapper/gradle-wrapper.jar"; then rm -f "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" # Pause for a bit before looping in case the server throttled us. sleep 5 @@ -225,7 +223,6 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" diff --git a/wrapper.gradle b/wrapper.gradle index 0e0d6da201123..4cd9fc65f9d45 100644 --- a/wrapper.gradle +++ b/wrapper.gradle @@ -23,7 +23,7 @@ // and not the version installed on the machine running the task. // Read more about the wrapper here: https://docs.gradle.org/current/userguide/gradle_wrapper.html wrapper { - gradleVersion = project.gradleVersion + gradleVersion = versions.gradle } // Custom task to inject support for downloading the gradle wrapper jar if it doesn't exist. @@ -35,14 +35,12 @@ task bootstrapWrapper() { def wrapperBasePath = "\$APP_HOME/gradle/wrapper" def wrapperJarPath = wrapperBasePath + "/gradle-wrapper.jar" - // Add a trailing zero to the version if needed. - def fullVersion = project.gradleVersion.count(".") == 1 ? "${project.gradleVersion}.0" : versions.gradle // Leverages the wrapper jar checked into the gradle project on github because the jar isn't // available elsewhere. Using raw.githubusercontent.com instead of github.com because // github.com servers deprecated TLSv1/TLSv1.1 support some time ago, so older versions // of curl (built against OpenSSL library that doesn't support TLSv1.2) would fail to // fetch the jar. - def wrapperBaseUrl = "https://raw.githubusercontent.com/gradle/gradle/v$fullVersion/gradle/wrapper" + def wrapperBaseUrl = "https://raw.githubusercontent.com/gradle/gradle/v$versions.gradle/gradle/wrapper" def wrapperJarUrl = wrapperBaseUrl + "/gradle-wrapper.jar" def bootstrapString = """ From a6fe1c261ea2c7aa072cb88279ead9e27c104544 Mon Sep 17 00:00:00 2001 From: dejan2609 Date: Mon, 6 Oct 2025 14:14:16 +0200 Subject: [PATCH 3/3] KAFKA-19174 review follow-up: opting for a simple solution for permissions related link: https://docs.gradle.org/9.1.0/userguide/upgrading_major_version_9.html#reproducible_archives_by_default --- build.gradle | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/build.gradle b/build.gradle index c3cc954e4bb7a..1b4444b960813 100644 --- a/build.gradle +++ b/build.gradle @@ -327,9 +327,15 @@ subprojects { tasks.register('uploadArchives').configure { dependsOn(publish) } } - tasks.withType(AbstractTestTask).configureEach { - failOnNoDiscoveredTests = false - } + tasks.withType(AbstractArchiveTask).configureEach { + reproducibleFileOrder = false + preserveFileTimestamps = true + useFileSystemPermissions() + } + + tasks.withType(AbstractTestTask).configureEach { + failOnNoDiscoveredTests = false + } // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder. if (!project.name.equals('connect')) { @@ -1222,7 +1228,6 @@ project(':core') { from project.file("$rootDir/docs") into 'site-docs' duplicatesStrategy 'exclude' - preserveFileTimestamps = true } tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) { @@ -1268,27 +1273,6 @@ project(':core') { from(project(':streams:examples').jar) { into("libs/") } from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") } duplicatesStrategy 'exclude' - preserveFileTimestamps = true - filePermissions { - user { - read = true - write = true - } - group.read = true - other.read = true - } - dirPermissions { - unix('rwxr-xr-x') - } - eachFile { - if (name.endsWith( ".sh")) { - permissions { - user.execute = true - group.execute = true - other.execute = true - } - } - } } jar {