feat: custom version for Kafka

CHANGELOG.md

@@ -128,6 +128,7 @@ All notable changes to this project will be documented in this file.
 - nifi: Remove `2.2.0` ([#1114]).
 - kafka: Remove `3.7.1` and `3.8.0` ([#1117]).
 - spark-connect-client: Remove `3.5.5` ([#1142]).
+- kafka: Enable custom versions ([#1171]).
 - omid: Enable custom versions ([#1174]).
 
 [nifi-iceberg-bundle]: https://github.com/stackabletech/nifi-iceberg-bundle
@@ -194,6 +195,7 @@ All notable changes to this project will be documented in this file.
 [#1165]: https://github.com/stackabletech/docker-images/pull/1165
 [#1168]: https://github.com/stackabletech/docker-images/pull/1168
 [#1170]: https://github.com/stackabletech/docker-images/pull/1170
+[#1171]: https://github.com/stackabletech/docker-images/pull/1171
 [#1173]: https://github.com/stackabletech/docker-images/pull/1173
 [#1174]: https://github.com/stackabletech/docker-images/pull/1174
 [#1179]: https://github.com/stackabletech/docker-images/pull/1179

kafka/Dockerfile

@@ -6,6 +6,7 @@ FROM stackable/image/kafka/kcat AS kcat
 FROM stackable/image/java-devel AS kafka-builder
 
 ARG PRODUCT
+ARG RELEASE
 ARG SCALA
 ARG OPA_AUTHORIZER
 ARG JMX_EXPORTER
@@ -21,22 +22,27 @@ COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackab
 RUN <<EOF
 cd "$(/stackable/patchable --images-repo-root=src checkout kafka ${PRODUCT})"
 
+NEW_VERSION="${PRODUCT}-stackable${RELEASE}"
+
+# The comment in gradle.properties mentions to update the version in all of these files
+find . -type f -print0 | xargs -0 sed -i "s/\-stackable0\.0\.0\-dev/-stackable${RELEASE}/g"
+
 # Create snapshot of the source code including custom patches
-tar -czf /stackable/kafka-${PRODUCT}-src.tar.gz .
+tar -czf /stackable/kafka-${NEW_VERSION}-src.tar.gz .
 
 # TODO: Try to install gradle via package manager (if possible) instead of fetching it from the internet
 # We don't specify "-x test" to skip the tests, as we might bump some Kafka internal dependencies in the future and
 # it's a good idea to run the tests in this case.
 ./gradlew clean releaseTarGz
 ./gradlew cyclonedxBom
-tar -xf core/build/distributions/kafka_${SCALA}-${PRODUCT}.tgz -C /stackable
-cp build/reports/bom.json /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json
-rm -rf /stackable/kafka_${SCALA}-${PRODUCT}/site-docs/
+tar -xf core/build/distributions/kafka_${SCALA}-${NEW_VERSION}.tgz -C /stackable
+cp build/reports/bom.json /stackable/kafka_${SCALA}-${NEW_VERSION}.cdx.json
+rm -rf /stackable/kafka_${SCALA}-${NEW_VERSION}/site-docs/
 (cd .. && rm -rf ${PRODUCT})
 
 # TODO (@NickLarsenNZ): Compile from source: https://github.com/StyraInc/opa-kafka-plugin
 curl https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \
-  -o /stackable/kafka_${SCALA}-${PRODUCT}/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
+  -o /stackable/kafka_${SCALA}-${NEW_VERSION}/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar
 
 # JMX exporter
 curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \
@@ -65,9 +71,9 @@ LABEL \
   summary="The Stackable image for Apache Kafka." \
   description="This image is deployed by the Stackable Operator for Apache Kafka."
 
-COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT}
-COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}.cdx.json /stackable/kafka_${SCALA}-${PRODUCT}/kafka_${SCALA}-${PRODUCT}.cdx.json
-COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka-${PRODUCT}-src.tar.gz  /stackable
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE} /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}.cdx.json /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}.cdx.json
+COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka-${PRODUCT}-stackable${RELEASE}-src.tar.gz  /stackable
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/jmx/ /stackable/jmx/
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat /stackable/bin/kcat-${KAFKA_KCAT}
 COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT}-src.tar.gz /stackable
@@ -94,13 +100,13 @@ chown -h ${STACKABLE_USER_UID}:0 /stackable/bin/kcat
 # kcat was located in /stackable/kcat - legacy
 ln -s /stackable/bin/kcat /stackable/kcat
 chown -h ${STACKABLE_USER_UID}:0 /stackable/kcat
-ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka
+ln -s /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE} /stackable/kafka
 chown -h ${STACKABLE_USER_UID}:0 /stackable/kafka
 
 # fix missing permissions
 chmod g=u /stackable/bin
 chmod g=u /stackable/jmx
-chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}
+chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}
 chmod g=u /stackable/*-src.tar.gz
 EOF
 

kafka/stackable/patches/3.7.2/0003-Build-custom-Stackable-version.patch

@@ -0,0 +1,116 @@
+From 508c8053273b091b599c5ab9804cee877483703f Mon Sep 17 00:00:00 2001
+From: dervoeti <[email protected]>
+Date: Wed, 18 Jun 2025 12:15:17 +0200
+Subject: Build custom Stackable version
+
+---
+ docs/js/templateData.js                                         | 2 +-
+ gradle.properties                                               | 2 +-
+ kafka-merge-pr.py                                               | 2 +-
+ streams/quickstart/java/pom.xml                                 | 2 +-
+ .../java/src/main/resources/archetype-resources/pom.xml         | 2 +-
+ streams/quickstart/pom.xml                                      | 2 +-
+ tests/kafkatest/__init__.py                                     | 2 +-
+ tests/kafkatest/version.py                                      | 2 +-
+ 8 files changed, 8 insertions(+), 8 deletions(-)
+
+diff --git a/docs/js/templateData.js b/docs/js/templateData.js
+index 7700e24003..8affa51e55 100644
+--- a/docs/js/templateData.js
++++ b/docs/js/templateData.js
+@@ -19,6 +19,6 @@ limitations under the License.
+ var context={
+     "version": "37",
+     "dotVersion": "3.7",
+-    "fullDotVersion": "3.7.2",
++    "fullDotVersion": "3.7.2-stackable0.0.0-dev",
+     "scalaVersion": "2.13"
+ };
+diff --git a/gradle.properties b/gradle.properties
+index 08ae0ded1e..779dcfb829 100644
+--- a/gradle.properties
++++ b/gradle.properties
+@@ -23,7 +23,7 @@ group=org.apache.kafka
+ #  - streams/quickstart/pom.xml
+ #  - streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+ #  - streams/quickstart/java/pom.xml
+-version=3.7.2
++version=3.7.2-stackable0.0.0-dev
+ scalaVersion=2.13.12
+ # Adding swaggerVersion in gradle.properties to have a single version in place for swagger
+ # New version of Swagger 2.2.14 requires minimum JDK 11.
+diff --git a/kafka-merge-pr.py b/kafka-merge-pr.py
+index c16c811178..41b6794b8c 100755
+--- a/kafka-merge-pr.py
++++ b/kafka-merge-pr.py
+@@ -70,7 +70,7 @@ TEMP_BRANCH_PREFIX = "PR_TOOL"
+
+ DEV_BRANCH_NAME = "trunk"
+
+-DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.7.2-SNAPSHOT")
++DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.7.2-stackable0.0.0-dev-SNAPSHOT")
+
+ ORIGINAL_HEAD = ""
+
+diff --git a/streams/quickstart/java/pom.xml b/streams/quickstart/java/pom.xml
+index 1335f0d58b..e9e8cfe1ec 100644
+--- a/streams/quickstart/java/pom.xml
++++ b/streams/quickstart/java/pom.xml
+@@ -26,7 +26,7 @@
+     <parent>
+         <groupId>org.apache.kafka</groupId>
+         <artifactId>streams-quickstart</artifactId>
+-        <version>3.7.2</version>
++        <version>3.7.2-stackable0.0.0-dev</version>
+         <relativePath>..</relativePath>
+     </parent>
+
+diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+index 445fab4b83..21cb230fc4 100644
+--- a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
++++ b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+@@ -29,7 +29,7 @@
+
+     <properties>
+         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+-        <kafka.version>3.7.2</kafka.version>
++        <kafka.version>3.7.2-stackable0.0.0-dev</kafka.version>
+         <slf4j.version>1.7.36</slf4j.version>
+     </properties>
+
+diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml
+index 94ac32fcac..2cc1fcd600 100644
+--- a/streams/quickstart/pom.xml
++++ b/streams/quickstart/pom.xml
+@@ -22,7 +22,7 @@
+     <groupId>org.apache.kafka</groupId>
+     <artifactId>streams-quickstart</artifactId>
+     <packaging>pom</packaging>
+-    <version>3.7.2</version>
++    <version>3.7.2-stackable0.0.0-dev</version>
+
+     <name>Kafka Streams :: Quickstart</name>
+
+diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py
+index 1c66c2bcfd..9274ce1fd0 100644
+--- a/tests/kafkatest/__init__.py
++++ b/tests/kafkatest/__init__.py
+@@ -22,4 +22,4 @@
+ # Instead, in development branches, the version should have a suffix of the form ".devN"
+ #
+ # For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be something like "1.0.0.dev0"
+-__version__ = '3.7.2'
++__version__ = '3.7.2-stackable0.0.0-dev'
+diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py
+index bed7cffac2..55dc3cbc7d 100644
+--- a/tests/kafkatest/version.py
++++ b/tests/kafkatest/version.py
+@@ -122,7 +122,7 @@ def get_version(node=None):
+         return DEV_BRANCH
+
+ DEV_BRANCH = KafkaVersion("dev")
+-DEV_VERSION = KafkaVersion("3.7.2-SNAPSHOT")
++DEV_VERSION = KafkaVersion("3.7.2-stackable0.0.0-dev")
+
+ LATEST_METADATA_VERSION = "3.7"
+

kafka/stackable/patches/3.9.0/0003-Build-custom-Stackable-version.patch

@@ -0,0 +1,137 @@
+From de4d98f9cd65d93bb4bb061c233f6647efc951f4 Mon Sep 17 00:00:00 2001
+From: dervoeti <[email protected]>
+Date: Wed, 18 Jun 2025 12:16:46 +0200
+Subject: Build custom Stackable version
+
+---
+ docs/js/templateData.js                                       | 2 +-
+ docs/streams/upgrade-guide.html                               | 2 +-
+ gradle.properties                                             | 2 +-
+ kafka-merge-pr.py                                             | 2 +-
+ streams/quickstart/java/pom.xml                               | 2 +-
+ .../java/src/main/resources/archetype-resources/pom.xml       | 2 +-
+ streams/quickstart/pom.xml                                    | 2 +-
+ tests/kafkatest/__init__.py                                   | 2 +-
+ tests/kafkatest/version.py                                    | 4 ++--
+ 9 files changed, 10 insertions(+), 10 deletions(-)
+
+diff --git a/docs/js/templateData.js b/docs/js/templateData.js
+index 13d352fc70..42d3549382 100644
+--- a/docs/js/templateData.js
++++ b/docs/js/templateData.js
+@@ -19,6 +19,6 @@ limitations under the License.
+ var context={
+     "version": "39",
+     "dotVersion": "3.9",
+-    "fullDotVersion": "3.9.0",
++    "fullDotVersion": "3.9.0-stackable0.0.0-dev",
+     "scalaVersion": "2.13"
+ };
+diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html
+index 6a30e6671b..6d2ecb81aa 100644
+--- a/docs/streams/upgrade-guide.html
++++ b/docs/streams/upgrade-guide.html
+@@ -133,7 +133,7 @@
+         More details about the new config <code>StreamsConfig#TOPOLOGY_OPTIMIZATION</code> can be found in <a href="https://cwiki.apache.org/confluence/display/KAFKA/KIP-295%3A+Add+Streams+Configuration+Allowing+for+Optional+Topology+Optimization">KIP-295</a>.
+     </p>
+
+-    <h3><a id="streams_api_changes_390" href="#streams_api_changes_390">Streams API changes in 3.9.0</a></h3>
++    <h3><a id="streams_api_changes_390" href="#streams_api_changes_390">Streams API changes in 3.9.0-stackable0.0.0-dev</a></h3>
+
+     <p>
+         The introduction of <a href="https://cwiki.apache.org/confluence/display/KAFKA/KIP-1033%3A+Add+Kafka+Streams+exception+handler+for+exceptions+occurring+during+processing">KIP-1033</a>
+diff --git a/gradle.properties b/gradle.properties
+index 0c14658608..b79ee9f0cf 100644
+--- a/gradle.properties
++++ b/gradle.properties
+@@ -23,7 +23,7 @@ group=org.apache.kafka
+ #  - streams/quickstart/pom.xml
+ #  - streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+ #  - streams/quickstart/java/pom.xml
+-version=3.9.0
++version=3.9.0-stackable0.0.0-dev
+ scalaVersion=2.13.14
+ # Adding swaggerVersion in gradle.properties to have a single version in place for swagger
+ # New version of Swagger 2.2.14 requires minimum JDK 11.
+diff --git a/kafka-merge-pr.py b/kafka-merge-pr.py
+index a775a38435..4a7fbac34e 100755
+--- a/kafka-merge-pr.py
++++ b/kafka-merge-pr.py
+@@ -70,7 +70,7 @@ TEMP_BRANCH_PREFIX = "PR_TOOL"
+
+ DEV_BRANCH_NAME = "trunk"
+
+-DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.9.0")
++DEFAULT_FIX_VERSION = os.environ.get("DEFAULT_FIX_VERSION", "3.9.0-stackable0.0.0-dev")
+
+ ORIGINAL_HEAD = ""
+
+diff --git a/streams/quickstart/java/pom.xml b/streams/quickstart/java/pom.xml
+index 78e1a70fa7..8698b4eb35 100644
+--- a/streams/quickstart/java/pom.xml
++++ b/streams/quickstart/java/pom.xml
+@@ -26,7 +26,7 @@
+     <parent>
+         <groupId>org.apache.kafka</groupId>
+         <artifactId>streams-quickstart</artifactId>
+-        <version>3.9.0</version>
++        <version>3.9.0-stackable0.0.0-dev</version>
+         <relativePath>..</relativePath>
+     </parent>
+
+diff --git a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+index 2bc235eb16..90d566d1a0 100644
+--- a/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
++++ b/streams/quickstart/java/src/main/resources/archetype-resources/pom.xml
+@@ -29,7 +29,7 @@
+
+     <properties>
+         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+-        <kafka.version>3.9.0</kafka.version>
++        <kafka.version>3.9.0-stackable0.0.0-dev</kafka.version>
+         <slf4j.version>1.7.36</slf4j.version>
+     </properties>
+
+diff --git a/streams/quickstart/pom.xml b/streams/quickstart/pom.xml
+index b5101583ee..3c08985e2a 100644
+--- a/streams/quickstart/pom.xml
++++ b/streams/quickstart/pom.xml
+@@ -22,7 +22,7 @@
+     <groupId>org.apache.kafka</groupId>
+     <artifactId>streams-quickstart</artifactId>
+     <packaging>pom</packaging>
+-    <version>3.9.0</version>
++    <version>3.9.0-stackable0.0.0-dev</version>
+
+     <name>Kafka Streams :: Quickstart</name>
+
+diff --git a/tests/kafkatest/__init__.py b/tests/kafkatest/__init__.py
+index a190e05067..0aa8fa7095 100644
+--- a/tests/kafkatest/__init__.py
++++ b/tests/kafkatest/__init__.py
+@@ -22,4 +22,4 @@
+ # Instead, in development branches, the version should have a suffix of the form ".devN"
+ #
+ # For example, when Kafka is at version 1.0.0-SNAPSHOT, this should be something like "1.0.0.dev0"
+-__version__ = '3.9.0'
++__version__ = '3.9.0-stackable0.0.0-dev'
+diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py
+index 4825a46039..06fd3029f9 100644
+--- a/tests/kafkatest/version.py
++++ b/tests/kafkatest/version.py
+@@ -122,7 +122,7 @@ def get_version(node=None):
+         return DEV_BRANCH
+
+ DEV_BRANCH = KafkaVersion("dev")
+-DEV_VERSION = KafkaVersion("3.9.0-SNAPSHOT")
++DEV_VERSION = KafkaVersion("3.9.0-stackable0.0.0-dev")
+
+ # This should match the LATEST_PRODUCTION version defined in MetadataVersion.java
+ LATEST_STABLE_METADATA_VERSION = "3.9-IV0"
+@@ -276,5 +276,5 @@ V_3_8_0 = KafkaVersion("3.8.0")
+ LATEST_3_8 = V_3_8_0
+
+ # 3.9.x version
+-V_3_9_0 = KafkaVersion("3.9.0")
++V_3_9_0 = KafkaVersion("3.9.0-stackable0.0.0-dev")
+ LATEST_3_9 = V_3_9_0