KRaft and Strimzi Operator Support

BROADLEAF.MD

@@ -4,12 +4,12 @@
 - `./gradlew test` (Full test suite takes 4 hours!!)
 - At the time of this writing, `SocketServerTest` consistently fails - even without any changes. As a result, ignoring this particular test result.
 - `./gradlew clean releaseTarGz`
-- For local testing: `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.22-jre-21.0.8_9-r1 BUILD_AND_LOAD_LOCAL_PLATFORM_ONLY=true docker buildx bake -f docker-bake.hcl`
+- For local testing: `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft2 BUILD_AND_LOAD_LOCAL_PLATFORM_ONLY=true docker buildx bake -f docker-bake.hcl`
 - For release: 
-  - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.22-jre-21.0.8_9-r1 LIMIT_PLATFORM="linux/arm64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl`
-  - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.22-jre-21.0.8_9-r1 LIMIT_PLATFORM="linux/amd64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl`
-  - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.22-jre-21.0.8_9-r1 docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl --push`
-  - (Increment the `r1` to a higher value if only changing java or os deps without changing app, jre, or os versions)
+  - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft2 LIMIT_PLATFORM="linux/arm64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl`
+  - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft2 LIMIT_PLATFORM="linux/amd64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl`
+  - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft2 docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl --push`
+  - (Increment the `kraft1` to a higher value if only changing java or os deps without changing app, jre, or os versions)
 ## Update Notes
 - The `Dockerfile` at the root of this project was adapted from the `jvm` instruction set located at `docker/jvm/Dockerfile`, and was updated to pull from a local kafka build (i.e. `./core/build/distributions/`) instead of the Apache Distribution Repo and references existing scripts and resources in the `./docker` directory.
 - A `docker-bake.hcl` file was added in support of a multi-platform builds

Dockerfile

@@ -16,7 +16,9 @@
 # limitations under the License.
 ###############################################################################
 
-FROM eclipse-temurin:21.0.8_9-jre-alpine-3.22 AS build-jsa
+# Stage 1: Build Java Shared Archive (JSA)
+# Adapted from the official Apache Dockerfile
+FROM eclipse-temurin:21.0.9_10-jre-alpine-3.23 AS build-jsa
 
 USER root
 
@@ -27,17 +29,76 @@ ARG DISTRO_NAME=kafka_2.13-3.9.1
 COPY core/build/distributions/$DISTRO_NAME.tgz /
 
 RUN set -eux ; \
+    # 1. Add Alpine Edge repositories
+    echo "https://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories; \
+    echo "https://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories; \
+    # 2. Update and upgrade apk-tools
     apk update ; \
-    apk upgrade ; \
+    apk add --upgrade apk-tools; \
+    # 3. Force upgrade to Edge versions
+    apk upgrade --available ; \
+    # 4. Install build dependencies (wget, gcompat, etc.)
     apk add --no-cache wget gcompat gpg gpg-agent procps bash; \
     mkdir opt/kafka; \
     tar xfz $DISTRO_NAME.tgz -C /opt/kafka --strip-components 1;
 
 # Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command
 RUN /etc/kafka/docker/jsa_launch
 
+# Stage 2: Extract Strimzi components from the official Strimzi Kafka image
+# IMPORTANT: When updating Kafka - review Strimzi and Kafka compatibility matrix 
+# to ensure appropriate strimzi source and image versions: https://strimzi.io/downloads/
+FROM quay.io/strimzi/kafka:0.47.0-kafka-3.9.1 AS strimzi-source-extractor
 
-FROM eclipse-temurin:21.0.8_9-jre-alpine-3.22
+USER root
+
+# Create directories to copy content out
+RUN mkdir -p /tmp/strimzi-extracted/scripts \
+             /tmp/strimzi-extracted/kafka-exporter \
+             /tmp/strimzi-extracted/prometheus-jmx-exporter \
+             /tmp/strimzi-extracted/kafka-libs \
+             /tmp/strimzi-extracted/cruise-control \
+             /tmp/strimzi-extracted/usr-bin             
+
+#####
+# Add Kafka Scripts
+# Strimzi Step 1: Copy Strimzi Kafka scripts (e.g., kafka_run.sh) from base image
+# Exclude non-strimzi scripts (i.e we only need ones defined here https://github.com/strimzi/strimzi-kafka-operator/tree/0.47.0/docker-images/kafka-based/kafka/scripts)
+#####
+RUN cp -r /opt/kafka/* /tmp/strimzi-extracted/scripts
+RUN rm -rf /tmp/strimzi-extracted/scripts/LICENSE /tmp/strimzi-extracted/scripts/NOTICE /tmp/strimzi-extracted/scripts/bin /tmp/strimzi-extracted/scripts/config /tmp/strimzi-extracted/scripts/libs /tmp/strimzi-extracted/scripts/licenses /tmp/strimzi-extracted/scripts/plugins /tmp/strimzi-extracted/scripts/site-docs
+
+#####
+# Exclude Kafka Exporter
+# Strimzi Step 2: Strimzi comes with Kafka Exporter Scripts from base image
+# However due to more strict security policies, we are opting to exclude this go-based dependency
+# RUN cp -r /opt/kafka-exporter/* /tmp/strimzi-extracted/kafka-exporter
+#####
+
+#####
+# Add Prometheus JMX Exporter
+# Strimzi Step 3: Copy Prometheus JMX Exporter contents from base image
+#####
+RUN cp -r /opt/prometheus-jmx-exporter/* /tmp/strimzi-extracted/prometheus-jmx-exporter
+
+#####
+# Add Strimzi agents, 3rd party libs, & Other Kafka Libs
+# Strimzi Step 4: Copy Strimzi Agents and other libraries from Kafka's libs directory in the base image
+#####
+RUN cp -r /opt/kafka/libs/* /tmp/strimzi-extracted/kafka-libs
+
+#####
+# Add Cruise Control
+# Strimzi Step 5: Copy Cruise Control libraries and scripts
+#####
+RUN cp -r /opt/cruise-control/* /tmp/strimzi-extracted/cruise-control
+
+# Verify the files were copied
+RUN ls -la /tmp/strimzi-extracted/scripts /tmp/strimzi-extracted/kafka-exporter /tmp/strimzi-extracted/prometheus-jmx-exporter /tmp/strimzi-extracted/kafka-libs /tmp/strimzi-extracted/cruise-control
+
+# Stage 3: Main Kafka image build
+# Adapted from the official Apache Dockerfile
+FROM eclipse-temurin:21.0.9_10-jre-alpine-3.23
 
 # exposed ports
 EXPOSE 9092
@@ -58,9 +119,17 @@ COPY core/build/distributions/$DISTRO_NAME.tgz /
 # We assume appuser UID for standard Kubernetes, and an arbitrary UID + root group (0)
 # for OpenShift. Thus, grant both of those ownership here.
 RUN set -eux ; \
+    # 1. Add Alpine Edge repositories
+    echo "https://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories; \
+    echo "https://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories; \
+    # 2. Update and upgrade apk-tools
     apk update ; \
-    apk upgrade ; \
-    apk add --no-cache wget gcompat gpg gpg-agent procps bash su-exec; \
+    apk add --upgrade apk-tools; \
+    # 3. Force upgrade all OS packages to Edge versions (patches CVEs)
+    apk upgrade --available ; \
+    # 4. Install runtime dependencies
+    apk add --no-cache wget gcompat gpg gpg-agent procps bash su-exec tini grep curl; \
+    # 5. Continue with Kafka installation and configuration
     mkdir opt/kafka; \
     tar xfz $DISTRO_NAME.tgz -C /opt/kafka --strip-components 1; \
     mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \
@@ -72,12 +141,20 @@ RUN set -eux ; \
     cp /opt/kafka/config/log4j.properties /etc/kafka/docker/log4j.properties; \
     cp /opt/kafka/config/tools-log4j.properties /etc/kafka/docker/tools-log4j.properties; \
     rm $DISTRO_NAME.tgz; \
+    # 6. Cleanup (remove build-only tools if desired, though gpg/wget were just installed above)
     apk del wget gpg gpg-agent; \
     apk cache clean;
 
+#####
+# Needed to support an adapted entrypoint in support of
+# backwared-compatible BLC installations that may have been referencing
+# a Confluent-based Kafka Image, with added support for initialization 
+# via the Strimzi Operator for K8 installations (https://strimzi.io/)
+#####    
 COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa
 COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa
 COPY --chown=appuser:0 docker/resources/common-scripts /etc/kafka/docker
+RUN chmod +x /etc/kafka/docker/copy-jars.sh
 COPY --chown=appuser:0 docker/jvm/launch /etc/kafka/docker/launch
 
 VOLUME ["/etc/kafka/secrets", "/var/lib/kafka/data", "/mnt/shared/config"]
@@ -89,8 +166,68 @@ RUN mkdir /etc/confluent/docker
 COPY --chown=appuser:0 run.sh /etc/confluent/docker/run
 RUN chmod 755 /etc/confluent/docker/run
 
+# --- Start of Strimzi Support ---
+
+ENV KAFKA_HOME=/opt/kafka
+ENV KAFKA_VERSION=3.9.1
+ENV STRIMZI_VERSION=0.47.0
+ENV KAFKA_EXPORTER_HOME=/opt/kafka-exporter
+ENV JMX_EXPORTER_HOME=/opt/prometheus-jmx-exporter
+ENV CRUISE_CONTROL_HOME=/opt/cruise-control
+
+# Create a symlink to tini in /usr/bin as referenced by strimzi scripts
+RUN ln -s /sbin/tini /usr/bin/tini
+
+# Create necessary directories for Strimzi components
+RUN mkdir -p ${KAFKA_HOME}/strimzi-scripts \
+             ${KAFKA_HOME}/strimzi-kafka-libs \
+             ${KAFKA_EXPORTER_HOME} \
+             ${CRUISE_CONTROL_HOME} \
+             ${JMX_EXPORTER_HOME}
+
+# Copy Strimzi Kafka scripts
+COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/scripts ${KAFKA_HOME}/strimzi-scripts
+RUN chmod -R +x ${KAFKA_HOME}/strimzi-scripts
+RUN mv ${KAFKA_HOME}/strimzi-scripts/* ${KAFKA_HOME}
+RUN rm -rf ${KAFKA_HOME}/strimzi-scripts
+
+# Do Nothing with Kafka Exporter Directory
+# As mentioned above, due to stricter security policies we are opting to exclude the kafka-exporter binary
+# COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/kafka-exporter ${KAFKA_EXPORTER_HOME}
+# RUN chmod -R +x ${KAFKA_EXPORTER_HOME}/kafka_exporter
+
+# Copy Prometheus JMX Exporter directory
+COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/prometheus-jmx-exporter ${JMX_EXPORTER_HOME}
+
+# Copy Strimzi Agents and other Kafka libraries
+# Use a bind mount to access files from the previous stage without copying them permanently into a layer first
+RUN --mount=type=bind,from=strimzi-source-extractor,source=/tmp/strimzi-extracted/kafka-libs,target=/tmp/strimzi-kafka-libs-source \
+    /etc/kafka/docker/copy-jars.sh /tmp/strimzi-kafka-libs-source ${KAFKA_HOME}/libs && \
+    chown -R appuser:root ${KAFKA_HOME}/libs
+
+# Copy Cruise Control libraries
+# Use a bind mount to access files from the previous stage
+# We copy libs using the deduplication script, and other files directly
+RUN --mount=type=bind,from=strimzi-source-extractor,source=/tmp/strimzi-extracted/cruise-control,target=/tmp/cruise-control-source \
+    bash -c '/etc/kafka/docker/copy-jars.sh /tmp/cruise-control-source/libs ${CRUISE_CONTROL_HOME}/libs ${KAFKA_HOME}/libs && \
+    find /tmp/cruise-control-source -maxdepth 1 -mindepth 1 -not -name libs -exec cp -r {} ${CRUISE_CONTROL_HOME}/ \;' && \
+    chown -R appuser:root ${CRUISE_CONTROL_HOME} && \
+    chmod -R +x ${CRUISE_CONTROL_HOME}
+
+# Important to set this to the kafka home as strimzi scripts have relative path references
+WORKDIR $KAFKA_HOME
+
+# --- End of Strimzi Support ---
+
 # For compatibility with standard Kubernetes, we explicitly switch to a non-root UID. OpenShift
 # will ignore these settings and run as an arbitrary UID in the root group.
 USER appuser
 
+#####
+# Adapted Entrypoint in order to support 
+# backwared-compatible BLC installations that may have been referencing
+# a Confluent-based Kafka Image. Note that this entrypoint is not used
+# when running this image via the Strimzi Operator as Strimzi
+# provides its own entrypoint. See https://github.com/strimzi/strimzi-kafka-operator
+#####
 CMD ["/etc/confluent/docker/run"]

build.gradle

@@ -161,6 +161,8 @@ allprojects {
           // ZooKeeper (potentially older and containing CVEs)
           libs.nettyHandler,
           libs.nettyTransportNativeEpoll,
+          libs.nettyCodecHttp,
+          libs.nettyCodecHttp2,
           // be explicit about the reload4j version instead of relying on the transitive versions
           libs.reload4j,
           libs.commonsLang3
@@ -972,6 +974,16 @@ project(':core') {
     // ZooKeeperMain depends on commons-cli but declares the dependency as `provided`
     implementation libs.commonsCli
 
+    // Strimzi and cruise control deps
+    // Overridden for CVEs
+    implementation libs.nettyCodecHttp
+    implementation libs.nettyCodecHttp2
+    implementation libs.nimbusJoseJwt
+    implementation libs.log4J
+    implementation libs.vertxCore
+    implementation libs.vertxWeb
+    // End Strimzi and cruise control deps
+
     compileOnly libs.reload4j
 
     testImplementation project(':clients').sourceSets.test.output

docker/resources/common-scripts/configureDefaults

@@ -18,8 +18,6 @@ declare -A env_defaults
 env_defaults=(
 # Replace CLUSTER_ID with a unique base64 UUID using "bin/kafka-storage.sh random-uuid"
   ["CLUSTER_ID"]="5L6g3nShT-eMCtK--X86sw"
-  # Default Zookeeper connection string - override with KAFKA_ZOOKEEPER_CONNECT env var
-  ["KAFKA_ZOOKEEPER_CONNECT"]="localhost:2181"
 )
 
 for key in "${!env_defaults[@]}"; do

docker/resources/common-scripts/copy-jars.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+#
+# Copies files from SOURCE to TARGET.
+# 1. If JAR exists in TARGET (ignoring version), skip.
+# 2. If JAR exists in ADDITIONAL_CHECK_DIR (ignoring version), copy the version from ADDITIONAL_CHECK_DIR to TARGET.
+# 3. Otherwise, copy from SOURCE to TARGET.
+#
+# Usage: copy-jars.sh <SOURCE_DIR> <TARGET_DIR> [ADDITIONAL_CHECK_DIR]
+
+set -e
+
+SOURCE_DIR="$1"
+TARGET_DIR="$2"
+ADDITIONAL_CHECK_DIR="$3"
+
+if [ -z "$SOURCE_DIR" ] || [ -z "$TARGET_DIR" ]; then
+    echo "Usage: $0 <SOURCE_DIR> <TARGET_DIR> [ADDITIONAL_CHECK_DIR]"
+    exit 1
+fi
+
+# Ensure target exists
+mkdir -p "$TARGET_DIR"
+
+echo "Copying files from $SOURCE_DIR to $TARGET_DIR with version deduplication..."
+if [ -n "$ADDITIONAL_CHECK_DIR" ]; then
+    echo "Also checking for duplicates in $ADDITIONAL_CHECK_DIR"
+fi
+
+# Helper to extract base name (artifact ID)
+get_base_name() {
+    local filename=$1
+    # Remove version suffix starting with a hyphen followed by a digit
+    echo "$filename" | sed -E 's/-[0-9].*\.jar$//'
+}
+
+# Process JAR files
+# We use a loop over the glob to handle filenames with spaces correctly
+shopt -s nullglob
+for src_jar in "$SOURCE_DIR"/*.jar; do
+    filename=$(basename "$src_jar")
+    base_name=$(get_base_name "$filename")
+
+    duplicate_in_target=false
+
+    # Check against all jars in target
+    for target_jar in "$TARGET_DIR"/*.jar; do
+        t_filename=$(basename "$target_jar")
+        t_base_name=$(get_base_name "$t_filename")
+
+        if [ "$base_name" == "$t_base_name" ]; then
+            echo "Skipping $filename (duplicate of $t_filename in target)"
+            duplicate_in_target=true
+            break
+        fi
+    done
+
+    if [ "$duplicate_in_target" = true ]; then
+        continue
+    fi
+
+    # Check additional directory if provided
+    copied_from_additional=false
+    if [ -n "$ADDITIONAL_CHECK_DIR" ] && [ -d "$ADDITIONAL_CHECK_DIR" ]; then
+        for check_jar in "$ADDITIONAL_CHECK_DIR"/*.jar; do
+            c_filename=$(basename "$check_jar")
+            c_base_name=$(get_base_name "$c_filename")
+
+            if [ "$base_name" == "$c_base_name" ]; then
+                echo "Found duplicate in additional dir: $c_filename. Copying that version to target."
+                cp "$check_jar" "$TARGET_DIR/"
+                copied_from_additional=true
+                break
+            fi
+        done
+    fi
+
+    if [ "$copied_from_additional" = false ]; then
+        echo "Copying $filename"
+        cp "$src_jar" "$TARGET_DIR/"
+    fi
+done
+
+# Process non-JAR files and directories
+# We copy them if they don't exist in target (exact name match)
+for src_file in "$SOURCE_DIR"/*; do
+    filename=$(basename "$src_file")
+
+    # Skip if it's a jar (already handled)
+    if [[ "$filename" == *.jar ]]; then
+        continue
+    fi
+
+    if [ ! -e "$TARGET_DIR/$filename" ]; then
+        echo "Copying $filename"
+        cp -r "$src_file" "$TARGET_DIR/"
+    else
+        echo "Skipping $filename (exists)"
+    fi
+done