From f68981704f2511625660dee83ef8faa34d96899a Mon Sep 17 00:00:00 2001 From: Elbert Bautista Date: Mon, 21 Jul 2025 11:48:34 -0500 Subject: [PATCH 1/6] initial kraft and strimzi support --- Dockerfile | 117 +++++++++++++++++- .../common-scripts/configureDefaults | 2 - 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f15365a8f3651..dc9a6a63c41ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,8 @@ # limitations under the License. ############################################################################### +# Stage 1: Build Java Shared Archive (JSA) +# Adapted from the official Apache Dockerfile FROM eclipse-temurin:21.0.8_9-jre-alpine-3.22 AS build-jsa USER root @@ -36,7 +38,67 @@ RUN set -eux ; \ # Generate jsa files using dynamic CDS for kafka server start command and kafka storage format command RUN /etc/kafka/docker/jsa_launch +# Stage 2: Extract Strimzi components from the official Strimzi Kafka image +# IMPORTANT: When updating Kafka - review Strimzi and Kafka compatibility matrix +# to ensure appropriate strimzi source and image versions: https://strimzi.io/downloads/ +FROM quay.io/strimzi/kafka:0.47.0-kafka-3.9.1 AS strimzi-source-extractor +USER root + +ARG STRIMZI_VERSION=0.47.0 + +# Create directories to copy content out +RUN mkdir -p /tmp/strimzi-extracted/scripts \ + /tmp/strimzi-extracted/kafka-exporter \ + /tmp/strimzi-extracted/prometheus-jmx-exporter \ + /tmp/strimzi-extracted/kafka-libs \ + /tmp/strimzi-extracted/cruise-control \ + /tmp/strimzi-extracted/usr-bin + +##### +# Add Kafka +# Strimzi Step 1: Copy Strimzi Kafka scripts (e.g., kafka_run.sh) directly from source +# Download and extract the Strimzi Kafka Operator tar file +##### +RUN curl -L https://github.com/strimzi/strimzi-kafka-operator/archive/$STRIMZI_VERSION.tar.gz -o strimzi-kafka-operator.tar.gz && \ + tar -xzf strimzi-kafka-operator.tar.gz && \ + mv strimzi-kafka-operator-$STRIMZI_VERSION strimzi-kafka-operator + +# Copy the required directories to the current working directory +RUN cp -r strimzi-kafka-operator/docker-images/kafka-based/kafka/scripts /tmp/strimzi-extracted/scripts + +##### +# Add Kafka Exporter +# Strimzi Step 2: Copy Kafka Exporter Scripts from base image +##### +RUN cp -r /opt/kafka-exporter/* /tmp/strimzi-extracted/kafka-exporter + +##### +# Add Prometheus JMX Exporter +# Strimzi Step 3: Copy Prometheus JMX Exporter contents from base image +##### +RUN cp -r /opt/prometheus-jmx-exporter/* /tmp/strimzi-extracted/prometheus-jmx-exporter + +##### +# Add Strimzi agents, 3rd party libs, & Other Kafka Libs +# Strimzi Step 4: Copy Strimzi Agents and other libraries from Kafka's libs directory in the base image +##### +RUN cp -r /opt/kafka/libs/* /tmp/strimzi-extracted/kafka-libs + +##### +# Add Cruise Control +# Strimzi Step 5: Copy Cruise Control libraries and scripts +##### +RUN cp -r /opt/cruise-control/* /tmp/strimzi-extracted/cruise-control + +# Clean up downloaded files to reduce image size +RUN rm -rf strimzi-kafka-operator.tar.gz strimzi-kafka-operator + +# Verify the files were copied +RUN ls -la /tmp/strimzi-extracted/scripts /tmp/strimzi-extracted/kafka-exporter /tmp/strimzi-extracted/prometheus-jmx-exporter /tmp/strimzi-extracted/kafka-libs /tmp/strimzi-extracted/cruise-control + +# Stage 3: Main Kafka image build +# Adapted from the official Apache Dockerfile FROM eclipse-temurin:21.0.8_9-jre-alpine-3.22 # exposed ports @@ -60,7 +122,7 @@ COPY core/build/distributions/$DISTRO_NAME.tgz / RUN set -eux ; \ apk update ; \ apk upgrade ; \ - apk add --no-cache wget gcompat gpg gpg-agent procps bash su-exec; \ + apk add --no-cache wget gcompat gpg gpg-agent procps bash su-exec tini grep curl; \ mkdir opt/kafka; \ tar xfz $DISTRO_NAME.tgz -C /opt/kafka --strip-components 1; \ mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \ @@ -75,6 +137,12 @@ RUN set -eux ; \ apk del wget gpg gpg-agent; \ apk cache clean; +##### +# Adapted Entrypoint in order to support +# backwared-compatible BLC installations that may have been referencing +# a Confluent-based Kafka Image, with added support for initialization +# via the Strimzi Operator for K8 installations (https://strimzi.io/) +##### COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa COPY --chown=appuser:0 docker/resources/common-scripts /etc/kafka/docker @@ -89,6 +157,53 @@ RUN mkdir /etc/confluent/docker COPY --chown=appuser:0 run.sh /etc/confluent/docker/run RUN chmod 755 /etc/confluent/docker/run +# --- Start of Strimzi Support --- + +ENV KAFKA_HOME=/opt/kafka +ENV KAFKA_VERSION=3.9.1 +ENV STRIMZI_VERSION=0.47.0 +ENV KAFKA_EXPORTER_HOME=/opt/kafka-exporter +ENV JMX_EXPORTER_HOME=/opt/prometheus-jmx-exporter +ENV CRUISE_CONTROL_HOME=/opt/cruise-control + +# Create a symlink to tini in /usr/bin as referenced by strimzi scripts +RUN ln -s /sbin/tini /usr/bin/tini + +# Create necessary directories for Strimzi components +RUN mkdir -p ${KAFKA_HOME}/strimzi-scripts \ + ${KAFKA_HOME}/strimzi-kafka-libs \ + ${KAFKA_EXPORTER_HOME} \ + ${CRUISE_CONTROL_HOME} \ + ${JMX_EXPORTER_HOME} + +# Copy Strimzi Kafka scripts +COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/scripts ${KAFKA_HOME}/strimzi-scripts +RUN chmod -R +x ${KAFKA_HOME}/strimzi-scripts +RUN mv ${KAFKA_HOME}/strimzi-scripts/scripts/* ${KAFKA_HOME} +RUN rm -rf ${KAFKA_HOME}/strimzi-scripts + +# Copy Kafka Exporter directory +COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/kafka-exporter ${KAFKA_EXPORTER_HOME} +RUN chmod -R +x ${KAFKA_EXPORTER_HOME}/kafka_exporter + +# Copy Prometheus JMX Exporter directory +COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/prometheus-jmx-exporter ${JMX_EXPORTER_HOME} + +# Copy Strimzi Agents and other Kafka libraries +COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/kafka-libs/ ${KAFKA_HOME}/strimzi-kafka-libs/ +# Copy from strimzi temp directory into main kafka lib, skip files if they already exist +RUN cp -n ${KAFKA_HOME}/strimzi-kafka-libs/* ${KAFKA_HOME}/libs +RUN rm -rf ${KAFKA_HOME}/strimzi-kafka-libs + +# Copy Cruise Control libraries +COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/cruise-control/ ${CRUISE_CONTROL_HOME} +RUN chmod -R +x ${CRUISE_CONTROL_HOME} || true + +# Important to set this to the kafka home as strimzi scripts have relative path references +WORKDIR $KAFKA_HOME + +# --- End of Strimzi Support --- + # For compatibility with standard Kubernetes, we explicitly switch to a non-root UID. OpenShift # will ignore these settings and run as an arbitrary UID in the root group. USER appuser diff --git a/docker/resources/common-scripts/configureDefaults b/docker/resources/common-scripts/configureDefaults index dd0c23cec11ed..14d28548a83eb 100755 --- a/docker/resources/common-scripts/configureDefaults +++ b/docker/resources/common-scripts/configureDefaults @@ -18,8 +18,6 @@ declare -A env_defaults env_defaults=( # Replace CLUSTER_ID with a unique base64 UUID using "bin/kafka-storage.sh random-uuid" ["CLUSTER_ID"]="5L6g3nShT-eMCtK--X86sw" - # Default Zookeeper connection string - override with KAFKA_ZOOKEEPER_CONNECT env var - ["KAFKA_ZOOKEEPER_CONNECT"]="localhost:2181" ) for key in "${!env_defaults[@]}"; do From e2f2a492878fd77bb5158a2dc558eec48b4f2162 Mon Sep 17 00:00:00 2001 From: Elbert Bautista Date: Thu, 28 Aug 2025 09:06:49 -0500 Subject: [PATCH 2/6] exclude kafka exporter capabilities due to stricter security policies --- Dockerfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index dc9a6a63c41ad..48df974bdcaa0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,10 +68,11 @@ RUN curl -L https://github.com/strimzi/strimzi-kafka-operator/archive/$STRIMZI_V RUN cp -r strimzi-kafka-operator/docker-images/kafka-based/kafka/scripts /tmp/strimzi-extracted/scripts ##### -# Add Kafka Exporter -# Strimzi Step 2: Copy Kafka Exporter Scripts from base image +# Exclude Kafka Exporter +# Strimzi Step 2: Strimzi comes with Kafka Exporter Scripts from base image +# However due to more strict security policies, we are opting to exclude this go-based dependency +# RUN cp -r /opt/kafka-exporter/* /tmp/strimzi-extracted/kafka-exporter ##### -RUN cp -r /opt/kafka-exporter/* /tmp/strimzi-extracted/kafka-exporter ##### # Add Prometheus JMX Exporter @@ -182,9 +183,10 @@ RUN chmod -R +x ${KAFKA_HOME}/strimzi-scripts RUN mv ${KAFKA_HOME}/strimzi-scripts/scripts/* ${KAFKA_HOME} RUN rm -rf ${KAFKA_HOME}/strimzi-scripts -# Copy Kafka Exporter directory -COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/kafka-exporter ${KAFKA_EXPORTER_HOME} -RUN chmod -R +x ${KAFKA_EXPORTER_HOME}/kafka_exporter +# Do Nothing with Kafka Exporter Directory +# As mentioned above, due to stricter security policies we are opting to exclude the kafka-exporter binary +# COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/kafka-exporter ${KAFKA_EXPORTER_HOME} +# RUN chmod -R +x ${KAFKA_EXPORTER_HOME}/kafka_exporter # Copy Prometheus JMX Exporter directory COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/prometheus-jmx-exporter ${JMX_EXPORTER_HOME} From 85c87f44f044c107b0024a86a3851c00ba52bf4d Mon Sep 17 00:00:00 2001 From: Elbert Bautista Date: Wed, 1 Oct 2025 08:56:27 -0500 Subject: [PATCH 3/6] update kafka strimzi support in Dockerfile --- Dockerfile | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 48df974bdcaa0..0d2749bc508f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,8 +45,6 @@ FROM quay.io/strimzi/kafka:0.47.0-kafka-3.9.1 AS strimzi-source-extractor USER root -ARG STRIMZI_VERSION=0.47.0 - # Create directories to copy content out RUN mkdir -p /tmp/strimzi-extracted/scripts \ /tmp/strimzi-extracted/kafka-exporter \ @@ -56,16 +54,12 @@ RUN mkdir -p /tmp/strimzi-extracted/scripts \ /tmp/strimzi-extracted/usr-bin ##### -# Add Kafka -# Strimzi Step 1: Copy Strimzi Kafka scripts (e.g., kafka_run.sh) directly from source -# Download and extract the Strimzi Kafka Operator tar file +# Add Kafka Scripts +# Strimzi Step 1: Copy Strimzi Kafka scripts (e.g., kafka_run.sh) from base image +# Exclude non-strimzi scripts (i.e we only need ones defined here https://github.com/strimzi/strimzi-kafka-operator/tree/0.47.0/docker-images/kafka-based/kafka/scripts) ##### -RUN curl -L https://github.com/strimzi/strimzi-kafka-operator/archive/$STRIMZI_VERSION.tar.gz -o strimzi-kafka-operator.tar.gz && \ - tar -xzf strimzi-kafka-operator.tar.gz && \ - mv strimzi-kafka-operator-$STRIMZI_VERSION strimzi-kafka-operator - -# Copy the required directories to the current working directory -RUN cp -r strimzi-kafka-operator/docker-images/kafka-based/kafka/scripts /tmp/strimzi-extracted/scripts +RUN cp -r /opt/kafka/* /tmp/strimzi-extracted/scripts +RUN rm -rf /tmp/strimzi-extracted/scripts/LICENSE /tmp/strimzi-extracted/scripts/NOTICE /tmp/strimzi-extracted/scripts/bin /tmp/strimzi-extracted/scripts/config /tmp/strimzi-extracted/scripts/libs /tmp/strimzi-extracted/scripts/licenses /tmp/strimzi-extracted/scripts/plugins /tmp/strimzi-extracted/scripts/site-docs ##### # Exclude Kafka Exporter @@ -92,9 +86,6 @@ RUN cp -r /opt/kafka/libs/* /tmp/strimzi-extracted/kafka-libs ##### RUN cp -r /opt/cruise-control/* /tmp/strimzi-extracted/cruise-control -# Clean up downloaded files to reduce image size -RUN rm -rf strimzi-kafka-operator.tar.gz strimzi-kafka-operator - # Verify the files were copied RUN ls -la /tmp/strimzi-extracted/scripts /tmp/strimzi-extracted/kafka-exporter /tmp/strimzi-extracted/prometheus-jmx-exporter /tmp/strimzi-extracted/kafka-libs /tmp/strimzi-extracted/cruise-control @@ -139,7 +130,7 @@ RUN set -eux ; \ apk cache clean; ##### -# Adapted Entrypoint in order to support +# Needed to support an adapted entrypoint in support of # backwared-compatible BLC installations that may have been referencing # a Confluent-based Kafka Image, with added support for initialization # via the Strimzi Operator for K8 installations (https://strimzi.io/) @@ -180,7 +171,7 @@ RUN mkdir -p ${KAFKA_HOME}/strimzi-scripts \ # Copy Strimzi Kafka scripts COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/scripts ${KAFKA_HOME}/strimzi-scripts RUN chmod -R +x ${KAFKA_HOME}/strimzi-scripts -RUN mv ${KAFKA_HOME}/strimzi-scripts/scripts/* ${KAFKA_HOME} +RUN mv ${KAFKA_HOME}/strimzi-scripts/* ${KAFKA_HOME} RUN rm -rf ${KAFKA_HOME}/strimzi-scripts # Do Nothing with Kafka Exporter Directory @@ -210,4 +201,11 @@ WORKDIR $KAFKA_HOME # will ignore these settings and run as an arbitrary UID in the root group. USER appuser +##### +# Adapted Entrypoint in order to support +# backwared-compatible BLC installations that may have been referencing +# a Confluent-based Kafka Image. Note that this entrypoint is not used +# when running this image via the Strimzi Operator as Strimzi +# provides its own entrypoint. See https://github.com/strimzi/strimzi-kafka-operator +##### CMD ["/etc/confluent/docker/run"] \ No newline at end of file From f384e1ea57c8b86ee9eb9ac673c0f162f01f2f73 Mon Sep 17 00:00:00 2001 From: jefffischer Date: Wed, 21 Jan 2026 08:32:44 -0600 Subject: [PATCH 4/6] Update for CVEs. Deduplicate the same jars with different versions. (#5) --- BROADLEAF.MD | 10 +- Dockerfile | 22 +++-- build.gradle | 12 +++ docker/resources/common-scripts/copy-jars.sh | 99 ++++++++++++++++++++ gradle/dependencies.gradle | 17 +++- 5 files changed, 143 insertions(+), 17 deletions(-) create mode 100644 docker/resources/common-scripts/copy-jars.sh diff --git a/BROADLEAF.MD b/BROADLEAF.MD index dc8c4cdb32414..46c4362f2a48f 100644 --- a/BROADLEAF.MD +++ b/BROADLEAF.MD @@ -4,12 +4,12 @@ - `./gradlew test` (Full test suite takes 4 hours!!) - At the time of this writing, `SocketServerTest` consistently fails - even without any changes. As a result, ignoring this particular test result. - `./gradlew clean releaseTarGz` -- For local testing: `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.22-jre-21.0.8_9-r1 BUILD_AND_LOAD_LOCAL_PLATFORM_ONLY=true docker buildx bake -f docker-bake.hcl` +- For local testing: `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft1 BUILD_AND_LOAD_LOCAL_PLATFORM_ONLY=true docker buildx bake -f docker-bake.hcl` - For release: - - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.22-jre-21.0.8_9-r1 LIMIT_PLATFORM="linux/arm64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl` - - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.22-jre-21.0.8_9-r1 LIMIT_PLATFORM="linux/amd64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl` - - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.22-jre-21.0.8_9-r1 docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl --push` - - (Increment the `r1` to a higher value if only changing java or os deps without changing app, jre, or os versions) + - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft1 LIMIT_PLATFORM="linux/arm64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl` + - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft1 LIMIT_PLATFORM="linux/amd64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl` + - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft1 docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl --push` + - (Increment the `kraft1` to a higher value if only changing java or os deps without changing app, jre, or os versions) ## Update Notes - The `Dockerfile` at the root of this project was adapted from the `jvm` instruction set located at `docker/jvm/Dockerfile`, and was updated to pull from a local kafka build (i.e. `./core/build/distributions/`) instead of the Apache Distribution Repo and references existing scripts and resources in the `./docker` directory. - A `docker-bake.hcl` file was added in support of a multi-platform builds \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 0d2749bc508f9..9ac205469782b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ # Stage 1: Build Java Shared Archive (JSA) # Adapted from the official Apache Dockerfile -FROM eclipse-temurin:21.0.8_9-jre-alpine-3.22 AS build-jsa +FROM eclipse-temurin:21.0.9_10-jre-alpine-3.23 AS build-jsa USER root @@ -91,7 +91,7 @@ RUN ls -la /tmp/strimzi-extracted/scripts /tmp/strimzi-extracted/kafka-exporter # Stage 3: Main Kafka image build # Adapted from the official Apache Dockerfile -FROM eclipse-temurin:21.0.8_9-jre-alpine-3.22 +FROM eclipse-temurin:21.0.9_10-jre-alpine-3.23 # exposed ports EXPOSE 9092 @@ -138,6 +138,7 @@ RUN set -eux ; \ COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa COPY --chown=appuser:0 docker/resources/common-scripts /etc/kafka/docker +RUN chmod +x /etc/kafka/docker/copy-jars.sh COPY --chown=appuser:0 docker/jvm/launch /etc/kafka/docker/launch VOLUME ["/etc/kafka/secrets", "/var/lib/kafka/data", "/mnt/shared/config"] @@ -183,14 +184,19 @@ RUN rm -rf ${KAFKA_HOME}/strimzi-scripts COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/prometheus-jmx-exporter ${JMX_EXPORTER_HOME} # Copy Strimzi Agents and other Kafka libraries -COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/kafka-libs/ ${KAFKA_HOME}/strimzi-kafka-libs/ -# Copy from strimzi temp directory into main kafka lib, skip files if they already exist -RUN cp -n ${KAFKA_HOME}/strimzi-kafka-libs/* ${KAFKA_HOME}/libs -RUN rm -rf ${KAFKA_HOME}/strimzi-kafka-libs +# Use a bind mount to access files from the previous stage without copying them permanently into a layer first +RUN --mount=type=bind,from=strimzi-source-extractor,source=/tmp/strimzi-extracted/kafka-libs,target=/tmp/strimzi-kafka-libs-source \ + /etc/kafka/docker/copy-jars.sh /tmp/strimzi-kafka-libs-source ${KAFKA_HOME}/libs && \ + chown -R appuser:root ${KAFKA_HOME}/libs # Copy Cruise Control libraries -COPY --from=strimzi-source-extractor --chown=appuser:root /tmp/strimzi-extracted/cruise-control/ ${CRUISE_CONTROL_HOME} -RUN chmod -R +x ${CRUISE_CONTROL_HOME} || true +# Use a bind mount to access files from the previous stage +# We copy libs using the deduplication script, and other files directly +RUN --mount=type=bind,from=strimzi-source-extractor,source=/tmp/strimzi-extracted/cruise-control,target=/tmp/cruise-control-source \ + bash -c '/etc/kafka/docker/copy-jars.sh /tmp/cruise-control-source/libs ${CRUISE_CONTROL_HOME}/libs ${KAFKA_HOME}/libs && \ + find /tmp/cruise-control-source -maxdepth 1 -mindepth 1 -not -name libs -exec cp -r {} ${CRUISE_CONTROL_HOME}/ \;' && \ + chown -R appuser:root ${CRUISE_CONTROL_HOME} && \ + chmod -R +x ${CRUISE_CONTROL_HOME} # Important to set this to the kafka home as strimzi scripts have relative path references WORKDIR $KAFKA_HOME diff --git a/build.gradle b/build.gradle index 2a50ecbf309ab..b067d295ba83b 100644 --- a/build.gradle +++ b/build.gradle @@ -161,6 +161,8 @@ allprojects { // ZooKeeper (potentially older and containing CVEs) libs.nettyHandler, libs.nettyTransportNativeEpoll, + libs.nettyCodecHttp, + libs.nettyCodecHttp2, // be explicit about the reload4j version instead of relying on the transitive versions libs.reload4j, libs.commonsLang3 @@ -972,6 +974,16 @@ project(':core') { // ZooKeeperMain depends on commons-cli but declares the dependency as `provided` implementation libs.commonsCli + // Strimzi and cruise control deps + // Overridden for CVEs + implementation libs.nettyCodecHttp + implementation libs.nettyCodecHttp2 + implementation libs.nimbusJoseJwt + implementation libs.log4J + implementation libs.vertxCore + implementation libs.vertxWeb + // End Strimzi and cruise control deps + compileOnly libs.reload4j testImplementation project(':clients').sourceSets.test.output diff --git a/docker/resources/common-scripts/copy-jars.sh b/docker/resources/common-scripts/copy-jars.sh new file mode 100644 index 0000000000000..33a9342c872e3 --- /dev/null +++ b/docker/resources/common-scripts/copy-jars.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# +# Copies files from SOURCE to TARGET. +# 1. If JAR exists in TARGET (ignoring version), skip. +# 2. If JAR exists in ADDITIONAL_CHECK_DIR (ignoring version), copy the version from ADDITIONAL_CHECK_DIR to TARGET. +# 3. Otherwise, copy from SOURCE to TARGET. +# +# Usage: copy-jars.sh [ADDITIONAL_CHECK_DIR] + +set -e + +SOURCE_DIR="$1" +TARGET_DIR="$2" +ADDITIONAL_CHECK_DIR="$3" + +if [ -z "$SOURCE_DIR" ] || [ -z "$TARGET_DIR" ]; then + echo "Usage: $0 [ADDITIONAL_CHECK_DIR]" + exit 1 +fi + +# Ensure target exists +mkdir -p "$TARGET_DIR" + +echo "Copying files from $SOURCE_DIR to $TARGET_DIR with version deduplication..." +if [ -n "$ADDITIONAL_CHECK_DIR" ]; then + echo "Also checking for duplicates in $ADDITIONAL_CHECK_DIR" +fi + +# Helper to extract base name (artifact ID) +get_base_name() { + local filename=$1 + # Remove version suffix starting with a hyphen followed by a digit + echo "$filename" | sed -E 's/-[0-9].*\.jar$//' +} + +# Process JAR files +# We use a loop over the glob to handle filenames with spaces correctly +shopt -s nullglob +for src_jar in "$SOURCE_DIR"/*.jar; do + filename=$(basename "$src_jar") + base_name=$(get_base_name "$filename") + + duplicate_in_target=false + + # Check against all jars in target + for target_jar in "$TARGET_DIR"/*.jar; do + t_filename=$(basename "$target_jar") + t_base_name=$(get_base_name "$t_filename") + + if [ "$base_name" == "$t_base_name" ]; then + echo "Skipping $filename (duplicate of $t_filename in target)" + duplicate_in_target=true + break + fi + done + + if [ "$duplicate_in_target" = true ]; then + continue + fi + + # Check additional directory if provided + copied_from_additional=false + if [ -n "$ADDITIONAL_CHECK_DIR" ] && [ -d "$ADDITIONAL_CHECK_DIR" ]; then + for check_jar in "$ADDITIONAL_CHECK_DIR"/*.jar; do + c_filename=$(basename "$check_jar") + c_base_name=$(get_base_name "$c_filename") + + if [ "$base_name" == "$c_base_name" ]; then + echo "Found duplicate in additional dir: $c_filename. Copying that version to target." + cp "$check_jar" "$TARGET_DIR/" + copied_from_additional=true + break + fi + done + fi + + if [ "$copied_from_additional" = false ]; then + echo "Copying $filename" + cp "$src_jar" "$TARGET_DIR/" + fi +done + +# Process non-JAR files and directories +# We copy them if they don't exist in target (exact name match) +for src_file in "$SOURCE_DIR"/*; do + filename=$(basename "$src_file") + + # Skip if it's a jar (already handled) + if [[ "$filename" == *.jar ]]; then + continue + fi + + if [ ! -e "$TARGET_DIR/$filename" ]; then + echo "Copying $filename" + cp -r "$src_file" "$TARGET_DIR/" + else + echo "Skipping $filename (exists)" + fi +done diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index f5901ce1f13e8..9c0a85698eaea 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -112,7 +112,7 @@ versions += [ jaxrs: "2.1.1", jfreechart: "1.0.0", jopt: "5.0.4", - jose4j: "0.9.4", + jose4j: "0.9.6", junit: "5.10.2", jqwik: "1.8.3", kafka_0100: "0.10.0.1", @@ -143,7 +143,7 @@ versions += [ lz4: "1.8.0", mavenArtifact: "3.9.6", metrics: "2.2.0", - netty: "4.1.125.Final", + netty: "4.1.130.Final", opentelemetryProto: "1.0.0-alpha", protobuf: "3.25.5", // a dependency of opentelemetryProto pcollections: "4.0.1", @@ -165,7 +165,10 @@ versions += [ // When updating the zstd version, please do as well in docker/native/native-image-configs/resource-config.json // Also make sure the compression levels in org.apache.kafka.common.record.CompressionType are still valid zstd: "1.5.6-4", - junitPlatform: "1.10.2" + junitPlatform: "1.10.2", + nimbusJoseJwt: "9.37.4", + log4J: "2.25.3", + vertx: "4.5.24" ] libs += [ @@ -247,6 +250,8 @@ libs += [ mockitoJunitJupiter: "org.mockito:mockito-junit-jupiter:$mockitoVersion", nettyHandler: "io.netty:netty-handler:$versions.netty", nettyTransportNativeEpoll: "io.netty:netty-transport-native-epoll:$versions.netty", + nettyCodecHttp: "io.netty:netty-codec-http:$versions.netty", + nettyCodecHttp2: "io.netty:netty-codec-http2:$versions.netty", pcollections: "org.pcollections:pcollections:$versions.pcollections", opentelemetryProto: "io.opentelemetry.proto:opentelemetry-proto:$versions.opentelemetryProto", protobuf: "com.google.protobuf:protobuf-java:$versions.protobuf", @@ -267,5 +272,9 @@ libs += [ jfreechart: "jfreechart:jfreechart:$versions.jfreechart", mavenArtifact: "org.apache.maven:maven-artifact:$versions.mavenArtifact", zstd: "com.github.luben:zstd-jni:$versions.zstd", - httpclient: "org.apache.httpcomponents:httpclient:$versions.httpclient" + httpclient: "org.apache.httpcomponents:httpclient:$versions.httpclient", + nimbusJoseJwt: "com.nimbusds:nimbus-jose-jwt:$versions.nimbusJoseJwt", + log4J: "org.apache.logging.log4j:log4j-core:$versions.log4J", + vertxCore: "io.vertx:vertx-core:$versions.vertx", + vertxWeb: "io.vertx:vertx-web:$versions.vertx" ] From f6925a5cff501899fff3feb88c827f90d22bc65f Mon Sep 17 00:00:00 2001 From: jefffischer Date: Fri, 23 Jan 2026 10:13:34 -0600 Subject: [PATCH 5/6] Update for CVEs. --- Dockerfile | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9ac205469782b..d00055914a7cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,8 +29,15 @@ ARG DISTRO_NAME=kafka_2.13-3.9.1 COPY core/build/distributions/$DISTRO_NAME.tgz / RUN set -eux ; \ + # 1. Add Alpine Edge repositories + echo "https://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories; \ + echo "https://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories; \ + # 2. Update and upgrade apk-tools apk update ; \ - apk upgrade ; \ + apk add --upgrade apk-tools; \ + # 3. Force upgrade to Edge versions + apk upgrade --available ; \ + # 4. Install build dependencies (wget, gcompat, etc.) apk add --no-cache wget gcompat gpg gpg-agent procps bash; \ mkdir opt/kafka; \ tar xfz $DISTRO_NAME.tgz -C /opt/kafka --strip-components 1; @@ -112,9 +119,17 @@ COPY core/build/distributions/$DISTRO_NAME.tgz / # We assume appuser UID for standard Kubernetes, and an arbitrary UID + root group (0) # for OpenShift. Thus, grant both of those ownership here. RUN set -eux ; \ + # 1. Add Alpine Edge repositories + echo "https://dl-cdn.alpinelinux.org/alpine/edge/main" >> /etc/apk/repositories; \ + echo "https://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories; \ + # 2. Update and upgrade apk-tools apk update ; \ - apk upgrade ; \ + apk add --upgrade apk-tools; \ + # 3. Force upgrade all OS packages to Edge versions (patches CVEs) + apk upgrade --available ; \ + # 4. Install runtime dependencies apk add --no-cache wget gcompat gpg gpg-agent procps bash su-exec tini grep curl; \ + # 5. Continue with Kafka installation and configuration mkdir opt/kafka; \ tar xfz $DISTRO_NAME.tgz -C /opt/kafka --strip-components 1; \ mkdir -p /var/lib/kafka/data /etc/kafka/secrets; \ @@ -126,6 +141,7 @@ RUN set -eux ; \ cp /opt/kafka/config/log4j.properties /etc/kafka/docker/log4j.properties; \ cp /opt/kafka/config/tools-log4j.properties /etc/kafka/docker/tools-log4j.properties; \ rm $DISTRO_NAME.tgz; \ + # 6. Cleanup (remove build-only tools if desired, though gpg/wget were just installed above) apk del wget gpg gpg-agent; \ apk cache clean; From d9b0f8363c4c82559c6820c53253511bda023d5e Mon Sep 17 00:00:00 2001 From: jefffischer Date: Mon, 2 Feb 2026 08:59:25 -0600 Subject: [PATCH 6/6] kraft2 for os cves --- BROADLEAF.MD | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/BROADLEAF.MD b/BROADLEAF.MD index 46c4362f2a48f..e8a0b15928d6b 100644 --- a/BROADLEAF.MD +++ b/BROADLEAF.MD @@ -4,11 +4,11 @@ - `./gradlew test` (Full test suite takes 4 hours!!) - At the time of this writing, `SocketServerTest` consistently fails - even without any changes. As a result, ignoring this particular test result. - `./gradlew clean releaseTarGz` -- For local testing: `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft1 BUILD_AND_LOAD_LOCAL_PLATFORM_ONLY=true docker buildx bake -f docker-bake.hcl` +- For local testing: `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft2 BUILD_AND_LOAD_LOCAL_PLATFORM_ONLY=true docker buildx bake -f docker-bake.hcl` - For release: - - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft1 LIMIT_PLATFORM="linux/arm64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl` - - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft1 LIMIT_PLATFORM="linux/amd64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl` - - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft1 docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl --push` + - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft2 LIMIT_PLATFORM="linux/arm64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl` + - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft2 LIMIT_PLATFORM="linux/amd64" docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl` + - `FULLY_QUALIFIED_MAIN_IMAGE_TAG=repository.broadleafcommerce.com:5001/broadleaf/kafka:3.9.1-alpine-3.23-jre-21.0.9_10-kraft2 docker buildx bake --builder blc-devops-multiplatform-builder -f docker-bake.hcl --push` - (Increment the `kraft1` to a higher value if only changing java or os deps without changing app, jre, or os versions) ## Update Notes - The `Dockerfile` at the root of this project was adapted from the `jvm` instruction set located at `docker/jvm/Dockerfile`, and was updated to pull from a local kafka build (i.e. `./core/build/distributions/`) instead of the Apache Distribution Repo and references existing scripts and resources in the `./docker` directory.