From cf3dd96642b7ef0b5c0f7bb410a6fe93e4571613 Mon Sep 17 00:00:00 2001 From: Daniel Zatovic Date: Wed, 2 Jul 2025 10:48:51 +0200 Subject: [PATCH 1/6] overlay profiles: Enable cryptsetup in SDK systemd The cryptsetup useflag is required for signing sysexts built with systemd-repart. Signed-off-by: Daniel Zatovic --- .../coreos-overlay/profiles/coreos/targets/sdk/package.use | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use index 41459225643..641b433bda1 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/sdk/package.use @@ -32,3 +32,6 @@ x11-libs/pixman static-libs # Get latest EDK2 firmware for Secure Boot on arm64. app-emulation/qemu -pin-upstream-blobs + +# Needed for signed sysexts using systemd-repart +sys-apps/systemd cryptsetup From c22d1c206c9fd214209e319544b393bf142b0adb Mon Sep 17 00:00:00 2001 From: Daniel Zatovic Date: Wed, 30 Jul 2025 10:24:18 +0200 Subject: [PATCH 2/6] sysext: Sign OS-dependent sysexts Generate an ephemeral sysext signing key, that is injected into the image's sysext root of trust. All OS-dependent sysexts will be signed by this key and the private key (stored in /tmp) will be discarded on SDK container exit. Signed-off-by: Daniel Zatovic --- build_library/prod_image_util.sh | 4 ++++ build_library/sysext_prod_builder | 32 ++++++++++++++++++++------ build_library/vm_image_util.sh | 2 +- build_sysext | 19 ++++++++++++---- sdk_lib/sdk_entry.sh | 37 +++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 12 deletions(-) diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 7463f26d402..7a0c34b50ca 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -170,6 +170,10 @@ EOF # Remove source locale data, only need to ship the compiled archive. sudo rm -rf ${root_fs_dir}/usr/share/i18n/ + # Inject ephemeral sysext signing certificate + sudo mkdir -p "${root_fs_dir}/usr/lib/verity.d" + sudo cp "${SYSEXT_SIGNING_KEY_DIR}/sysexts.crt" "${root_fs_dir}/usr/lib/verity.d" + # Finish image will move files from /etc to /usr/share/flatcar/etc. # Note that image filesystem contents generated by finish_image will not # include sysext contents (only the sysext squashfs files themselves). diff --git a/build_library/sysext_prod_builder b/build_library/sysext_prod_builder index d90fb4a1da2..87599feb803 100755 --- a/build_library/sysext_prod_builder +++ b/build_library/sysext_prod_builder @@ -63,7 +63,7 @@ create_prod_sysext() { # The --install_root_basename="${name}-base-sysext-rootfs" flag is # important - it sets the name of a rootfs directory, which is used # to determine the package target in coreos/base/profile.bashrc - sudo "FLATCAR_BUILD_ID=$FLATCAR_BUILD_ID" "${SCRIPTS_DIR}/build_sysext" \ + sudo -E "FLATCAR_BUILD_ID=$FLATCAR_BUILD_ID" "${SCRIPTS_DIR}/build_sysext" \ --board="${BOARD}" \ --image_builddir="${workdir}/sysext-build" \ --squashfs_base="${base_sysext}" \ @@ -99,6 +99,14 @@ sysext_mountdir="${BUILD_DIR}/prod-sysext-work/mounts" sysext_base="${sysext_workdir}/base-os.squashfs" function cleanup() { + IFS=':' read -r -a mounted_sysexts <<< "$sysext_lowerdirs" + # skip the rootfs + mounted_sysexts=("${mounted_sysexts[@]:1}") + + for sysext in "${mounted_sysexts[@]}"; do + sudo systemd-dissect --umount --rmdir "$sysext" + done + sudo umount "${sysext_mountdir}"/* || true rm -rf "${sysext_workdir}" || true } @@ -116,6 +124,7 @@ sudo mksquashfs "${root_fs_dir}" "${sysext_base}" -noappend -xattrs-exclude '^bt # for combined overlay later. prev_pkginfo="" sysext_lowerdirs="${sysext_mountdir}/rootfs-lower" +mkdir -p "${sysext_mountdir}" for sysext in ${sysexts_list//,/ }; do # format is ":/" name="${sysext%|*}" @@ -129,12 +138,21 @@ for sysext in ${sysexts_list//,/ }; do "${grp_pkg}" \ "${prev_pkginfo}" - mkdir -p "${sysext_mountdir}/${name}" \ - "${sysext_mountdir}/${name}_pkginfo" - sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}.raw" \ - "${sysext_mountdir}/${name}" - sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}_pkginfo.raw" \ - "${sysext_mountdir}/${name}_pkginfo" + sudo systemd-dissect \ + --read-only \ + --mount \ + --mkdir \ + --image-policy='root=encrypted+unprotected+absent:usr=encrypted+unprotected+absent' \ + "${sysext_output_dir}/${name}.raw" \ + "${sysext_mountdir}/${name}" + + sudo systemd-dissect \ + --read-only \ + --mount \ + --mkdir \ + --image-policy='root=encrypted+unprotected+absent:usr=encrypted+unprotected+absent' \ + "${sysext_output_dir}/${name}_pkginfo.raw" \ + "${sysext_mountdir}/${name}_pkginfo" sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}" sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}_pkginfo" diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index ac83929cfa6..159fede04e4 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -602,7 +602,7 @@ install_oem_sysext() { fi mkdir -p "${built_sysext_dir}" - sudo "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}" + sudo -E "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}" local installed_sysext_oem_dir='/oem/sysext' local installed_sysext_file_prefix="${oem_sysext}-${version}" diff --git a/build_sysext b/build_sysext index 92d6abc4009..be7e5047746 100755 --- a/build_sysext +++ b/build_sysext @@ -304,14 +304,25 @@ if [[ -n "${invalid_files}" ]]; then die "Invalid file ownership: ${invalid_files}" fi -mksquashfs "${BUILD_DIR}/${FLAGS_install_root_basename}" "${BUILD_DIR}/${SYSEXTNAME}.raw" \ - -noappend -xattrs-exclude '^btrfs.' -comp "${FLAGS_compression}" ${FLAGS_mksquashfs_opts} +systemd-repart \ + --private-key="${SYSEXT_SIGNING_KEY_DIR}/sysexts.key" \ + --certificate="${SYSEXT_SIGNING_KEY_DIR}/sysexts.crt" \ + --make-ddi=sysext \ + --copy-source="${BUILD_DIR}/${FLAGS_install_root_basename}" \ + "${BUILD_DIR}/${SYSEXTNAME}.raw" + rm -rf "${BUILD_DIR}"/{fs-root,"${FLAGS_install_root_basename}",workdir} # Generate reports mkdir "${BUILD_DIR}/img-rootfs" -mount -rt squashfs -o loop,nodev "${BUILD_DIR}/${SYSEXTNAME}.raw" "${BUILD_DIR}/img-rootfs" +systemd-dissect --read-only \ + --mount \ + --mkdir \ + --image-policy='root=encrypted+unprotected+absent:usr=encrypted+unprotected+absent' \ + "${BUILD_DIR}/${SYSEXTNAME}.raw" \ + "${BUILD_DIR}/img-rootfs" + write_contents "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents.txt" write_contents_with_technical_details "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents_wtd.txt" write_disk_space_usage_in_paths "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_disk_usage.txt" -umount "${BUILD_DIR}/img-rootfs" +systemd-dissect --umount --rmdir "${BUILD_DIR}/img-rootfs" diff --git a/sdk_lib/sdk_entry.sh b/sdk_lib/sdk_entry.sh index 6458bf8271a..262de3751b1 100755 --- a/sdk_lib/sdk_entry.sh +++ b/sdk_lib/sdk_entry.sh @@ -88,6 +88,43 @@ if ! grep -q 'export MODULE_SIGNING_KEY_DIR=' /home/sdk/.bashrc; then fi fi +# Ensure sysext signing keys exist; regenerate if directory or files missing +if grep -q 'export SYSEXT_SIGNING_KEY_DIR' /home/sdk/.bashrc; then + _existing_sysext_dir=$(source /home/sdk/.bashrc 2>/dev/null; echo "$SYSEXT_SIGNING_KEY_DIR") + if [[ -z "$_existing_sysext_dir" || ! -d "$_existing_sysext_dir" || ! -s "$_existing_sysext_dir/sysexts.key" || ! -s "$_existing_sysext_dir/sysexts.crt" ]]; then + # Drop stale export so block below regenerates + sed -i -e '/export SYSEXT_SIGNING_KEY_DIR=/d' /home/sdk/.bashrc + fi +fi +grep -q 'export SYSEXT_SIGNING_KEY_DIR' /home/sdk/.bashrc || { + if [[ ${COREOS_OFFICIAL:-0} -eq 1 ]]; then + SYSEXT_SIGNING_KEY_DIR=$(su sdk -c "mktemp -d") + else + SYSEXT_SIGNING_KEY_DIR="/home/sdk/.sysext-signing-keys" + su sdk -c "mkdir -p ${SYSEXT_SIGNING_KEY_DIR@Q}" + fi + if [[ ! "$SYSEXT_SIGNING_KEY_DIR" || ! -d "$SYSEXT_SIGNING_KEY_DIR" ]]; then + echo "Failed to create directory for sysext signing keys." + else + echo "export SYSEXT_SIGNING_KEY_DIR='$SYSEXT_SIGNING_KEY_DIR'" >> /home/sdk/.bashrc + fi + pushd "$SYSEXT_SIGNING_KEY_DIR" > /dev/null + build_id=$(source "/mnt/host/source/.repo/manifests/version.txt"; echo "$FLATCAR_BUILD_ID") + # Generate sysext signing key only if missing or empty + if [[ ! -s sysexts.key || ! -s sysexts.crt ]]; then + su sdk -c "openssl req -new -nodes -utf8 \ + -x509 -batch -sha256 \ + -days 36000 \ + -outform PEM \ + -out sysexts.crt \ + -keyout sysexts.key \ + -newkey 4096 \ + -subj '/CN=Flatcar sysext key/OU=$build_id'" \ + || echo "Generating sysext signing key failed" + fi + popd > /dev/null +} + # This is ugly. # We need to sudo su - sdk -c so the SDK user gets a fresh login. # 'sdk' is member of multiple groups, and plain docker USER only From 309d071a6fa791e806ae003fdb1e0b8195c76628 Mon Sep 17 00:00:00 2001 From: Daniel Zatovic Date: Thu, 23 Oct 2025 10:25:23 +0200 Subject: [PATCH 3/6] sysext: Add OS-dependent sysext compression We removed the sysext compression, because we double-compression is redundant for sysexts stored in already coimpressed BTRFS /usr. However, OS-dependent sysexts that are downloaded on-demand were now also uncompressed. This commit brings back the compression via SYSTEMD_REPART_MKFS_OPTIONS_EROFS option. Signed-off-by: Daniel Zatovic --- build_library/sysext_prod_builder | 4 ++++ build_library/vm_image_util.sh | 4 ++++ build_sysext | 33 ++++++++++++++++++++++--------- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/build_library/sysext_prod_builder b/build_library/sysext_prod_builder index 87599feb803..8e570806303 100755 --- a/build_library/sysext_prod_builder +++ b/build_library/sysext_prod_builder @@ -63,11 +63,15 @@ create_prod_sysext() { # The --install_root_basename="${name}-base-sysext-rootfs" flag is # important - it sets the name of a rootfs directory, which is used # to determine the package target in coreos/base/profile.bashrc + # + # Built-in sysexts are stored in the compressed /usr partition, so we + # disable compression to avoid double-compression. sudo -E "FLATCAR_BUILD_ID=$FLATCAR_BUILD_ID" "${SCRIPTS_DIR}/build_sysext" \ --board="${BOARD}" \ --image_builddir="${workdir}/sysext-build" \ --squashfs_base="${base_sysext}" \ --generate_pkginfo \ + --compression=none \ --install_root_basename="${name}-base-sysext-rootfs" \ "${build_sysext_opts[@]}" \ "${name}" "${grp_pkg[@]}" diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 159fede04e4..73eb43b8125 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -585,11 +585,15 @@ install_oem_sysext() { # important - it sets the name of a rootfs directory, which is # used to determine the package target in # coreos/base/profile.bashrc + # + # OEM sysexts are stored in the compressed partition, so we disable + # compression to avoid double-compression. local build_sysext_flags=( --board="${BOARD}" --squashfs_base="${VM_SRC_SYSEXT_IMG}" --image_builddir="${built_sysext_dir}" --metapkgs="${metapkg}" + --compression=none --install_root_basename="${VM_IMG_TYPE}-oem-sysext-rootfs" ) local overlay_path mangle_fs diff --git a/build_sysext b/build_sysext index be7e5047746..0b7ea4d5f32 100755 --- a/build_sysext +++ b/build_sysext @@ -35,10 +35,10 @@ DEFINE_boolean generate_pkginfo "${FLAGS_FALSE}" \ "Generate an additional squashfs '_pkginfo.raw' with portage package meta-information (/var/db ...). Useful for creating sysext dependencies; see 'base_pkginfo' below." DEFINE_string base_pkginfo "" \ "Colon-separated list of pkginfo squashfs paths / files generated via 'generate_pkginfo' to base this sysext on. The corresponding base sysexts are expected to be merged with the sysext generated." -DEFINE_string compression "zstd" \ - "Compression to use for sysext squashfs. One of 'gzip', 'lzo', 'lz4', 'xz', or 'zstd'. Must be supported by the Flatcar squashfs kernel module in order for the sysext to work." -DEFINE_string mksquashfs_opts "" \ - "Additional command line options to pass to mksquashfs. See 'man 1 mksquashfs'. If is 'zstd' (the default), this option defaults to '-Xcompression-level 22 -b 512K'. Otherwise the default is empty." +DEFINE_string compression "lz4hc" \ + "Compression to use for sysext EROFS image. Options: 'lz4', 'lz4hc', 'zstd', or 'none'. Default is 'lz4hc'." +DEFINE_string mkerofs_opts "" \ + "Additional mkfs.erofs options to pass via SYSTEMD_REPART_MKFS_OPTIONS_EROFS. If not specified, defaults are used based on compression type." DEFINE_boolean ignore_version_mismatch "${FLAGS_FALSE}" \ "Ignore version mismatch between SDK board packages and base squashfs. DANGEROUS." DEFINE_string install_root_basename "${default_install_root_basename}" \ @@ -112,10 +112,6 @@ fi BUILD_DIR=$(realpath "${FLAGS_image_builddir}") mkdir -p "${BUILD_DIR}" -if [[ "${FLAGS_compression}" = "zstd" && -z "${FLAGS_mksquashfs_opts}" ]] ; then - FLAGS_mksquashfs_opts="-Xcompression-level 22 -b 512k" -fi - source "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 source "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 @@ -248,7 +244,7 @@ if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then mkdir -p "${BUILD_DIR}/img-pkginfo/var/db" cp -R "${BUILD_DIR}/${FLAGS_install_root_basename}/var/db/pkg" "${BUILD_DIR}/img-pkginfo/var/db/" mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" \ - -noappend -xattrs-exclude '^btrfs.' -comp "${FLAGS_compression}" ${FLAGS_mksquashfs_opts} + -noappend -xattrs-exclude '^btrfs.' -comp zstd -Xcompression-level 22 -b 512k fi info "Writing ${SYSEXTNAME}_packages.txt" @@ -304,6 +300,25 @@ if [[ -n "${invalid_files}" ]]; then die "Invalid file ownership: ${invalid_files}" fi +# Set up EROFS compression options based on compression type +if [[ "${FLAGS_compression}" != "none" ]]; then + export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="-z${FLAGS_compression}" + + if [[ -n "${FLAGS_mkerofs_opts}" ]]; then + # User provided custom options + export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="${SYSTEMD_REPART_MKFS_OPTIONS_EROFS} ${FLAGS_mkerofs_opts}" + elif [[ "${FLAGS_compression}" = "lz4hc" ]]; then + # Default options for lz4hc + export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="${SYSTEMD_REPART_MKFS_OPTIONS_EROFS},12 -C65536 -Efragments,ztailpacking" + elif [[ "${FLAGS_compression}" = "zstd" ]]; then + # Default options for zstd + export SYSTEMD_REPART_MKFS_OPTIONS_EROFS="${SYSTEMD_REPART_MKFS_OPTIONS_EROFS},level=22 -C524288 -Efragments,ztailpacking" + fi + info "Building sysext with ${FLAGS_compression} compression" +else + info "Building sysext without compression (built-in sysexts)" +fi + systemd-repart \ --private-key="${SYSEXT_SIGNING_KEY_DIR}/sysexts.key" \ --certificate="${SYSEXT_SIGNING_KEY_DIR}/sysexts.crt" \ From 7942aad6bc8f4efcef68694fceca700969006afd Mon Sep 17 00:00:00 2001 From: Daniel Zatovic Date: Wed, 5 Nov 2025 19:10:48 +0100 Subject: [PATCH 4/6] changelog: Add entry for signed OS-dependent sysexts Signed-off-by: Daniel Zatovic --- changelog/changes/2025-11-05-signed-os-dependent-sysexts.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/changes/2025-11-05-signed-os-dependent-sysexts.md diff --git a/changelog/changes/2025-11-05-signed-os-dependent-sysexts.md b/changelog/changes/2025-11-05-signed-os-dependent-sysexts.md new file mode 100644 index 00000000000..196b9266b12 --- /dev/null +++ b/changelog/changes/2025-11-05-signed-os-dependent-sysexts.md @@ -0,0 +1 @@ +- OS-dependent sysexts (e.g., docker-flatcar, containerd-flatcar) are now cryptographically signed using dm-verity roothash signatures. This enables stricter sysext policies via systemd-sysext and provides a foundation for verifying user-provided extensions in future releases. The format changed from squashfs to erofs-based Discoverable Disk Images (DDI). ([scripts#3162](https://github.com/flatcar/scripts/pull/3162)) From 0251858bd9a645ac72a44ea28be6d2b7c1a00664 Mon Sep 17 00:00:00 2001 From: Daniel Zatovic Date: Mon, 5 Jan 2026 16:44:32 +0100 Subject: [PATCH 5/6] sysext: Move OEM sysext build to image phase Move OEM sysext building from the vms phase to the image phase. This ensures OEM sysexts are signed with the same ephemeral key as other sysexts, which is generated during image build and discarded afterward. - Add create_oem_sysexts() to build all OEM sysexts during image build - Add oem_sysexts.sh with OEM sysext definitions - Update install_oem_sysext() to use prebuilt sysexts - Add OEM sysext download to vms.sh for CI builds Signed-off-by: Daniel Zatovic --- build_image | 12 ++++- build_library/oem_sysexts.sh | 84 +++++++++++++++++++++++++++++++ build_library/prod_image_util.sh | 48 ++++++++++++++++++ build_library/vm_image_util.sh | 58 ++++----------------- build_packages | 86 +++++++++++++++++--------------- build_sysext | 2 +- ci-automation/image.sh | 2 +- ci-automation/image_changes.sh | 33 +----------- ci-automation/vms.sh | 11 ++++ 9 files changed, 212 insertions(+), 124 deletions(-) create mode 100644 build_library/oem_sysexts.sh diff --git a/build_image b/build_image index c446b7e6583..aa0e57f1580 100755 --- a/build_image +++ b/build_image @@ -60,10 +60,12 @@ different forms. This scripts can be used to build the following: prod - Production image for CoreOS. This image is for booting (default if no argument is given). prodtar - Production container tar ball (implies prod). This can e.g. be used to run the Flatcar production image as a container (run machinectl import-tar or docker import). container - Developer image with single filesystem, bootable by nspawn. +sysext - Build extra sysexts (podman, python, zfs, etc.). +oem_sysext - Build OEM sysexts for all supported platforms. Examples: -build_image --board= [prod] [prodtar] [container] - builds developer and production images/tars. +build_image --board= [prod] [prodtar] [container] [sysext] [oem_sysext] - builds developer and production images/tars. ... " show_help_if_requested "$@" @@ -81,7 +83,7 @@ DEFINE_string version "" \ # Parse command line. FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV:-prod}" +eval set -- "${FLAGS_ARGV:-prod oem_sysext}" # Only now can we die on error. shflags functions leak non-zero error codes, # so will die prematurely if 'switch_to_strict_mode' is specified before now. @@ -103,17 +105,20 @@ fi . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/vm_image_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1 PROD_IMAGE=0 PROD_TAR=0 CONTAINER=0 SYSEXT=0 +OEM_SYSEXT=0 for arg in "$@"; do case "${arg}" in prod) PROD_IMAGE=1 ;; prodtar) PROD_IMAGE=1 PROD_TAR=1 ;; container) CONTAINER=1 ;; sysext) SYSEXT=1 ;; + oem_sysext) OEM_SYSEXT=1 ;; *) die_notrace "Unknown image type ${arg}" ;; esac done @@ -187,6 +192,9 @@ fi if [[ "${SYSEXT}" -eq 1 ]]; then create_prod_sysexts "${FLATCAR_PRODUCTION_IMAGE_NAME}" fi +if [[ "${OEM_SYSEXT}" -eq 1 ]]; then + create_oem_sysexts "${FLATCAR_PRODUCTION_IMAGE_NAME}" +fi if [[ ${FLAGS_extract_update} -eq ${FLAGS_TRUE} ]]; then zip_update_tools diff --git a/build_library/oem_sysexts.sh b/build_library/oem_sysexts.sh new file mode 100644 index 00000000000..a6bcae319ef --- /dev/null +++ b/build_library/oem_sysexts.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# OEM sysext helpers. + +get_oem_overlay_root() { + local scripts_repo="$1" + local overlay_root="/mnt/host/source/src/third_party/coreos-overlay" + + if [[ ! -d "${overlay_root}" ]]; then + overlay_root="${scripts_repo}/sdk_container/src/third_party/coreos-overlay" + fi + + if [[ ! -d "${overlay_root}" ]]; then + echo "No coreos-overlay repo found (tried SDK and ${scripts_repo})" >&2 + exit 1 + fi + + printf '%s' "${overlay_root}" +} + +_get_oem_ids() { + local scripts_repo arch list_var_name + scripts_repo=${1}; shift + arch=${1}; shift + list_var_name=${1}; shift + + local overlay_root + overlay_root=$(get_oem_overlay_root "${scripts_repo}") + + local -a ebuilds=("${overlay_root}/coreos-base/common-oem-files/common-oem-files-"*'.ebuild') + if [[ ${#ebuilds[@]} -eq 0 ]] || [[ ! -e ${ebuilds[0]} ]]; then + echo "No coreos-base/common-oem-files ebuilds?!" >&2 + exit 1 + fi + + # This defines local COMMON_OEMIDS, AMD64_ONLY_OEMIDS, + # ARM64_ONLY_OEMIDS and OEMIDS variable. We don't use the last + # one. Also defines global-by-default EAPI, which we make local + # here to avoid making it global. + local EAPI + source "${ebuilds[0]}" flatcar-local-variables + + local -n arch_oemids_ref="${arch^^}_ONLY_OEMIDS" + local all_oemids=( + "${COMMON_OEMIDS[@]}" + "${arch_oemids_ref[@]}" + ) + + mapfile -t "${list_var_name}" < <(printf '%s\n' "${all_oemids[@]}" | sort) +} + +# Gets a list of OEMs that are using sysexts. +# +# 1 - scripts repo +# 2 - arch +# 3 - name of an array variable to store the result in +get_oem_id_list() { + _get_oem_ids "$@" +} + +# Gets a list of OEM sysext descriptors. +# +# 1 - scripts repo +# 2 - arch +# 3 - name of an array variable to store the result in +# +# Format: "name|metapackage|useflags" +get_oem_sysext_matrix() { + local scripts_repo arch list_var_name + scripts_repo=${1}; shift + arch=${1}; shift + list_var_name=${1}; shift + + local -a oem_ids + _get_oem_ids "${scripts_repo}" "${arch}" oem_ids + + local -a matrix=() + local oem_id + for oem_id in "${oem_ids[@]}"; do + matrix+=("oem-${oem_id}|coreos-base/oem-${oem_id}|${oem_id}") + done + + local -n matrix_ref="${list_var_name}" + matrix_ref=("${matrix[@]}") +} diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 7a0c34b50ca..659c16f2e47 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -3,6 +3,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +source "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1 + # Lookup the current version of a binary package, downloading it if needed. # Usage: get_binary_pkg some-pkg/name # Prints: some-pkg/name-1.2.3 @@ -273,6 +275,52 @@ create_prod_sysexts() { done } +create_oem_sysexts() { + local image_name="$1" + local image_sysext_base="${image_name%.bin}_sysext.squashfs" + local overlay_path + overlay_path=$(portageq get_repo_path / coreos-overlay) + + local -a oem_sysexts + get_oem_sysext_matrix "${SCRIPT_ROOT}" "${ARCH}" oem_sysexts + + local sysext name metapkg useflags + for sysext in "${oem_sysexts[@]}"; do + IFS="|" read -r name metapkg useflags <<< "${sysext}" + + # Check for manglefs script in the package's files directory + local mangle_script="${overlay_path}/${metapkg}/files/manglefs.sh" + if [[ ! -x "${mangle_script}" ]]; then + mangle_script= + fi + + sudo rm -f "${BUILD_DIR}/${name}.raw" \ + "${BUILD_DIR}/flatcar_test_update-${name}.gz" \ + "${BUILD_DIR}/${name}_"* + + info "Building OEM sysext ${name} with USE=${useflags}" + # The --install_root_basename="${name}-oem-sysext-rootfs" flag is + # important - it sets the name of a rootfs directory, which is + # used to determine the package target in + # coreos/base/profile.bashrc + # + # OEM sysexts use no compression here since they will be stored + # in a compressed OEM partition. + USE="${useflags}" sudo -E "${SCRIPT_ROOT}/build_sysext" --board="${BOARD}" \ + --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \ + --image_builddir="${BUILD_DIR}" \ + --metapkgs="${metapkg}" \ + --install_root_basename="${name}-oem-sysext-rootfs" \ + --compression=none \ + ${mangle_script:+--manglefs_script="${mangle_script}"} \ + "${name}" + delta_generator \ + -private_key "/usr/share/update_engine/update-payload-key.key.pem" \ + -new_image "${BUILD_DIR}/${name}.raw" \ + -out_file "${BUILD_DIR}/flatcar_test_update-${name}.gz" + done +} + sbsign_prod_image() { local image_name="$1" local disk_layout="$2" diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 73eb43b8125..ba715812195 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -568,7 +568,8 @@ install_oem_package() { sudo rm -rf "${oem_tmp}" } -# Write the OEM sysext file into the OEM partition. +# Install the prebuilt OEM sysext file into the OEM partition. +# The sysext should have been built by 'build_image oem_sysext'. install_oem_sysext() { local oem_sysext=$(_get_vm_opt OEM_SYSEXT) @@ -576,63 +577,24 @@ install_oem_sysext() { return 0 fi - local built_sysext_dir="${FLAGS_to}/${oem_sysext}-sysext" - local built_sysext_filename="${oem_sysext}.raw" - local built_sysext_path="${built_sysext_dir}/${built_sysext_filename}" + local prebuilt_sysext_filename="${oem_sysext}.raw" + local prebuilt_sysext_path="${FLAGS_from}/${prebuilt_sysext_filename}" local version="${FLATCAR_VERSION}" - local metapkg="coreos-base/${oem_sysext}" - # The --install_root_basename="${name}-oem-sysext-rootfs" flag is - # important - it sets the name of a rootfs directory, which is - # used to determine the package target in - # coreos/base/profile.bashrc - # - # OEM sysexts are stored in the compressed partition, so we disable - # compression to avoid double-compression. - local build_sysext_flags=( - --board="${BOARD}" - --squashfs_base="${VM_SRC_SYSEXT_IMG}" - --image_builddir="${built_sysext_dir}" - --metapkgs="${metapkg}" - --compression=none - --install_root_basename="${VM_IMG_TYPE}-oem-sysext-rootfs" - ) - local overlay_path mangle_fs - overlay_path=$(portageq get_repo_path / coreos-overlay) - mangle_fs="${overlay_path}/${metapkg}/files/manglefs.sh" - if [[ -x "${mangle_fs}" ]]; then - build_sysext_flags+=( - --manglefs_script="${mangle_fs}" - ) - fi - mkdir -p "${built_sysext_dir}" - sudo -E "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}" + if [[ ! -f "${prebuilt_sysext_path}" ]]; then + die "Prebuilt OEM sysext not found at ${prebuilt_sysext_path}. Run 'build_image oem_sysext' first." + fi local installed_sysext_oem_dir='/oem/sysext' local installed_sysext_file_prefix="${oem_sysext}-${version}" local installed_sysext_filename="${installed_sysext_file_prefix}.raw" local installed_sysext_abspath="${installed_sysext_oem_dir}/${installed_sysext_filename}" - info "Installing ${oem_sysext} sysext" + + info "Installing ${oem_sysext} sysext from prebuilt image" sudo install -Dpm 0644 \ - "${built_sysext_path}" \ + "${prebuilt_sysext_path}" \ "${VM_TMP_ROOT}${installed_sysext_abspath}" || die "Could not install ${oem_sysext} sysext" - # Move sysext image and reports to a destination directory to - # upload them, thus making them available as separate artifacts to - # download. - local upload_dir to_move - upload_dir="$(_dst_dir)" - for to_move in "${built_sysext_dir}/${oem_sysext}"*; do - mv "${to_move}" "${upload_dir}/${to_move##*/}" - done - # Generate dev-key-signed update payload for testing - delta_generator \ - -private_key "/usr/share/update_engine/update-payload-key.key.pem" \ - -new_image "${upload_dir}/${built_sysext_filename}" \ - -out_file "${upload_dir}/flatcar_test_update-${oem_sysext}.gz" - # Remove sysext_dir if building sysext and installing it - # succeeded. - rm -rf "${built_sysext_dir}" # Mark the installed sysext as active. sudo touch "${VM_TMP_ROOT}${installed_sysext_oem_dir}/active-${oem_sysext}" diff --git a/build_packages b/build_packages index d300edff828..3dd77365e2b 100755 --- a/build_packages +++ b/build_packages @@ -118,6 +118,7 @@ fi . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1 # Setup all the emerge command/flags. EMERGE_FLAGS=( --update --deep --newuse --verbose --backtrack=30 --select ) @@ -288,50 +289,55 @@ fi export KBUILD_BUILD_USER="${BUILD_USER:-build}" export KBUILD_BUILD_HOST="${BUILD_HOST:-pony-truck.infra.kinvolk.io}" +# Build sysext packages from an array of sysext definitions. +# Usage: build_sysext_packages "description" "${SYSEXT_ARRAY[@]}" +# Array format: "name|packages|useflags|arches" +build_sysext_packages() { + local description="$1" + shift + local sysexts=("$@") + + info "Merging ${description} packages now" + for sysext in "${sysexts[@]}"; do + local sysext_name package_atoms useflags arches + IFS="|" read -r sysext_name package_atoms useflags arches <<< "$sysext" + [[ -z ${arches} || ,${arches}, == *,"${ARCH}",* ]] || continue + + info "Building packages for $sysext_name sysext with USE=$useflags" + IFS=, + for package in $package_atoms; do + # --buildpkgonly does not install dependencies, so we install them + # separately before building the binary package + sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \ + env USE="$useflags" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \ + "${EMERGE_FLAGS[@]}" \ + --quiet \ + --onlydeps \ + --binpkg-respect-use=y \ + "${package}" + + sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \ + env USE="$useflags" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \ + "${EMERGE_FLAGS[@]}" \ + --quiet \ + --buildpkgonly \ + --binpkg-respect-use=y \ + "${package}" + done + unset IFS + done +} + info "Merging board packages now" sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "$@" -info "Merging sysext packages now" -for sysext in "${EXTRA_SYSEXTS[@]}"; do - IFS="|" read -r SYSEXT_NAME PACKAGE_ATOMS USEFLAGS ARCHES <<< "$sysext" - - arch_array=("${ARCHES//,/ }") - if [[ -n $ARCHES ]]; then - should_skip=1 - for arch in "${arch_array[@]}"; do - if [[ $arch == "$ARCH" ]]; then - should_skip=0 - fi - done - if [[ $should_skip -eq 1 ]]; then - continue - fi - fi - +build_sysext_packages "extra sysexts" "${EXTRA_SYSEXTS[@]}" - info "Building packages for $SYSEXT_NAME sysext with USE=$USEFLAGS" - IFS=, - for package in $PACKAGE_ATOMS; do - # --buildpkgonly does not install dependencies, so we install them - # separately before building the binary package - sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \ - env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \ - "${EMERGE_FLAGS[@]}" \ - --quiet \ - --onlydeps \ - --binpkg-respect-use=y \ - "${package}" - - sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \ - env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \ - "${EMERGE_FLAGS[@]}" \ - --quiet \ - --buildpkgonly \ - --binpkg-respect-use=y \ - "${package}" - done - unset IFS -done +declare -a oem_sysexts +get_oem_sysext_matrix . "${ARCH}" oem_sysexts +if [[ ${#oem_sysexts[@]} -gt 0 ]]; then + build_sysext_packages "OEM sysexts" "${oem_sysexts[@]}" +fi info "Removing obsolete packages" # The return value of emerge is not clearly reliable. It may fail with diff --git a/build_sysext b/build_sysext index 0b7ea4d5f32..4a0d450e0b8 100755 --- a/build_sysext +++ b/build_sysext @@ -216,7 +216,7 @@ if [[ ${#} -lt 1 ]]; then show_help_if_requested -h fi -info "Building '${SYSEXTNAME}' squashfs with (meta-)packages '${@}' in '${BUILD_DIR}' using '${FLAGS_compression}' compression". +info "Building '${SYSEXTNAME}' sysext with (meta-)packages '${@}' in '${BUILD_DIR}' using '${FLAGS_compression}' compression". for package; do echo "Installing package into sysext image: $package" diff --git a/ci-automation/image.sh b/ci-automation/image.sh index 09ca5e904fc..552c382e042 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -103,7 +103,7 @@ function _image_build_impl() { --base_sysexts="${base_sysexts_param}" \ --output_root="${CONTAINER_IMAGE_ROOT}" \ --only_store_compressed \ - prodtar container sysext + prodtar container sysext oem_sysext # copy resulting images + push to buildcache local images_out="images/" diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh index 6de6c5cee38..f92f1e6c9c9 100644 --- a/ci-automation/image_changes.sh +++ b/ci-automation/image_changes.sh @@ -241,38 +241,7 @@ function git_tag_for_nightly() { git_tag_ref=$(git -C "${scripts_repo}" describe --tags --abbrev=0 --match='*-nightly-*' --exclude='*-INTERMEDIATE' "${search_object}") } -# Gets a list of OEMs that are using sysexts. -# -# 1 - scripts repo -# 2 - arch -# 3 - name of an array variable to store the result in -function get_oem_id_list() { - local scripts_repo arch list_var_name - scripts_repo=${1}; shift - arch=${1}; shift - list_var_name=${1}; shift - - local -a ebuilds=("${scripts_repo}/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-"*'.ebuild') - if [[ ${#ebuilds[@]} -eq 0 ]] || [[ ! -e ${ebuilds[0]} ]]; then - echo "No coreos-base/common-oem-files ebuilds?!" >&2 - exit 1 - fi - - # This defines local COMMON_OEMIDS, AMD64_ONLY_OEMIDS, - # ARM64_ONLY_OEMIDS and OEMIDS variable. We don't use the last - # one. Also defines global-by-default EAPI, which we make local - # here to avoid making it global. - local EAPI - source "${ebuilds[0]}" flatcar-local-variables - - local -n arch_oemids_ref="${arch^^}_ONLY_OEMIDS" - local all_oemids=( - "${COMMON_OEMIDS[@]}" - "${arch_oemids_ref[@]}" - ) - - mapfile -t "${list_var_name}" < <(printf '%s\n' "${all_oemids[@]}" | sort) -} +source build_library/oem_sysexts.sh function get_base_sysext_list() { local scripts_repo=${1}; shift diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index b882d19a04e..dc716438d1f 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -116,6 +116,17 @@ function _vm_build_impl() { for file in flatcar_production_image.bin.bz2 flatcar_production_image_sysext.squashfs flatcar_production_image.vmlinuz version.txt; do copy_from_buildcache "images/${arch}/${vernum}/${file}" "${images_in}" done + + # Download prebuilt OEM sysexts + source build_library/oem_sysexts.sh + local -a oem_ids + get_oem_id_list . "${arch}" oem_ids + local oem_id name + for oem_id in "${oem_ids[@]}"; do + name="oem-${oem_id}" + copy_from_buildcache "images/${arch}/${vernum}/${name}.raw" "${images_in}" + done + lbunzip2 "${images_in}/flatcar_production_image.bin.bz2" ./run_sdk_container -x ./ci-cleanup.sh -n "${vms_container}" -C "${packages_image}" \ -v "${vernum}" \ From 5d499d564f591f8554e62fe4b976f6df5fdb75ea Mon Sep 17 00:00:00 2001 From: Daniel Zatovic Date: Wed, 7 Jan 2026 14:18:59 +0100 Subject: [PATCH 6/6] changelog: Mention OEM sysext signing changes Update the changelog entry to include information about OEM sysexts being signed and built during the image phase. Signed-off-by: Daniel Zatovic --- changelog/changes/2025-11-05-signed-os-dependent-sysexts.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/changes/2025-11-05-signed-os-dependent-sysexts.md b/changelog/changes/2025-11-05-signed-os-dependent-sysexts.md index 196b9266b12..3de3637f9cd 100644 --- a/changelog/changes/2025-11-05-signed-os-dependent-sysexts.md +++ b/changelog/changes/2025-11-05-signed-os-dependent-sysexts.md @@ -1 +1 @@ -- OS-dependent sysexts (e.g., docker-flatcar, containerd-flatcar) are now cryptographically signed using dm-verity roothash signatures. This enables stricter sysext policies via systemd-sysext and provides a foundation for verifying user-provided extensions in future releases. The format changed from squashfs to erofs-based Discoverable Disk Images (DDI). ([scripts#3162](https://github.com/flatcar/scripts/pull/3162)) +- OS-dependent sysexts (e.g., docker-flatcar, containerd-flatcar, podman, zfs, nvidia) are now cryptographically signed using dm-verity roothash signatures. This enables stricter sysext policies via systemd-sysext and provides a foundation for verifying user-provided extensions in future releases. The format changed from squashfs to erofs-based Discoverable Disk Images (DDI). OEM sysexts (e.g., oem-azure, oem-gce) are now also signed and built during the image phase to ensure consistent signing with the same ephemeral key. ([scripts#3162](https://github.com/flatcar/scripts/pull/3162))