Fix build.yml tag inconsistency

.github/workflows/build.yml

@@ -12,96 +12,269 @@ on:
   workflow_dispatch:
 
 jobs:
+  prebuild:
+    runs-on: rehosting-arc
+    outputs:
+      targets: ${{ steps.find_targets.outputs.targets }}
+      versions: ${{ steps.find_targets.outputs.versions }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.ref }}
+
+      - name: Log git revisions of all linux projects
+        run: |
+          echo "Main repo revision:" && git rev-parse HEAD
+          echo
+          echo "Submodule revisions:" && git submodule status
+          echo
+          echo "Full submodule SHAs:" && git submodule foreach 'echo $name: $(git rev-parse HEAD)'
+
+      - name: Ensure local bare clone of base Linux repo
+        run: |
+          set -eux
+          BASE_REPO_DIR="/home/runner/_shared/linux"
+          BASE_REPO_URL="https://github.com/rehosting/linux"
+
+
+          # Clone bare base repo if missing
+          if [ ! -d "$BASE_REPO_DIR" ]; then
+            echo "Cloning bare base repo to $BASE_REPO_DIR"
+            git clone --bare "$BASE_REPO_URL" "$BASE_REPO_DIR"
+            cd $BASE_REPO_DIR && git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
+          fi
+          # Always fetch latest from upstream
+          git fetch origin --prune --tags --force
+
+      - name: Ensure linux cache exists
+        run: |
+          BASE_CACHE_DIR="/home/runner/_shared/linux_builder/cache"
+
+          if [ ! -d "$BASE_CACHE_DIR" ]; then
+            mkdir -p "$BASE_CACHE_DIR"
+          fi
+
+      - name: Setup shared Linux kernel sources
+        run: |
+          RUNS_PARENT="/home/runner/_shared/runs"
+          RUNS_DIR="$RUNS_PARENT/$GITHUB_RUN_ID"
+
+          mkdir -p "$RUNS_DIR"
+          mkdir -p "$RUNS_DIR/linux"
+
+          BASE_REPO_DIR="/home/runner/_shared/linux"
+          BASE_REPO_URL="https://github.com/rehosting/linux"
+
+          # Update .gitmodules to use local bare repo
+          sed -i "s|url = https://github.com/rehosting/linux.git|url = file://$BASE_REPO_DIR|g" .gitmodules
+
+          # Initialize and update submodules for all versions
+          GIT_ALLOW_PROTOCOL=file:https git submodule update --init --depth 1 --jobs 2
+
+          # move into the shared run directory
+          mv linux/* "$RUNS_DIR/linux/"
+
+      - name: Find valid targets and versions sets
+        id: find_targets
+        run: |
+          TARGETS_SET=()
+          VERSIONS_SET=()
+          for version_dir in configs/*/; do
+            version=$(basename "$version_dir")
+            VERSIONS_SET+=("$version")
+            for config_file in "$version_dir"*; do
+              if [[ -f "$config_file" && ! "$config_file" =~ \.inc$  && ! "$config_file" =~ \.unused$ ]]; then
+                target=$(basename "$config_file")
+                TARGETS_SET+=("$target")
+              fi
+            done
+          done
+          UNIQUE_TARGETS=$(printf "%s\n" "${TARGETS_SET[@]}" | sort -u | awk '{printf "\"%s\",",$0}' | sed 's/,$//')
+          UNIQUE_VERSIONS=$(printf "%s\n" "${VERSIONS_SET[@]}" | sort -u | awk '{printf "\"%s\",",$0}' | sed 's/,$//')
+          TARGETS_OUTPUT="[${UNIQUE_TARGETS}]"
+            VERSIONS_OUTPUT="[${UNIQUE_VERSIONS}]"
+          echo "targets=$TARGETS_OUTPUT" >> $GITHUB_OUTPUT
+          echo "versions=$VERSIONS_OUTPUT" >> $GITHUB_OUTPUT
+          echo "Found valid targets: $TARGETS_OUTPUT"
+          echo "Found valid versions: $VERSIONS_OUTPUT"
+
   build:
-    # Only publish on tags. run git tag vX and git push origin vX
-    # runs-on: self-hosted
-    runs-on: ubuntu-latest
+    needs: prebuild
+    runs-on: rehosting-arc
     if: github.event.pull_request.draft == false
 
-
     strategy:
       matrix:
-        target: [armel, arm64, mipsel, mipseb, mips64el, mips64eb, x86_64]
-        version: ["4.10"] # XXX: quotes are necessary, otherwise 4.10 -> 4.1
+        target_version: ${{ fromJSON(needs.prebuild.outputs.targets) }}
 
     steps:
-      - uses: actions/checkout@v4 # Clones to $GITHUB_WORKSPACE
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          #submodules: 'true'
+          ref: ${{ github.ref }}
 
-      # Instead of getting submodules with checkout, we can do it manually to control depth.
-      # We don't want a full Linux history
-      - name: Pull kernel source
-        run: git submodule update --init --depth 1
+      - name: Extract target and version
+        id: extract
+        run: |
+          TARGET="${{ matrix.target_version }}"
+          echo "target=$TARGET" >> $GITHUB_OUTPUT
+          echo "Building target: $TARGET"
+
+      - name: Trust Harbor's self-signed certificate
+        run: |
+          echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
+          openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
+          sudo update-ca-certificates
 
-      - name: Cache kernel build objects
-        uses: actions/cache@v4
+      - name: Log in to Rehosting Arc Registry
+        uses: docker/login-action@v3
         with:
-          path: |
-            cache
-          key: ${{ runner.os }}-kernel-build-${{ matrix.kernel_version }}-${{ matrix.target }}-${{ hashFiles('**/Dockerfile', '**/*.sh') }}
-          restore-keys: |
-            ${{ runner.os }}-kernel-build-${{ matrix.kernel_version }}-${{ matrix.target }}-
-            ${{ runner.os }}-kernel-build-${{ matrix.kernel_version }}-
-            ${{ runner.os }}-kernel-build-
-
-      - name: Lint configs
-        run: ./build.sh --targets ${{ matrix.target }} --versions ${{ matrix.version }} --config-only
-
-      - name: Build Kernel for ${{ matrix.target }}
-        run: ./build.sh --targets ${{ matrix.target }} --versions ${{ matrix.version }}
-
-      # Temporarily store each target's build output
-      - name: Save ${{ matrix.target }} ${{ matrix.version }} build output
-        uses: actions/upload-artifact@v4
+          registry: ${{secrets.REHOSTING_ARC_REGISTRY}}
+          username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
+          password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
         with:
-          name: build-output-${{ matrix.target }}.${{ matrix.version }}
-          path: kernels-latest.tar.gz
+          driver-opts: |
+            image=moby/buildkit:master
+            network=host
+          buildkitd-config-inline: |
+            [registry."${{ secrets.REHOSTING_ARC_REGISTRY }}"]
+              insecure = true
+              http = true
+
+      - name: Build kernel_builder docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: false
+          tags: |
+            rehosting/linux_builder:latest
+          build-args: |
+            REGISTRY=${{ secrets.REHOSTING_ARC_REGISTRY }}/proxy
+            TARGET=${{ matrix.target_version }}
+          cache-from: |
+            type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:${{ matrix.target_version }}_cache,mode=max
+          cache-to: |
+            type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/rehosting/linux_builder:${{ matrix.target_version }}_cache,mode=max
+          outputs: type=docker
+
+      - name: Build Kernel for ${{ matrix.target_version }}
+        run: |
+            set -eux
+            TARGET="${{ matrix.target_version }}"
+            VERSIONS_JSON='${{ needs.prebuild.outputs.versions }}'
+            BASE_REPO_DIR="/home/runner/_shared/linux"
+            BASE_CACHE_DIR="/home/runner/_shared/linux_builder/cache"
+            BASE_REPO_URL="https://github.com/rehosting/linux"
+            RUNS_PARENT="/home/runner/_shared/runs"
+            RUNS_DIR="$RUNS_PARENT/$GITHUB_RUN_ID"
+            RUNS_DIR_LINUX="$RUNS_DIR/linux"
+
+            # Convert JSON array to space-separated string
+            if [ -z "$VERSIONS_JSON" ] || [ "$VERSIONS_JSON" = "[]" ]; then
+              VERSIONS=""
+            else
+              VERSIONS=$(echo "$VERSIONS_JSON" | jq -r '.[]' | xargs)
+            fi
+
+            ./build.sh --targets "$TARGET" ${VERSIONS:+--versions "$VERSIONS"} --cache-dir "$BASE_CACHE_DIR" --extra-docker-opts "-v $RUNS_DIR_LINUX:/app/linux"
 
+            BUILD_OUTPUT="$RUNS_DIR/build-output"
+            mkdir -p $BUILD_OUTPUT
+            mv kernels-latest.tar.gz $BUILD_OUTPUT/kernels-latest-${TARGET}.tar.gz
+            mv kernel-devel-all.tar.gz $BUILD_OUTPUT/kernel-devel-all-${TARGET}.tar.gz
+
   aggregate:
     if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
     needs: build
-    runs-on: ubuntu-latest
+    runs-on: rehosting-arc
+    env:
+      MATRIX_VERSIONS: ${{ toJSON(needs.build.strategy.matrix.version) }}
     permissions:
       actions: write
       contents: write
     steps:
-      - name: Download all build artifacts
-        uses: actions/download-artifact@v4
+      - name: Trust Harbor's self-signed certificate
+        run: |
+          echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
+          openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
+          sudo update-ca-certificates
+      - name: Log in to Rehosting Arc Registry
+        uses: docker/login-action@v3
         with:
-          path: downloaded-kernels
+          registry: ${{secrets.REHOSTING_ARC_REGISTRY}}
+          username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
+          password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
 
       - name: Combine all kernels into a single archive
         run: |
+          set -eux
+          RUNS_PARENT="/home/runner/_shared/runs"
+          RUNS_DIR="$RUNS_PARENT/$GITHUB_RUN_ID"
+          BUILD_OUTPUT="$RUNS_DIR/build-output"
+
+          echo "[DEBUG] Listing available per-target kernel archives:"
+          find "$BUILD_OUTPUT" -maxdepth 1 -name "kernels-latest-*.tar.gz" -print || true
+
           rm -rf combined-kernels && mkdir combined-kernels
-          # Extract each kernels-latest.tar.gz archive
-          for archive in $(find downloaded-kernels -name "*.tar.gz"); do
+
+          for archive in "$BUILD_OUTPUT"/kernels-latest-*.tar.gz; do
+            [ -e "$archive" ] || continue
+            echo "[DEBUG] Extracting $archive into combined-kernels"
             tar -xzf "$archive" -C combined-kernels
           done
 
-          # Combine OSI profiles for each kernel version
-          # otherwise the extracted kernel configs willc lobber each other
-          if [ -d combined-kernels/kernels/4.10 ] ; then
-            for archive in $(find downloaded-kernels -name "*.tar.gz"); do
-              tar -O -xf "$archive" "kernels/4.10/osi.config";
-            done > combined-kernels/kernels/4.10/osi.config
-          fi
+          echo "[DEBUG] Contents of combined-kernels after extraction:"
+          find combined-kernels || true
 
-          if [ -d combined-kernels/kernels/6.7 ] ; then
-            for archive in $(find downloaded-kernels -name "*.tar.gz"); do
-              tar -O -xf "$archive" "kernels/6.7/osi.config";
-            done > combined-kernels/kernels/6.7/osi.config
+          # Merge osi.config for every detected version directory
+          if [ -d combined-kernels/kernels ]; then
+            for vdir in combined-kernels/kernels/*; do
+              [ -d "$vdir" ] || continue
+              version=$(basename "$vdir")
+              echo "[DEBUG] Merging osi.config for version $version"
+              {
+                for archive in "$BUILD_OUTPUT"/kernels-latest-*.tar.gz; do
+                  [ -e "$archive" ] || continue
+                  tar -O -xf "$archive" "kernels/$version/osi.config" 2>/dev/null || true
+                done
+              } > "combined-kernels/kernels/$version/osi.config"
+            done
           fi
 
+          tar -czvf kernels-latest.tar.gz -C combined-kernels .
 
+      - name: Aggregate all kernel-devel artifacts
+        run: |
+          set -eux
+          RUNS_PARENT="/home/runner/_shared/runs"
+          RUNS_DIR="$RUNS_PARENT/$GITHUB_RUN_ID"
+          BUILD_OUTPUT="$RUNS_DIR/build-output"
 
-          # Create a new single archive from the combined content
-          tar -czvf kernels-latest.tar.gz -C combined-kernels .
+          mkdir -p kernel-devel-all
+          for archive in "$BUILD_OUTPUT"/kernel-devel-all-*.tar.gz; do
+            [ -e "$archive" ] || continue
+            echo "[DEBUG] Extracting $archive into kernel-devel-all/"
+            tar -xzf "$archive" -C kernel-devel-all
+          done
+          tar -czvf kernel-devel-all.tar.gz -C kernel-devel-all .
 
       - name: Create and publish release
         uses: softprops/action-gh-release@v1
         with:
-          files: kernels-latest.tar.gz
+          files: |
+            kernels-latest.tar.gz
+            kernel-devel-all.tar.gz
           token: ${{ secrets.GITHUB_TOKEN }}
-          tag_name: ${{ github.ref }}
+          tag_name: ${{ github.ref_name }}
+
+      - name: Cleanup per-run kernel clones
+        if: always()
+        run: |
+          RUNS_PARENT="/home/runner/_shared/runs"
+          RUNS_DIR="$RUNS_PARENT/$GITHUB_RUN_ID"
+          echo "Cleaning up kernel clones in $RUNS_DIR"
+          rm -rf "$RUNS_DIR"

.github/workflows/clear_cache.yml

@@ -0,0 +1,30 @@
+name: Clear kernel cache
+
+on:
+  workflow_dispatch:
+
+jobs:
+  clear-cache:
+    runs-on: rehosting-arc
+    steps:
+      - name: Trust Harbor's self-signed certificate
+        run: |
+          echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
+          openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
+          sudo update-ca-certificates
+      - uses: oras-project/setup-oras@v1.2.3
+      - name: Log in to Rehosting Arc Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.REHOSTING_ARC_REGISTRY }}
+          username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
+          password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
+      - name: Delete all kernel cache images
+        run: |
+          set -e
+          repo="${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/linux_builder_cache"
+          tags=$(oras repo tags $repo)
+          for tag in $tags; do
+            echo "Deleting $repo:$tag"
+            oras repo rm $repo:$tag || echo "Failed to delete $repo:$tag"
+          done

.gitmodules

@@ -3,6 +3,6 @@
 	url = https://github.com/rehosting/linux.git
 	branch = main_4.10
 [submodule "linux_6.7"]
-	path = linux/6.7
+	path = linux/6.13
 	url = https://github.com/rehosting/linux.git
 	branch = main_6.7

Dockerfile

@@ -1,11 +1,12 @@
-FROM golang:latest as go
-RUN git clone --depth 1 https://github.com/volatilityfoundation/dwarf2json.git \
-    && cd dwarf2json \
-    && go build
-
-FROM ghcr.io/panda-re/embedded-toolchains:latest
-COPY --from=go /go/dwarf2json/dwarf2json /bin/dwarf2json
-RUN apt-get update && apt-get -y install gdb xonsh flex bison libssl-dev
+ARG REGISTRY="docker.io"
+ARG TARGET="latest"
+FROM ${REGISTRY}/rehosting/embedded-toolchains:${TARGET}
 
 # Get panda for kernelinfo_gdb. Definitely a bit overkill to pull the whole repo
-RUN git clone --depth 1 https://github.com/panda-re/panda.git
+RUN mkdir /extract_kernelinfo && \
+    wget https://raw.githubusercontent.com/panda-re/panda-ng/refs/heads/main/plugins/osi_linux/utils/kernelinfo_gdb/extract_kernelinfo.py -O /extract_kernelinfo/extract_kernelinfo.py && \
+    wget https://raw.githubusercontent.com/panda-re/panda-ng/refs/heads/main/plugins/osi_linux/utils/kernelinfo_gdb/run.sh -O /extract_kernelinfo/run.sh && \
+    chmod +x /extract_kernelinfo/run.sh
+
+RUN wget https://github.com/volatilityfoundation/dwarf2json/releases/download/v0.9.0/dwarf2json-linux-amd64 -O /bin/dwarf2json && \
+	chmod +x /bin/dwarf2json