From 75374b468f90152a3d72ce01716c0afa18935132 Mon Sep 17 00:00:00 2001
From: Ben Vinegar <ben@benv.ca>
Date: Mon, 23 Feb 2026 22:27:37 -0500
Subject: [PATCH 1/9] ci: add inference smoke test on every PR

Add an end-to-end inference smoke test that verifies the control-agent
can complete a real LLM turn via session-control RPC.

- bin/ci/smoke-agent-inference.sh: sends a prompt via Unix socket RPC,
  subscribes to turn_end, validates the response contains expected token.
- Uses anthropic/claude-haiku (cheap) via new BAUDBOT_MODEL env override.
- CI_ANTHROPIC_API_KEY injected into agent .env for provider auth.
- start.sh: respect BAUDBOT_MODEL override before auto-detect.
- .env.schema: add BAUDBOT_MODEL.
- integration.yml: pass CI_ANTHROPIC_API_KEY to every droplet run.
- bin/ci/droplet.sh run: accept optional KEY=VALUE env var forwarding.
- Wired into setup-ubuntu.sh and setup-arch.sh after runtime smoke.
---
 .env.schema                       |   4 +
 .github/workflows/integration.yml |   3 +-
 bin/ci/droplet.sh                 |  16 ++-
 bin/ci/setup-arch.sh              |   3 +
 bin/ci/setup-ubuntu.sh            |   3 +
 bin/ci/smoke-agent-inference.sh   | 210 ++++++++++++++++++++++++++++++
 start.sh                          |   6 +-
 7 files changed, 238 insertions(+), 7 deletions(-)
 create mode 100755 bin/ci/smoke-agent-inference.sh

diff --git a/.env.schema b/.env.schema
index 363d272..9a0680b 100644
--- a/.env.schema
+++ b/.env.schema
@@ -31,6 +31,10 @@ GEMINI_API_KEY=
 # @docs(https://opencode.ai)
 OPENCODE_ZEN_API_KEY=
 
+# Override auto-detected model (e.g. anthropic/claude-haiku for CI)
+# @sensitive=false @type=string
+BAUDBOT_MODEL=
+
 # ── Slack ────────────────────────────────────────────────────────────────────
 
 # Slack bot OAuth token (required for direct Socket Mode, optional in broker mode)
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 7030aa3..ad0b37d 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -115,7 +115,8 @@ jobs:
           bash bin/ci/droplet.sh run \
             "${{ steps.droplet.outputs.DROPLET_IP }}" \
             ~/.ssh/ci_key \
-            "${{ matrix.setup_script }}"
+            "${{ matrix.setup_script }}" \
+            "CI_ANTHROPIC_API_KEY=${{ secrets.CI_ANTHROPIC_API_KEY }}"
 
       - name: Cleanup
         if: always()
diff --git a/bin/ci/droplet.sh b/bin/ci/droplet.sh
index d4aa8b2..b12d913 100755
--- a/bin/ci/droplet.sh
+++ b/bin/ci/droplet.sh
@@ -184,12 +184,20 @@ cmd_wait_ssh() {
 
 # ── run <ip> <ssh_private_key_file> <script> ──────────────────────────────────
 cmd_run() {
-  local ip="${1:?Usage: droplet.sh run <ip> <ssh_private_key_file> <script>}"
+  local ip="${1:?Usage: droplet.sh run <ip> <ssh_private_key_file> <script> [env_vars...]}"
   local key_file="${2:?}"
   local script="${3:?}"
-
-  ssh -o StrictHostKeyChecking=no -o BatchMode=yes \
-    -i "$key_file" "root@$ip" bash -s < "$script"
+  shift 3
+
+  # Remaining args are KEY=VALUE env vars forwarded to the remote script.
+  # Prepend export statements so the remote bash -s session inherits them.
+  {
+    for var in "$@"; do
+      printf 'export %s\n' "$var"
+    done
+    cat "$script"
+  } | ssh -o StrictHostKeyChecking=no -o BatchMode=yes \
+    -i "$key_file" "root@$ip" bash -s
 }
 
 # ── list ──────────────────────────────────────────────────────────────────────
diff --git a/bin/ci/setup-arch.sh b/bin/ci/setup-arch.sh
index 865a32e..f96eaef 100755
--- a/bin/ci/setup-arch.sh
+++ b/bin/ci/setup-arch.sh
@@ -70,6 +70,9 @@ bash /home/baudbot_admin/baudbot/bin/ci/smoke-cli.sh
 echo "=== Running runtime smoke checks ==="
 bash /home/baudbot_admin/baudbot/bin/ci/smoke-agent-runtime.sh
 
+echo "=== Running inference smoke check ==="
+bash /home/baudbot_admin/baudbot/bin/ci/smoke-agent-inference.sh
+
 echo "=== Installing test dependencies ==="
 export PATH="/home/baudbot_agent/opt/node/bin:$PATH"
 cd /home/baudbot_admin/baudbot
diff --git a/bin/ci/setup-ubuntu.sh b/bin/ci/setup-ubuntu.sh
index dbccaa9..10ed0b6 100755
--- a/bin/ci/setup-ubuntu.sh
+++ b/bin/ci/setup-ubuntu.sh
@@ -108,6 +108,9 @@ bash /home/baudbot_admin/baudbot/bin/ci/smoke-cli.sh
 echo "=== Running runtime smoke checks ==="
 bash /home/baudbot_admin/baudbot/bin/ci/smoke-agent-runtime.sh
 
+echo "=== Running inference smoke check ==="
+bash /home/baudbot_admin/baudbot/bin/ci/smoke-agent-inference.sh
+
 echo "=== Installing test dependencies ==="
 export PATH="/home/baudbot_agent/opt/node/bin:$PATH"
 cd /home/baudbot_admin/baudbot
diff --git a/bin/ci/smoke-agent-inference.sh b/bin/ci/smoke-agent-inference.sh
new file mode 100755
index 0000000..b8b5a04
--- /dev/null
+++ b/bin/ci/smoke-agent-inference.sh
@@ -0,0 +1,210 @@
+#!/usr/bin/env bash
+# Inference smoke-test for baudbot.
+#
+# Verifies that the control-agent can complete at least one real LLM turn
+# end-to-end via session-control RPC.
+#
+# Requires CI_ANTHROPIC_API_KEY in the environment (injected into the
+# agent's .env before starting baudbot).
+#
+# Expects baudbot to be already installed and stoppable via `sudo baudbot`.
+
+set -Eeuo pipefail
+
+readonly AGENT_USER="baudbot_agent"
+readonly AGENT_HOME="/home/${AGENT_USER}"
+readonly AGENT_ENV="${AGENT_HOME}/.config/.env"
+readonly CONTROL_DIR="${AGENT_HOME}/.pi/session-control"
+readonly CONTROL_ALIAS="${CONTROL_DIR}/control-agent.alias"
+readonly START_TIMEOUT_SECONDS=60
+readonly INFERENCE_TIMEOUT_SECONDS=120
+readonly EXPECTED_TOKEN="CI_INFERENCE_OK"
+
+started=0
+
+log() {
+  printf '[inference-smoke] %s\n' "$*"
+}
+
+cleanup() {
+  local exit_code=$?
+  if [[ $started -eq 1 ]]; then
+    log "cleanup: stopping baudbot"
+    sudo baudbot stop >/dev/null 2>&1 || true
+  fi
+  exit "$exit_code"
+}
+trap cleanup EXIT
+
+wait_for_control_socket() {
+  local deadline=$((SECONDS + START_TIMEOUT_SECONDS))
+  local target=""
+
+  while (( SECONDS < deadline )); do
+    if [[ -L "$CONTROL_ALIAS" ]]; then
+      target="$(readlink -- "$CONTROL_ALIAS" 2>/dev/null || true)"
+      if [[ -n "$target" ]]; then
+        if [[ "$target" != /* ]]; then
+          target="${CONTROL_DIR}/${target}"
+        fi
+        if [[ -S "$target" ]]; then
+          printf '%s\n' "$target"
+          return 0
+        fi
+      fi
+    fi
+    sleep 1
+  done
+
+  return 1
+}
+
+dump_diagnostics() {
+  log "--- diagnostics ---"
+  sudo baudbot status 2>&1 || true
+  log "--- end diagnostics ---"
+}
+
+# Send a message via session-control RPC and wait for turn_end.
+# Prints the assistant response content on success, exits non-zero on failure.
+rpc_send_wait_turn_end() {
+  local socket_path="$1"
+  local message="$2"
+  local timeout_seconds="$3"
+
+  sudo -u "$AGENT_USER" python3 - "$socket_path" "$message" "$timeout_seconds" <<'PY'
+import json
+import socket
+import sys
+
+sock_path = sys.argv[1]
+message = sys.argv[2]
+timeout_seconds = int(sys.argv[3])
+
+send_cmd = {"type": "send", "message": message, "mode": "steer"}
+subscribe_cmd = {"type": "subscribe", "event": "turn_end"}
+
+client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+try:
+    client.settimeout(timeout_seconds)
+    client.connect(sock_path)
+
+    # Send both commands
+    client.sendall((json.dumps(send_cmd) + "\n").encode("utf-8"))
+    client.sendall((json.dumps(subscribe_cmd) + "\n").encode("utf-8"))
+
+    buf = b""
+    send_response = None
+
+    while True:
+        chunk = client.recv(8192)
+        if not chunk:
+            print("connection closed before turn_end", file=sys.stderr)
+            sys.exit(1)
+        buf += chunk
+
+        while b"\n" in buf:
+            line, buf = buf.split(b"\n", 1)
+            line = line.strip()
+            if not line:
+                continue
+
+            try:
+                msg = json.loads(line.decode("utf-8", errors="replace"))
+            except json.JSONDecodeError:
+                continue
+
+            if msg.get("type") == "response":
+                cmd = msg.get("command", "")
+                if cmd == "send":
+                    if not msg.get("success", False):
+                        print(f"send failed: {msg.get('error', 'unknown')}", file=sys.stderr)
+                        sys.exit(1)
+                    send_response = msg
+                # Ignore subscribe response
+                continue
+
+            if msg.get("type") == "event" and msg.get("event") == "turn_end":
+                if send_response is None:
+                    print("received turn_end before send response", file=sys.stderr)
+                    sys.exit(1)
+                data = msg.get("data", {})
+                assistant_msg = data.get("message", {})
+                content = assistant_msg.get("content", "")
+                if not content:
+                    print("turn completed but no assistant content", file=sys.stderr)
+                    sys.exit(1)
+                print(content)
+                sys.exit(0)
+
+    print("stream ended without turn_end event", file=sys.stderr)
+    sys.exit(1)
+except socket.timeout:
+    print("timeout waiting for inference response", file=sys.stderr)
+    sys.exit(1)
+finally:
+    client.close()
+PY
+}
+
+readonly CI_MODEL="anthropic/claude-haiku"
+
+inject_ci_config() {
+  if [[ -z "${CI_ANTHROPIC_API_KEY:-}" ]]; then
+    log "ERROR: CI_ANTHROPIC_API_KEY is not set"
+    return 1
+  fi
+  log "injecting CI API key and model override into agent .env"
+  sed -i "s|^ANTHROPIC_API_KEY=.*|ANTHROPIC_API_KEY=${CI_ANTHROPIC_API_KEY}|" "$AGENT_ENV"
+  # Use a cheap model for the smoke test — no need to burn Sonnet/Opus tokens.
+  if grep -q "^BAUDBOT_MODEL=" "$AGENT_ENV" 2>/dev/null; then
+    sed -i "s|^BAUDBOT_MODEL=.*|BAUDBOT_MODEL=${CI_MODEL}|" "$AGENT_ENV"
+  else
+    echo "BAUDBOT_MODEL=${CI_MODEL}" >> "$AGENT_ENV"
+  fi
+}
+
+main() {
+  inject_ci_config
+
+  log "starting baudbot"
+  sudo baudbot start
+  started=1
+
+  log "waiting for control-agent socket"
+  local socket_path=""
+  if ! socket_path="$(wait_for_control_socket)"; then
+    log "control-agent socket did not become ready within ${START_TIMEOUT_SECONDS}s"
+    dump_diagnostics
+    return 1
+  fi
+  log "control socket ready: ${socket_path}"
+
+  log "sending inference prompt (timeout ${INFERENCE_TIMEOUT_SECONDS}s)"
+  local response=""
+  if ! response="$(rpc_send_wait_turn_end "$socket_path" \
+    "Reply with exactly: ${EXPECTED_TOKEN}" \
+    "$INFERENCE_TIMEOUT_SECONDS")"; then
+    log "inference failed"
+    dump_diagnostics
+    return 1
+  fi
+
+  # Validate response contains expected token
+  if [[ "$response" == *"$EXPECTED_TOKEN"* ]]; then
+    log "inference response contains expected token"
+  else
+    log "unexpected response (missing '${EXPECTED_TOKEN}'):"
+    log "  ${response:0:500}"
+    dump_diagnostics
+    return 1
+  fi
+
+  log "stopping baudbot"
+  sudo baudbot stop
+  started=0
+
+  log "inference smoke passed"
+}
+
+main "$@"
diff --git a/start.sh b/start.sh
index 5c062ea..189a971 100755
--- a/start.sh
+++ b/start.sh
@@ -136,8 +136,10 @@ fi
 # Set session name (read by auto-name.ts extension)
 export PI_SESSION_NAME="control-agent"
 
-# Pick model based on available API keys (first match wins)
-if [ -n "${ANTHROPIC_API_KEY:-}" ]; then
+# Pick model: explicit override or auto-detect from API keys (first match wins)
+if [ -n "${BAUDBOT_MODEL:-}" ]; then
+  MODEL="$BAUDBOT_MODEL"
+elif [ -n "${ANTHROPIC_API_KEY:-}" ]; then
   MODEL="anthropic/claude-opus-4-6"
 elif [ -n "${OPENAI_API_KEY:-}" ]; then
   MODEL="openai/gpt-5.2-codex"

From 9d971a8a5eceeed3cf211ac3c6772c6683e2e5fc Mon Sep 17 00:00:00 2001
From: Ben Vinegar <ben@benv.ca>
Date: Mon, 23 Feb 2026 22:46:19 -0500
Subject: [PATCH 2/9] ci: probe socket connectivity before sending inference
 RPC

The runtime smoke test leaves stale socket files after stopping
baudbot. When the inference smoke starts baudbot fresh, the alias
symlink still points to the old dead socket. Fix by verifying the
socket is connectable (not just that the file exists) before
proceeding with the RPC.
---
 bin/ci/smoke-agent-inference.sh | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/bin/ci/smoke-agent-inference.sh b/bin/ci/smoke-agent-inference.sh
index b8b5a04..ec5200b 100755
--- a/bin/ci/smoke-agent-inference.sh
+++ b/bin/ci/smoke-agent-inference.sh
@@ -36,6 +36,20 @@ cleanup() {
 }
 trap cleanup EXIT
 
+socket_is_connectable() {
+  local sock="$1"
+  sudo -u "$AGENT_USER" python3 -c "
+import socket, sys
+s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+try:
+    s.settimeout(2)
+    s.connect(sys.argv[1])
+    s.close()
+except Exception:
+    sys.exit(1)
+" "$sock" 2>/dev/null
+}
+
 wait_for_control_socket() {
   local deadline=$((SECONDS + START_TIMEOUT_SECONDS))
   local target=""
@@ -47,7 +61,7 @@ wait_for_control_socket() {
         if [[ "$target" != /* ]]; then
           target="${CONTROL_DIR}/${target}"
         fi
-        if [[ -S "$target" ]]; then
+        if [[ -S "$target" ]] && socket_is_connectable "$target"; then
           printf '%s\n' "$target"
           return 0
         fi

From df5273c93dafbf7f111d0d2c7daad9d7946c1516 Mon Sep 17 00:00:00 2001
From: Ben Vinegar <ben@benv.ca>
Date: Mon, 23 Feb 2026 22:52:12 -0500
Subject: [PATCH 3/9] ci: preserve .env ownership by running sed as agent user

sed -i as root changes file ownership to root:root, making the
.env unreadable by baudbot_agent. Run all .env modifications as
the agent user to preserve 600 permissions.
---
 bin/ci/smoke-agent-inference.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/bin/ci/smoke-agent-inference.sh b/bin/ci/smoke-agent-inference.sh
index ec5200b..46ea7c7 100755
--- a/bin/ci/smoke-agent-inference.sh
+++ b/bin/ci/smoke-agent-inference.sh
@@ -169,12 +169,13 @@ inject_ci_config() {
     return 1
   fi
   log "injecting CI API key and model override into agent .env"
-  sed -i "s|^ANTHROPIC_API_KEY=.*|ANTHROPIC_API_KEY=${CI_ANTHROPIC_API_KEY}|" "$AGENT_ENV"
+  # Run as the agent user to preserve file ownership and permissions (600).
+  sudo -u "$AGENT_USER" sed -i "s|^ANTHROPIC_API_KEY=.*|ANTHROPIC_API_KEY=${CI_ANTHROPIC_API_KEY}|" "$AGENT_ENV"
   # Use a cheap model for the smoke test — no need to burn Sonnet/Opus tokens.
   if grep -q "^BAUDBOT_MODEL=" "$AGENT_ENV" 2>/dev/null; then
-    sed -i "s|^BAUDBOT_MODEL=.*|BAUDBOT_MODEL=${CI_MODEL}|" "$AGENT_ENV"
+    sudo -u "$AGENT_USER" sed -i "s|^BAUDBOT_MODEL=.*|BAUDBOT_MODEL=${CI_MODEL}|" "$AGENT_ENV"
   else
-    echo "BAUDBOT_MODEL=${CI_MODEL}" >> "$AGENT_ENV"
+    sudo -u "$AGENT_USER" bash -c "echo 'BAUDBOT_MODEL=${CI_MODEL}' >> '${AGENT_ENV}'"
   fi
 }
 

From 28c173798c0ed9a082b2d2b14c00ae0dca95c502 Mon Sep 17 00:00:00 2001
From: Ben Vinegar <ben@benv.ca>
Date: Mon, 23 Feb 2026 22:56:22 -0500
Subject: [PATCH 4/9] ci: assert non-empty inference response instead of exact
 token

The control-agent has a full skill loaded, so it won't parrot back
an exact token. Just verify we got a non-trivial response (>= 10
chars), which proves the full inference pipeline works.
---
 bin/ci/smoke-agent-inference.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/bin/ci/smoke-agent-inference.sh b/bin/ci/smoke-agent-inference.sh
index 46ea7c7..5548d17 100755
--- a/bin/ci/smoke-agent-inference.sh
+++ b/bin/ci/smoke-agent-inference.sh
@@ -18,7 +18,7 @@ readonly CONTROL_DIR="${AGENT_HOME}/.pi/session-control"
 readonly CONTROL_ALIAS="${CONTROL_DIR}/control-agent.alias"
 readonly START_TIMEOUT_SECONDS=60
 readonly INFERENCE_TIMEOUT_SECONDS=120
-readonly EXPECTED_TOKEN="CI_INFERENCE_OK"
+readonly MIN_RESPONSE_LENGTH=10
 
 started=0
 
@@ -198,19 +198,19 @@ main() {
   log "sending inference prompt (timeout ${INFERENCE_TIMEOUT_SECONDS}s)"
   local response=""
   if ! response="$(rpc_send_wait_turn_end "$socket_path" \
-    "Reply with exactly: ${EXPECTED_TOKEN}" \
+    "Respond with a short greeting." \
     "$INFERENCE_TIMEOUT_SECONDS")"; then
     log "inference failed"
     dump_diagnostics
     return 1
   fi
 
-  # Validate response contains expected token
-  if [[ "$response" == *"$EXPECTED_TOKEN"* ]]; then
-    log "inference response contains expected token"
+  # Any non-trivial response proves end-to-end inference works.
+  if [[ ${#response} -ge $MIN_RESPONSE_LENGTH ]]; then
+    log "inference OK (${#response} chars)"
   else
-    log "unexpected response (missing '${EXPECTED_TOKEN}'):"
-    log "  ${response:0:500}"
+    log "response too short (${#response} chars):"
+    log "  ${response}"
     dump_diagnostics
     return 1
   fi

From f117a7ca8158f504b8bc81a3420e85c8ddb5be20 Mon Sep 17 00:00:00 2001
From: Ben Vinegar <ben@benv.ca>
Date: Mon, 23 Feb 2026 22:59:43 -0500
Subject: [PATCH 5/9] ci: inference smoke asks agent to run health check
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Instead of testing for a parroted token, ask the control-agent to
do its actual job: run a health check (session liveness, heartbeat)
and report status. Assert the response contains a positive health
signal (healthy, running, ok, etc). This validates the full stack:
socket RPC → model inference → tool use → coherent response.
---
 bin/ci/smoke-agent-inference.sh | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/bin/ci/smoke-agent-inference.sh b/bin/ci/smoke-agent-inference.sh
index 5548d17..ee45732 100755
--- a/bin/ci/smoke-agent-inference.sh
+++ b/bin/ci/smoke-agent-inference.sh
@@ -18,7 +18,9 @@ readonly CONTROL_DIR="${AGENT_HOME}/.pi/session-control"
 readonly CONTROL_ALIAS="${CONTROL_DIR}/control-agent.alias"
 readonly START_TIMEOUT_SECONDS=60
 readonly INFERENCE_TIMEOUT_SECONDS=120
-readonly MIN_RESPONSE_LENGTH=10
+# We ask the agent to run a health check and report status.
+# A successful inference proves: socket → RPC → model API → tool use → response.
+# We look for positive health signals in the response.
 
 started=0
 
@@ -195,22 +197,35 @@ main() {
   fi
   log "control socket ready: ${socket_path}"
 
-  log "sending inference prompt (timeout ${INFERENCE_TIMEOUT_SECONDS}s)"
+  log "sending health check prompt (timeout ${INFERENCE_TIMEOUT_SECONDS}s)"
   local response=""
   if ! response="$(rpc_send_wait_turn_end "$socket_path" \
-    "Respond with a short greeting." \
+    "Run a quick health check: verify your session is live, check heartbeat status, and report whether everything looks healthy. Keep it brief." \
     "$INFERENCE_TIMEOUT_SECONDS")"; then
-    log "inference failed"
+    log "inference failed — no response from model"
     dump_diagnostics
     return 1
   fi
 
-  # Any non-trivial response proves end-to-end inference works.
-  if [[ ${#response} -ge $MIN_RESPONSE_LENGTH ]]; then
-    log "inference OK (${#response} chars)"
+  # Normalize to lowercase for matching
+  local lower_response
+  lower_response="$(echo "$response" | tr '[:upper:]' '[:lower:]')"
+
+  # Look for positive health signals in the response
+  local health_ok=0
+  for signal in "healthy" "running" "active" "ok" "good" "operational" "green" "no issues" "everything looks"; do
+    if [[ "$lower_response" == *"$signal"* ]]; then
+      health_ok=1
+      break
+    fi
+  done
+
+  if [[ $health_ok -eq 1 ]]; then
+    log "health check passed — agent reports healthy"
+    log "  response (first 200 chars): ${response:0:200}"
   else
-    log "response too short (${#response} chars):"
-    log "  ${response}"
+    log "health check failed — no positive health signal in response:"
+    log "  ${response:0:500}"
     dump_diagnostics
     return 1
   fi

From 0675850256759fb6d5edf2ed91fea7c6c8fbc428 Mon Sep 17 00:00:00 2001
From: Ben Vinegar <ben@benv.ca>
Date: Mon, 23 Feb 2026 23:01:51 -0500
Subject: [PATCH 6/9] ci: validate inference health check via structured JSON
 schema

Ask the agent to respond with a typed JSON health report:
{status, session_alive, heartbeat_active, message}. Extract,
parse, validate the schema, then assert status=healthy and
session_alive=true. No fuzzy string matching.
---
 bin/ci/smoke-agent-inference.sh | 109 ++++++++++++++++++++++++++------
 1 file changed, 89 insertions(+), 20 deletions(-)

diff --git a/bin/ci/smoke-agent-inference.sh b/bin/ci/smoke-agent-inference.sh
index ee45732..05bffb6 100755
--- a/bin/ci/smoke-agent-inference.sh
+++ b/bin/ci/smoke-agent-inference.sh
@@ -18,9 +18,9 @@ readonly CONTROL_DIR="${AGENT_HOME}/.pi/session-control"
 readonly CONTROL_ALIAS="${CONTROL_DIR}/control-agent.alias"
 readonly START_TIMEOUT_SECONDS=60
 readonly INFERENCE_TIMEOUT_SECONDS=120
-# We ask the agent to run a health check and report status.
+# We ask the agent to run a health check and respond with structured JSON.
 # A successful inference proves: socket → RPC → model API → tool use → response.
-# We look for positive health signals in the response.
+# We parse the JSON and validate the health fields.
 
 started=0
 
@@ -197,38 +197,107 @@ main() {
   fi
   log "control socket ready: ${socket_path}"
 
+  local prompt
+  prompt=$(cat <<'PROMPT'
+Run a quick health check: verify your session is live and check heartbeat status.
+Then respond with ONLY a JSON object (no markdown fences, no other text) matching this schema:
+
+{
+  "status": "healthy" | "degraded" | "unhealthy",
+  "session_alive": true | false,
+  "heartbeat_active": true | false,
+  "message": "<one-line summary>"
+}
+PROMPT
+)
+
   log "sending health check prompt (timeout ${INFERENCE_TIMEOUT_SECONDS}s)"
   local response=""
   if ! response="$(rpc_send_wait_turn_end "$socket_path" \
-    "Run a quick health check: verify your session is live, check heartbeat status, and report whether everything looks healthy. Keep it brief." \
+    "$prompt" \
     "$INFERENCE_TIMEOUT_SECONDS")"; then
     log "inference failed — no response from model"
     dump_diagnostics
     return 1
   fi
 
-  # Normalize to lowercase for matching
-  local lower_response
-  lower_response="$(echo "$response" | tr '[:upper:]' '[:lower:]')"
+  log "raw response: ${response:0:500}"
+
+  # Extract JSON object from response (skip any surrounding text)
+  local json=""
+  json="$(echo "$response" | python3 -c "
+import json, sys, re
+text = sys.stdin.read()
+# Try to find a JSON object in the text
+match = re.search(r'\{[^{}]*\}', text, re.DOTALL)
+if not match:
+    print('NO_JSON', end='')
+    sys.exit(0)
+try:
+    obj = json.loads(match.group())
+    print(json.dumps(obj), end='')
+except json.JSONDecodeError:
+    print('INVALID_JSON', end='')
+")"
+
+  if [[ "$json" == "NO_JSON" ]]; then
+    log "health check failed — no JSON object in response"
+    dump_diagnostics
+    return 1
+  fi
+  if [[ "$json" == "INVALID_JSON" ]]; then
+    log "health check failed — malformed JSON in response"
+    dump_diagnostics
+    return 1
+  fi
 
-  # Look for positive health signals in the response
-  local health_ok=0
-  for signal in "healthy" "running" "active" "ok" "good" "operational" "green" "no issues" "everything looks"; do
-    if [[ "$lower_response" == *"$signal"* ]]; then
-      health_ok=1
-      break
-    fi
-  done
+  # Validate fields
+  local valid=""
+  valid="$(echo "$json" | python3 -c "
+import json, sys
+obj = json.load(sys.stdin)
+errors = []
+status = obj.get('status')
+if status not in ('healthy', 'degraded', 'unhealthy'):
+    errors.append(f'bad status: {status}')
+if not isinstance(obj.get('session_alive'), bool):
+    errors.append('missing/invalid session_alive')
+if not isinstance(obj.get('heartbeat_active'), bool):
+    errors.append('missing/invalid heartbeat_active')
+if errors:
+    print('FAIL:' + '; '.join(errors), end='')
+else:
+    print(json.dumps(obj), end='')
+")"
+
+  if [[ "$valid" == FAIL:* ]]; then
+    log "health check failed — schema validation: ${valid#FAIL:}"
+    dump_diagnostics
+    return 1
+  fi
 
-  if [[ $health_ok -eq 1 ]]; then
-    log "health check passed — agent reports healthy"
-    log "  response (first 200 chars): ${response:0:200}"
-  else
-    log "health check failed — no positive health signal in response:"
-    log "  ${response:0:500}"
+  # Check actual health values
+  local status session_alive heartbeat_active message
+  status="$(echo "$valid" | python3 -c "import json,sys; print(json.load(sys.stdin)['status'])")"
+  session_alive="$(echo "$valid" | python3 -c "import json,sys; print(json.load(sys.stdin)['session_alive'])")"
+  heartbeat_active="$(echo "$valid" | python3 -c "import json,sys; print(json.load(sys.stdin)['heartbeat_active'])")"
+  message="$(echo "$valid" | python3 -c "import json,sys; print(json.load(sys.stdin).get('message',''))")"
+
+  log "status=$status session_alive=$session_alive heartbeat_active=$heartbeat_active"
+  log "message: $message"
+
+  if [[ "$status" != "healthy" ]]; then
+    log "agent reports status=$status (expected healthy)"
     dump_diagnostics
     return 1
   fi
+  if [[ "$session_alive" != "True" ]]; then
+    log "agent reports session not alive"
+    dump_diagnostics
+    return 1
+  fi
+
+  log "health check passed"
 
   log "stopping baudbot"
   sudo baudbot stop

From 57c6874731089eb20db147064ec8db7a4221a216 Mon Sep 17 00:00:00 2001
From: Ben Vinegar <ben@benv.ca>
Date: Mon, 23 Feb 2026 23:08:22 -0500
Subject: [PATCH 7/9] ci: wait for startup turn + stream agent journal to CI
 log

- Subscribe to turn_end and drain the startup turn before sending
  the health check prompt (fixes race where we got the startup
  response instead of ours).
- Tail journalctl -u baudbot in the background so the CI log shows
  what the agent is doing in real time.
---
 bin/ci/smoke-agent-inference.sh | 70 +++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/bin/ci/smoke-agent-inference.sh b/bin/ci/smoke-agent-inference.sh
index 05bffb6..fc5f17c 100755
--- a/bin/ci/smoke-agent-inference.sh
+++ b/bin/ci/smoke-agent-inference.sh
@@ -28,8 +28,11 @@ log() {
   printf '[inference-smoke] %s\n' "$*"
 }
 
+journal_pid=""
+
 cleanup() {
   local exit_code=$?
+  [[ -n "$journal_pid" ]] && kill "$journal_pid" 2>/dev/null || true
   if [[ $started -eq 1 ]]; then
     log "cleanup: stopping baudbot"
     sudo baudbot stop >/dev/null 2>&1 || true
@@ -81,6 +84,58 @@ dump_diagnostics() {
   log "--- end diagnostics ---"
 }
 
+# Subscribe to turn_end and wait for the next turn to complete.
+# Used to drain the agent's startup turn before sending our prompt.
+rpc_wait_for_turn_end() {
+  local socket_path="$1"
+  local timeout_seconds="$2"
+
+  sudo -u "$AGENT_USER" python3 - "$socket_path" "$timeout_seconds" <<'PY'
+import json
+import socket
+import sys
+
+sock_path = sys.argv[1]
+timeout_seconds = int(sys.argv[2])
+
+subscribe_cmd = {"type": "subscribe", "event": "turn_end"}
+
+client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+try:
+    client.settimeout(timeout_seconds)
+    client.connect(sock_path)
+    client.sendall((json.dumps(subscribe_cmd) + "\n").encode("utf-8"))
+
+    buf = b""
+    while True:
+        chunk = client.recv(8192)
+        if not chunk:
+            print("connection closed before turn_end", file=sys.stderr)
+            sys.exit(1)
+        buf += chunk
+
+        while b"\n" in buf:
+            line, buf = buf.split(b"\n", 1)
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                msg = json.loads(line.decode("utf-8", errors="replace"))
+            except json.JSONDecodeError:
+                continue
+
+            if msg.get("type") == "event" and msg.get("event") == "turn_end":
+                print("ok")
+                sys.exit(0)
+
+except socket.timeout:
+    print("timeout waiting for turn_end", file=sys.stderr)
+    sys.exit(1)
+finally:
+    client.close()
+PY
+}
+
 # Send a message via session-control RPC and wait for turn_end.
 # Prints the assistant response content on success, exits non-zero on failure.
 rpc_send_wait_turn_end() {
@@ -211,6 +266,21 @@ Then respond with ONLY a JSON object (no markdown fences, no other text) matchin
 PROMPT
 )
 
+  # Stream agent logs in the background so CI output shows what the agent is doing.
+  sudo journalctl -u baudbot -f --no-pager -o cat &
+  local journal_pid=$!
+
+  # The agent's first turn is its startup routine (loading skill, running
+  # checklist). We need to wait for that to finish before sending our prompt,
+  # otherwise we'll get the startup turn_end instead of ours.
+  log "waiting for agent startup turn to complete"
+  if ! rpc_wait_for_turn_end "$socket_path" "$INFERENCE_TIMEOUT_SECONDS" >/dev/null; then
+    log "agent startup turn did not complete"
+    dump_diagnostics
+    return 1
+  fi
+  log "startup turn complete"
+
   log "sending health check prompt (timeout ${INFERENCE_TIMEOUT_SECONDS}s)"
   local response=""
   if ! response="$(rpc_send_wait_turn_end "$socket_path" \

From 6adcaa1e474e7c92010122b56d4aeb2b753b4fe1 Mon Sep 17 00:00:00 2001
From: Ben Vinegar <ben@benv.ca>
Date: Mon, 23 Feb 2026 23:18:41 -0500
Subject: [PATCH 8/9] ci: use follow_up + marker to survive multi-turn startup

The control-agent startup is multi-turn (checklist, tool calls, etc).
Instead of trying to drain it, send the health check as a follow_up
with a unique marker string. Re-subscribe to turn_end after each
event and keep consuming until we find the response containing our
marker. Also bump timeouts (300s inference, 15m job) to accommodate
the full startup + health check flow.
---
 .github/workflows/integration.yml |   2 +-
 bin/ci/smoke-agent-inference.sh   | 142 +++++++++---------------------
 2 files changed, 45 insertions(+), 99 deletions(-)

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index ad0b37d..7e44e37 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -71,7 +71,7 @@ jobs:
             setup_script: bin/ci/setup-arch.sh
 
     name: ${{ matrix.distro }}
-    timeout-minutes: 10
+    timeout-minutes: 15
 
     steps:
       - uses: actions/checkout@v4
diff --git a/bin/ci/smoke-agent-inference.sh b/bin/ci/smoke-agent-inference.sh
index fc5f17c..141dcbc 100755
--- a/bin/ci/smoke-agent-inference.sh
+++ b/bin/ci/smoke-agent-inference.sh
@@ -17,7 +17,7 @@ readonly AGENT_ENV="${AGENT_HOME}/.config/.env"
 readonly CONTROL_DIR="${AGENT_HOME}/.pi/session-control"
 readonly CONTROL_ALIAS="${CONTROL_DIR}/control-agent.alias"
 readonly START_TIMEOUT_SECONDS=60
-readonly INFERENCE_TIMEOUT_SECONDS=120
+readonly INFERENCE_TIMEOUT_SECONDS=300
 # We ask the agent to run a health check and respond with structured JSON.
 # A successful inference proves: socket → RPC → model API → tool use → response.
 # We parse the JSON and validate the health fields.
@@ -84,75 +84,28 @@ dump_diagnostics() {
   log "--- end diagnostics ---"
 }
 
-# Subscribe to turn_end and wait for the next turn to complete.
-# Used to drain the agent's startup turn before sending our prompt.
-rpc_wait_for_turn_end() {
-  local socket_path="$1"
-  local timeout_seconds="$2"
-
-  sudo -u "$AGENT_USER" python3 - "$socket_path" "$timeout_seconds" <<'PY'
-import json
-import socket
-import sys
-
-sock_path = sys.argv[1]
-timeout_seconds = int(sys.argv[2])
-
-subscribe_cmd = {"type": "subscribe", "event": "turn_end"}
-
-client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-try:
-    client.settimeout(timeout_seconds)
-    client.connect(sock_path)
-    client.sendall((json.dumps(subscribe_cmd) + "\n").encode("utf-8"))
-
-    buf = b""
-    while True:
-        chunk = client.recv(8192)
-        if not chunk:
-            print("connection closed before turn_end", file=sys.stderr)
-            sys.exit(1)
-        buf += chunk
-
-        while b"\n" in buf:
-            line, buf = buf.split(b"\n", 1)
-            line = line.strip()
-            if not line:
-                continue
-            try:
-                msg = json.loads(line.decode("utf-8", errors="replace"))
-            except json.JSONDecodeError:
-                continue
-
-            if msg.get("type") == "event" and msg.get("event") == "turn_end":
-                print("ok")
-                sys.exit(0)
-
-except socket.timeout:
-    print("timeout waiting for turn_end", file=sys.stderr)
-    sys.exit(1)
-finally:
-    client.close()
-PY
-}
-
-# Send a message via session-control RPC and wait for turn_end.
-# Prints the assistant response content on success, exits non-zero on failure.
-rpc_send_wait_turn_end() {
+# Send a follow-up message and wait for a turn_end whose response contains
+# a marker string. This handles the agent's multi-turn startup gracefully:
+# the message is queued after the current turn, and we keep consuming
+# turn_end events until we find one with our expected content.
+rpc_send_and_wait_for_marker() {
   local socket_path="$1"
   local message="$2"
-  local timeout_seconds="$3"
+  local marker="$3"
+  local timeout_seconds="$4"
 
-  sudo -u "$AGENT_USER" python3 - "$socket_path" "$message" "$timeout_seconds" <<'PY'
+  sudo -u "$AGENT_USER" python3 - "$socket_path" "$message" "$marker" "$timeout_seconds" <<'PY'
 import json
 import socket
 import sys
+import re
 
 sock_path = sys.argv[1]
 message = sys.argv[2]
-timeout_seconds = int(sys.argv[3])
+marker = sys.argv[3]
+timeout_seconds = int(sys.argv[4])
 
-send_cmd = {"type": "send", "message": message, "mode": "steer"}
+send_cmd = {"type": "send", "message": message, "mode": "follow_up"}
 subscribe_cmd = {"type": "subscribe", "event": "turn_end"}
 
 client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
@@ -160,17 +113,16 @@ try:
     client.settimeout(timeout_seconds)
     client.connect(sock_path)
 
-    # Send both commands
     client.sendall((json.dumps(send_cmd) + "\n").encode("utf-8"))
     client.sendall((json.dumps(subscribe_cmd) + "\n").encode("utf-8"))
 
     buf = b""
-    send_response = None
+    send_ack = False
 
     while True:
         chunk = client.recv(8192)
         if not chunk:
-            print("connection closed before turn_end", file=sys.stderr)
+            print("connection closed", file=sys.stderr)
             sys.exit(1)
         buf += chunk
 
@@ -185,31 +137,27 @@ try:
             except json.JSONDecodeError:
                 continue
 
-            if msg.get("type") == "response":
-                cmd = msg.get("command", "")
-                if cmd == "send":
-                    if not msg.get("success", False):
-                        print(f"send failed: {msg.get('error', 'unknown')}", file=sys.stderr)
-                        sys.exit(1)
-                    send_response = msg
-                # Ignore subscribe response
+            if msg.get("type") == "response" and msg.get("command") == "send":
+                if not msg.get("success", False):
+                    print(f"send failed: {msg.get('error', 'unknown')}", file=sys.stderr)
+                    sys.exit(1)
+                send_ack = True
+                # turn_end subscriptions are one-shot; re-subscribe so we
+                # keep receiving events across multiple startup turns.
+                client.sendall((json.dumps(subscribe_cmd) + "\n").encode("utf-8"))
                 continue
 
             if msg.get("type") == "event" and msg.get("event") == "turn_end":
-                if send_response is None:
-                    print("received turn_end before send response", file=sys.stderr)
-                    sys.exit(1)
                 data = msg.get("data", {})
-                assistant_msg = data.get("message", {})
-                content = assistant_msg.get("content", "")
-                if not content:
-                    print("turn completed but no assistant content", file=sys.stderr)
-                    sys.exit(1)
-                print(content)
-                sys.exit(0)
+                content = (data.get("message") or {}).get("content", "")
+                if marker in content:
+                    # This is the turn that answered our prompt
+                    print(content)
+                    sys.exit(0)
+                # Not our turn — re-subscribe for the next one
+                client.sendall((json.dumps(subscribe_cmd) + "\n").encode("utf-8"))
+                continue
 
-    print("stream ended without turn_end event", file=sys.stderr)
-    sys.exit(1)
 except socket.timeout:
     print("timeout waiting for inference response", file=sys.stderr)
     sys.exit(1)
@@ -252,12 +200,17 @@ main() {
   fi
   log "control socket ready: ${socket_path}"
 
+  # Unique marker so we can identify our response among startup turns.
+  local marker="CI_HEALTH_CHECK_RESPONSE"
+
   local prompt
-  prompt=$(cat <<'PROMPT'
+  prompt=$(cat <<PROMPT
 Run a quick health check: verify your session is live and check heartbeat status.
-Then respond with ONLY a JSON object (no markdown fences, no other text) matching this schema:
+Then respond with ONLY a JSON object (no markdown fences, no other text) matching this schema.
+You MUST include the marker field exactly as shown.
 
 {
+  "marker": "${marker}",
   "status": "healthy" | "degraded" | "unhealthy",
   "session_alive": true | false,
   "heartbeat_active": true | false,
@@ -268,23 +221,16 @@ PROMPT
 
   # Stream agent logs in the background so CI output shows what the agent is doing.
   sudo journalctl -u baudbot -f --no-pager -o cat &
-  local journal_pid=$!
-
-  # The agent's first turn is its startup routine (loading skill, running
-  # checklist). We need to wait for that to finish before sending our prompt,
-  # otherwise we'll get the startup turn_end instead of ours.
-  log "waiting for agent startup turn to complete"
-  if ! rpc_wait_for_turn_end "$socket_path" "$INFERENCE_TIMEOUT_SECONDS" >/dev/null; then
-    log "agent startup turn did not complete"
-    dump_diagnostics
-    return 1
-  fi
-  log "startup turn complete"
+  journal_pid=$!
 
+  # Send the health check as a follow-up (queued after the current turn).
+  # The agent may be mid-startup, so we keep consuming turn_end events
+  # until we find the one containing our marker.
   log "sending health check prompt (timeout ${INFERENCE_TIMEOUT_SECONDS}s)"
   local response=""
-  if ! response="$(rpc_send_wait_turn_end "$socket_path" \
+  if ! response="$(rpc_send_and_wait_for_marker "$socket_path" \
     "$prompt" \
+    "$marker" \
     "$INFERENCE_TIMEOUT_SECONDS")"; then
     log "inference failed — no response from model"
     dump_diagnostics

From 050179e7dae8aa6fc88d83f8131ba41fc605e114 Mon Sep 17 00:00:00 2001
From: Ben Vinegar <ben@benv.ca>
Date: Tue, 24 Feb 2026 07:47:57 -0500
Subject: [PATCH 9/9] ci: accept degraded status (Slack bridge has dummy tokens
 in CI)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The agent correctly reports 'degraded' because the bridge can't
auth with dummy Slack tokens. Both healthy and degraded are valid —
only 'unhealthy' indicates a core runtime failure.
---
 bin/ci/smoke-agent-inference.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bin/ci/smoke-agent-inference.sh b/bin/ci/smoke-agent-inference.sh
index 141dcbc..c030020 100755
--- a/bin/ci/smoke-agent-inference.sh
+++ b/bin/ci/smoke-agent-inference.sh
@@ -302,8 +302,11 @@ else:
   log "status=$status session_alive=$session_alive heartbeat_active=$heartbeat_active"
   log "message: $message"
 
-  if [[ "$status" != "healthy" ]]; then
-    log "agent reports status=$status (expected healthy)"
+  # In CI the Slack bridge has dummy tokens, so the agent correctly reports
+  # "degraded" (bridge auth failure). Both "healthy" and "degraded" are
+  # acceptable — "unhealthy" means core inference/session is broken.
+  if [[ "$status" == "unhealthy" ]]; then
+    log "agent reports unhealthy — core runtime failure"
     dump_diagnostics
     return 1
   fi