From 58443b0f8caa4bb6203bdb90ce6273b723eb43e0 Mon Sep 17 00:00:00 2001 From: Sam & Claude Date: Fri, 5 Jun 2026 11:45:58 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat(bhyve):=20QA=20gate=20in=20run-bhyve-t?= =?UTF-8?q?est.sh=20=E2=80=94=20assert=20clean=20boot,=20fail=20on=20regre?= =?UTF-8?q?ssions=20(Sam=20&=20Claude)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bhyve-test.sh captures the guest serial console; this adds a QA gate over that log so the harness can be used as a build gate (appliance runbook §3): - CRITICAL (gate the exit): no kernel panic, no reboot loop, boot reached login/sddm, and no "permission denied" (regression guard for the colibri socket EACCES we just fixed). - INFO (best-effort console markers): colibri socket ready, no store-open panic, clawdie-live-gpu ran. - Prints the deeper in-guest checklist (socket mode, glasspane query, GPU branch) that lives in guest logs, not the console. Exit is non-zero if the boot harness failed OR any critical assert failed. Also pass-through args to bhyve-test.sh (ISO path / --name). sh -n clean; gate logic unit-tested (clean boot → pass/exit0; panic+EACCES+no-login → fail). INFO/CRITICAL patterns should be confirmed against a real boot console on first run. Co-Authored-By: Claude Opus 4.8 --- scripts/run-bhyve-test.sh | 89 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 7 deletions(-) diff --git a/scripts/run-bhyve-test.sh b/scripts/run-bhyve-test.sh index 19598bc5..df4338d0 100755 --- a/scripts/run-bhyve-test.sh +++ b/scripts/run-bhyve-test.sh @@ -1,5 +1,16 @@ #!/bin/sh -# Wrapper script to run bhyve test with full logging and error capture +# Wrapper script to run bhyve test with full logging, error capture, and a +# QA gate that asserts the boot actually came up clean. +# +# bhyve-test.sh boots the image with com1,stdio, so the guest serial console is +# captured in the log below. The QA gate scans that console for required boot +# markers and forbidden failures (panics, restart loops, the colibri +# "permission denied" regression), and exits non-zero if a critical check fails +# — so this can be used as a build gate (see private appliance runbook §3). +# +# Console-scannable checks are automated. Deeper in-guest checks (socket mode, +# glasspane query, GPU branch) are printed as a checklist to run in the guest +# (10.99.0.2) — they live in log files, not the serial console. LOG_DIR="${PWD}/logs" mkdir -p "$LOG_DIR" @@ -10,12 +21,12 @@ echo "" >> "$LOG_FILE" # Run the actual test script, capturing both stdout and stderr echo "=== BHYVE TEST START ===" >> "$LOG_FILE" -echo "Command: $(pwd)/tmp/bhyve-test-setup.sh" >> "$LOG_FILE" +echo "Command: $(pwd)/scripts/bhyve-test.sh $*" >> "$LOG_FILE" echo "Started: $(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG_FILE" echo "" >> "$LOG_FILE" -# Run with tee to show output AND log it -timeout 1800 ./scripts/bhyve-test.sh 2>&1 | tee -a "$LOG_FILE" +# Run with tee to show output AND log it. Pass through any args (ISO path, --name). +timeout 1800 ./scripts/bhyve-test.sh "$@" 2>&1 | tee -a "$LOG_FILE" EXIT_CODE=$? echo "" >> "$LOG_FILE" @@ -27,9 +38,73 @@ echo "" >> "$LOG_FILE" # Check for errors in log echo "=== ERROR SCAN ===" >> "$LOG_FILE" echo "Errors found:" >> "$LOG_FILE" -grep -i "error\|failed\|panic\|fatal" "$LOG_FILE" | tail -20 >> "$LOG_FILE" || echo "No errors detected" >> "$LOG_FILE" - +grep -iE "error|failed|panic|fatal" "$LOG_FILE" | tail -20 >> "$LOG_FILE" || echo "No errors detected" >> "$LOG_FILE" echo "" >> "$LOG_FILE" + +# ── QA gate ──────────────────────────────────────────────────────────────── +# CRITICAL checks gate the exit code. INFO checks are reported only (their +# markers may not always reach the serial console depending on rc verbosity — +# confirm/adjust patterns against a real boot before promoting them to CRITICAL). +QA_FAIL=0 + +# present DESC PATTERN LEVEL +qa_present() { + if grep -qiE "$2" "$LOG_FILE"; then + echo " PASS $1" + else + echo " ${3} $1 (missing /$2/)" + [ "$3" = "FAIL" ] && QA_FAIL=1 + fi +} + +# absent DESC PATTERN MAXALLOWED LEVEL +qa_absent() { + # grep -c prints the count (0 on no match) but exits 1 when 0 — capture the + # number directly; do NOT use `|| echo 0` (that doubles the output). + _n=$(grep -ciE "$2" "$LOG_FILE" 2>/dev/null) + _n=${_n:-0} + if [ "$_n" -le "$3" ]; then + echo " PASS $1 (${_n} hits, max $3)" + else + echo " ${4} $1 (${_n} hits, max $3)" + [ "$4" = "FAIL" ] && QA_FAIL=1 + fi +} + +{ + echo "=== QA GATE ===" + echo "-- critical (gate the build) --" + qa_absent "no kernel panic" 'panic:|Fatal trap|panic @' 0 FAIL + qa_absent "no repeated reboot loop" '---<>---' 1 FAIL + qa_present "boot reached login/getty" 'login:|FreeBSD/amd64 \(.*\) \(ttyu|sddm' FAIL + qa_absent "no colibri permission-denied" 'permission denied|EACCES|Permission denied' 0 FAIL + + echo "-- informational (best-effort console markers) --" + qa_present "colibri-daemon socket ready" 'colibri-daemon socket ready|Herdr socket API listening' INFO + qa_absent "no store-open panic" 'failed to open coordination store' 0 INFO + qa_present "clawdie-live-gpu ran" 'selected live KMS|recommended branch|clawdie.live.gpu' INFO + + echo "" + echo "-- deeper in-guest checks (run in the guest @ 10.99.0.2; not on console) --" + echo " service colibri_daemon status # one stable PID" + echo " stat -f '%Sp %Su:%Sg' /var/run/colibri/colibri.sock # expect srwxrwx--- colibri:colibri" + echo " printf '{\"cmd\":\"status\"}\\n' | nc -U /var/run/colibri/colibri.sock # no EACCES" + echo " cat /var/log/clawdie-live-gpu.log # detected id + branch matches build" + echo " kldstat | grep nvidia # NVIDIA builds only" +} | tee -a "$LOG_FILE" + +echo "" | tee -a "$LOG_FILE" +if [ "$EXIT_CODE" -ne 0 ]; then + echo "QA GATE: boot harness exited non-zero (${EXIT_CODE})" | tee -a "$LOG_FILE" +fi +if [ "$QA_FAIL" -ne 0 ]; then + echo "QA GATE: FAIL — critical assertion(s) failed" | tee -a "$LOG_FILE" +else + echo "QA GATE: critical assertions PASS" | tee -a "$LOG_FILE" +fi + +echo "" echo "Log saved to: $LOG_FILE" -exit $EXIT_CODE +# Fail the gate if the harness failed OR any critical assertion failed. +[ "$EXIT_CODE" -eq 0 ] && [ "$QA_FAIL" -eq 0 ] -- 2.45.3 From 667ebdc210869ff3104c200a8b87cdbd5b78f3e8 Mon Sep 17 00:00:00 2001 From: Sam & Claude Date: Fri, 5 Jun 2026 12:05:32 +0200 Subject: [PATCH 2/2] fix(bhyve): EU display dates (DD.mmm.YYYY) per AGENTS date rule (Sam & Claude) User-facing timestamps in run-bhyve-test.sh used the ambiguous numeric form %Y-%m-%d. Switch to European display (date +%d.%b.%Y %H:%M:%S). Log filename keeps sortable ISO 8601 basic (machine artifact, per the rule's internal-storage exemption). Co-Authored-By: Claude Opus 4.8 --- scripts/run-bhyve-test.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/run-bhyve-test.sh b/scripts/run-bhyve-test.sh index df4338d0..4ee35c24 100755 --- a/scripts/run-bhyve-test.sh +++ b/scripts/run-bhyve-test.sh @@ -14,15 +14,18 @@ LOG_DIR="${PWD}/logs" mkdir -p "$LOG_DIR" +# Log filename uses sortable ISO 8601 basic (machine artifact name, per the +# AGENTS.md date rule's internal-storage exemption). User-facing timestamps +# below use European display format (DD.mmm.YYYY HH:MM:SS). LOG_FILE="${LOG_DIR}/bhyve-test-$(date +%Y%m%d-%H%M%S).log" -echo "Test started: $(date)" > "$LOG_FILE" +echo "Test started: $(date '+%d.%b.%Y %H:%M:%S')" > "$LOG_FILE" echo "Log: $LOG_FILE" >&2 echo "" >> "$LOG_FILE" # Run the actual test script, capturing both stdout and stderr echo "=== BHYVE TEST START ===" >> "$LOG_FILE" echo "Command: $(pwd)/scripts/bhyve-test.sh $*" >> "$LOG_FILE" -echo "Started: $(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG_FILE" +echo "Started: $(date '+%d.%b.%Y %H:%M:%S')" >> "$LOG_FILE" echo "" >> "$LOG_FILE" # Run with tee to show output AND log it. Pass through any args (ISO path, --name). @@ -31,7 +34,7 @@ EXIT_CODE=$? echo "" >> "$LOG_FILE" echo "=== BHYVE TEST END ===" >> "$LOG_FILE" -echo "Ended: $(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG_FILE" +echo "Ended: $(date '+%d.%b.%Y %H:%M:%S')" >> "$LOG_FILE" echo "Exit code: $EXIT_CODE" >> "$LOG_FILE" echo "" >> "$LOG_FILE" -- 2.45.3