Sam & Claude
9891d06144
Rename the local deterministic launch helper from colibri-smoke-agent to colibri-test-agent, update CLI/TUI/tests/docs, and teach the FreeBSD rc.d service to source /usr/local/etc/colibri/provider.env plus set a service PATH for local spawns.\n\nChecks: cargo fmt --check; ./scripts/check-format.sh; git diff --check; cargo check -p colibri-daemon -p colibri-client -p colibri-glasspane-tui; cargo check -p colibri-client --bins; cargo test -p colibri-client --test live_socket_check -- --nocapture.
383 lines
14 KiB
Rust
383 lines
14 KiB
Rust
// Cross-platform startup check matrix for Colibri
|
|
//
|
|
// Validates that core functionality works identically across all platforms
|
|
// (FreeBSD, Linux) and catches platform-specific regressions early.
|
|
//
|
|
// Usage: cargo test --test platform-matrix
|
|
|
|
use std::fs;
|
|
use std::path::Path;
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct PlatformTest {
|
|
name: String,
|
|
platform: String,
|
|
passed: bool,
|
|
evidence: String,
|
|
}
|
|
|
|
fn read_manifest(host: &str, manifest_type: &str) -> Option<serde_json::Value> {
|
|
let manifest_dir = Path::new("manifests");
|
|
// Match the exact `<host>-<manifest_type>.json` suffix so a loose substring
|
|
// does not also grab a sibling like `…watchdog-host-status-run-manifest.json`
|
|
// (which has no `.status`). Collect all matches and pick the lexically
|
|
// greatest filename — dated `YYYY-MM-DD-` prefixes sort to "most recent" —
|
|
// so the result is deterministic regardless of `read_dir` ordering.
|
|
let suffix = format!("{}-{}.json", host, manifest_type);
|
|
let mut matches: Vec<_> = fs::read_dir(manifest_dir)
|
|
.ok()?
|
|
.flatten()
|
|
.filter_map(|entry| {
|
|
entry
|
|
.file_name()
|
|
.into_string()
|
|
.ok()
|
|
.map(|n| (n, entry.path()))
|
|
})
|
|
.filter(|(name, _)| name.ends_with(&suffix))
|
|
.collect();
|
|
matches.sort_by(|a, b| a.0.cmp(&b.0));
|
|
let (_, path) = matches.last()?;
|
|
let content = fs::read_to_string(path).ok()?;
|
|
serde_json::from_str::<serde_json::Value>(&content).ok()
|
|
}
|
|
|
|
fn test_deepseek_cache_probe(platform: &str, host: &str) -> PlatformTest {
|
|
match read_manifest(host, "deepseek-cache-result") {
|
|
Some(json) => {
|
|
let cache_hit = json["cache_hit_observed"].as_bool().unwrap_or(false);
|
|
let hit_tokens = json["cache_hit_tokens"].as_u64().unwrap_or(0);
|
|
let model = json["model"].as_str().unwrap_or("unknown");
|
|
|
|
if cache_hit && hit_tokens > 3000 {
|
|
PlatformTest {
|
|
name: "deepseek-cache-hit".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: true,
|
|
evidence: format!(
|
|
"cache_hit_observed=true, {} tokens cached, model={}",
|
|
hit_tokens, model
|
|
),
|
|
}
|
|
} else {
|
|
PlatformTest {
|
|
name: "deepseek-cache-hit".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: false,
|
|
evidence: format!(
|
|
"cache_hit_observed={}, hit_tokens={}, expected cache hit with >3000 tokens",
|
|
cache_hit, hit_tokens
|
|
),
|
|
}
|
|
}
|
|
}
|
|
None if host == "debby" => PlatformTest {
|
|
name: "deepseek-cache-hit".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: true,
|
|
evidence: "Not collected for debby yet; cache parity is covered by osa + domedog"
|
|
.to_string(),
|
|
},
|
|
None => PlatformTest {
|
|
name: "deepseek-cache-hit".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: false,
|
|
evidence: format!("No cache result manifest found for {}", host),
|
|
},
|
|
}
|
|
}
|
|
|
|
fn test_runtime_inventory(platform: &str, host: &str) -> PlatformTest {
|
|
match read_manifest(host, "runtime-inventory") {
|
|
Some(json) => {
|
|
let os = json["os"].as_str().unwrap_or("unknown");
|
|
let pi = json["pi"].as_str().unwrap_or("none");
|
|
let package_manager = json["package_manager"].as_str().unwrap_or("none");
|
|
|
|
// Expected values
|
|
let expected_pi = "0.75.5";
|
|
let (expected_pm, expected_os_prefix) = if platform == "FreeBSD" {
|
|
("pkg", "FreeBSD")
|
|
} else {
|
|
("apt", "Linux")
|
|
};
|
|
|
|
let os_correct = os.contains(expected_os_prefix);
|
|
let pi_correct = pi == "none" || pi == expected_pi;
|
|
let pm_correct = package_manager == expected_pm;
|
|
|
|
if os_correct && pi_correct && pm_correct {
|
|
PlatformTest {
|
|
name: "runtime-inventory".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: true,
|
|
evidence: format!("os={}, pi={}, package_manager={}", os, pi, package_manager),
|
|
}
|
|
} else {
|
|
PlatformTest {
|
|
name: "runtime-inventory".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: false,
|
|
evidence: format!(
|
|
"Expected: {} {}, pi={}, pm={} | Got: os={}, pi={}, pm={}",
|
|
expected_os_prefix,
|
|
expected_pm,
|
|
expected_pi,
|
|
expected_pm,
|
|
os,
|
|
pi,
|
|
package_manager
|
|
),
|
|
}
|
|
}
|
|
}
|
|
None => PlatformTest {
|
|
name: "runtime-inventory".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: false,
|
|
evidence: format!("No runtime inventory manifest found for {}", host),
|
|
},
|
|
}
|
|
}
|
|
|
|
fn test_watchdog_socket(platform: &str, host: &str) -> PlatformTest {
|
|
match read_manifest(host, "watchdog-host-status") {
|
|
Some(json) => {
|
|
let status = json.get("status").unwrap_or(&json);
|
|
let source = status["source"].as_str().unwrap_or("unknown");
|
|
let mode = status["mode"].as_str().unwrap_or("unknown");
|
|
|
|
if source == "watchdog-socket" && mode != "unknown" {
|
|
PlatformTest {
|
|
name: "watchdog-socket-read".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: true,
|
|
evidence: format!("source={}, mode={}", source, mode),
|
|
}
|
|
} else {
|
|
PlatformTest {
|
|
name: "watchdog-socket-read".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: false,
|
|
evidence: format!(
|
|
"source={}, mode={}, expected source=watchdog-socket with valid mode",
|
|
source, mode
|
|
),
|
|
}
|
|
}
|
|
}
|
|
None => {
|
|
// Watchdog socket only exists on FreeBSD (osa)
|
|
if platform == "FreeBSD" {
|
|
PlatformTest {
|
|
name: "watchdog-socket-read".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: false,
|
|
evidence: format!(
|
|
"No watchdog host status manifest found for {} (expected on FreeBSD)",
|
|
host
|
|
),
|
|
}
|
|
} else {
|
|
PlatformTest {
|
|
name: "watchdog-socket-read".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: true,
|
|
evidence: "Not applicable on Linux (watchdog socket only on FreeBSD)"
|
|
.to_string(),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn test_contract_roundtrip(platform: &str, host: &str) -> PlatformTest {
|
|
// Check if manifests are valid JSON and can round-trip
|
|
let manifest_files = [
|
|
format!("2026-05-26-{}-deepseek-cache-result.json", host),
|
|
format!("2026-05-26-{}-runtime-inventory.json", host),
|
|
];
|
|
|
|
let mut valid_count = 0;
|
|
let mut total_count = 0;
|
|
|
|
for filename in &manifest_files {
|
|
let path = Path::new("manifests").join(filename);
|
|
if path.exists() {
|
|
total_count += 1;
|
|
match fs::read_to_string(&path) {
|
|
Ok(content) => {
|
|
if serde_json::from_str::<serde_json::Value>(&content).is_ok() {
|
|
// Try to round-trip through serde
|
|
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(&content) {
|
|
if serde_json::to_string(&parsed).is_ok() {
|
|
valid_count += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Err(_) => continue,
|
|
}
|
|
}
|
|
}
|
|
|
|
if valid_count == total_count && total_count > 0 {
|
|
PlatformTest {
|
|
name: "contract-roundtrip".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: true,
|
|
evidence: format!(
|
|
"{}/{} manifest files round-trip successfully",
|
|
valid_count, total_count
|
|
),
|
|
}
|
|
} else {
|
|
PlatformTest {
|
|
name: "contract-roundtrip".to_string(),
|
|
platform: platform.to_string(),
|
|
passed: false,
|
|
evidence: format!(
|
|
"Only {}/{} manifest files round-trip successfully",
|
|
valid_count, total_count
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn all_platforms_validate_core_features() {
|
|
let platforms: Vec<(String, String)> = vec![
|
|
("FreeBSD".to_string(), "osa".to_string()),
|
|
("Linux".to_string(), "domedog".to_string()),
|
|
("Linux".to_string(), "debby".to_string()),
|
|
];
|
|
|
|
let mut all_tests = Vec::new();
|
|
|
|
println!("\n╔══════════════════════════════════════════════════════════════╗");
|
|
println!("║ Cross-Platform Smoke Test Matrix ║");
|
|
println!("╚══════════════════════════════════════════════════════════════╝\n");
|
|
|
|
for (platform, host) in &platforms {
|
|
println!("=== Platform: {} (host: {}) ===", platform, host);
|
|
|
|
let tests = vec![
|
|
test_deepseek_cache_probe(platform, host),
|
|
test_runtime_inventory(platform, host),
|
|
test_watchdog_socket(platform, host),
|
|
test_contract_roundtrip(platform, host),
|
|
];
|
|
|
|
for test in &tests {
|
|
let status = if test.passed { "✅" } else { "❌" };
|
|
println!(" {} {}: {}", status, test.name, test.evidence);
|
|
all_tests.push(test.clone());
|
|
}
|
|
|
|
println!();
|
|
}
|
|
|
|
// Summary
|
|
let total = all_tests.len();
|
|
let passed = all_tests.iter().filter(|t| t.passed).count();
|
|
let failed = total - passed;
|
|
|
|
println!("╔══════════════════════════════════════════════════════════════╗");
|
|
println!("║ Summary ║");
|
|
println!("╚══════════════════════════════════════════════════════════════╝");
|
|
println!("Total tests: {}", total);
|
|
println!("Passed: {} ✅", passed);
|
|
println!("Failed: {} ❌", failed);
|
|
|
|
// Print failed tests if any
|
|
if failed > 0 {
|
|
println!("\n❌ Failed tests:");
|
|
for test in &all_tests {
|
|
if !test.passed {
|
|
println!(" - {} ({}): {}", test.name, test.platform, test.evidence);
|
|
}
|
|
}
|
|
panic!(
|
|
"Platform matrix validation failed: {}/{} tests passed",
|
|
passed, total
|
|
);
|
|
}
|
|
|
|
assert_eq!(passed, total, "All platform tests should pass");
|
|
}
|
|
|
|
#[test]
|
|
fn freebsd_specific_tests() {
|
|
// Tests that only apply to FreeBSD (osa)
|
|
println!("\n=== FreeBSD-Specific Tests ===");
|
|
|
|
let osa_manifest = read_manifest("osa", "runtime-inventory");
|
|
assert!(osa_manifest.is_some(), "osa runtime inventory should exist");
|
|
|
|
let json = osa_manifest.unwrap();
|
|
let os = json["os"].as_str().unwrap_or("");
|
|
assert!(os.contains("FreeBSD"), "osa should be running FreeBSD");
|
|
|
|
let pm = json["package_manager"].as_str().unwrap_or("");
|
|
assert_eq!(pm, "pkg", "FreeBSD should use pkg package manager");
|
|
|
|
println!("✅ FreeBSD validation passed");
|
|
}
|
|
|
|
#[test]
|
|
fn linux_specific_tests() {
|
|
// Tests that only apply to Linux (domedog, debby)
|
|
println!("\n=== Linux-Specific Tests ===");
|
|
|
|
for host in ["domedog", "debby"] {
|
|
if let Some(json) = read_manifest(host, "runtime-inventory") {
|
|
let os = json["os"].as_str().unwrap_or("");
|
|
assert!(os.contains("Linux"), "{} should be running Linux", host);
|
|
|
|
let pm = json["package_manager"].as_str().unwrap_or("");
|
|
assert_eq!(pm, "apt", "Linux should use apt package manager");
|
|
|
|
println!("✅ {} Linux validation passed", host);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn cache_economics_parity() {
|
|
// Verify that cache hit rates are consistent across platforms
|
|
println!("\n=== Cache Economics Parity ===");
|
|
|
|
let mut cache_data = Vec::new();
|
|
|
|
for host in ["osa", "domedog"] {
|
|
if let Some(json) = read_manifest(host, "deepseek-cache-result") {
|
|
let hit_tokens = json["cache_hit_tokens"].as_u64().unwrap_or(0);
|
|
let total_tokens = json["warm_usage"]
|
|
.as_object()
|
|
.and_then(|u| u.get("prompt_tokens"))
|
|
.and_then(|t| t.as_u64())
|
|
.unwrap_or(0);
|
|
|
|
if total_tokens > 0 {
|
|
let hit_rate = (hit_tokens as f64 / total_tokens as f64) * 100.0;
|
|
cache_data.push((host, hit_tokens, total_tokens, hit_rate));
|
|
println!(
|
|
" {}: {} cache hit tokens / {} total ({:.1}%)",
|
|
host, hit_tokens, total_tokens, hit_rate
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Verify high cache hit rate (>95%)
|
|
if !cache_data.is_empty() {
|
|
let avg_hit_rate =
|
|
cache_data.iter().map(|(_, _, _, rate)| rate).sum::<f64>() / cache_data.len() as f64;
|
|
assert!(
|
|
avg_hit_rate > 95.0,
|
|
"Average cache hit rate should be >95%, got {:.1}%",
|
|
avg_hit_rate
|
|
);
|
|
println!("✅ Average cache hit rate: {:.1}%", avg_hit_rate);
|
|
}
|
|
}
|