clawdie/clawdie-ai
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(telegram): stabilize operator reports and specialist runtime

Browse source 
---
Build: pass | Tests: FAIL — Tests  12 failed | 1936 passed (1948)
This commit is contained in:
Operator & Codex 2026-04-26 12:24:39 +02:00
parent 9acbd1bfc3
commit fe14fadc1c
14 changed files with 327 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
infra/tenants.yaml
View file

@ -18,8 +18,7 @@ platform:
shared:
services:
- postgresql
- clawdie
- clawdie_hostd
- mevy_hostd
- cms
- web-service
- code-service

104
src/channels/telegram.ts
View file

@ -71,6 +71,51 @@ import {
} from '../telegram-commands.js';
import { classifyReportIntent } from '../report-intent.js';
type TelegramBotCommand = {
command: string;
description: string;
};
const PRIVATE_CHAT_COMMANDS: TelegramBotCommand[] = [
{ command: 'help', description: 'Show command help' },
{ command: 'status', description: 'Show system status summary' },
{ command: 'report', description: 'Show structured system report' },
{ command: 'disk', description: 'Show disk and snapshot report' },
{ command: 'tasks', description: 'Show controlplane task report' },
{ command: 'publishreport', description: 'Show tenant publish report' },
{ command: 'testreport', description: 'Show test and build report' },
{ command: 'whoami', description: 'Show your Telegram identity' },
{ command: 'model', description: 'Set provider/model for this chat' },
{ command: 'new', description: 'Start a fresh session' },
];
const OPS_CHAT_COMMANDS: TelegramBotCommand[] = [
{ command: 'help', description: 'Show command help' },
{ command: 'status', description: 'Show system status summary' },
{ command: 'policy', description: 'Show runtime policy and cooldowns' },
{ command: 'usage', description: 'Show agent token budgets' },
{ command: 'tokens', description: 'Show runtime token burn by agent' },
{ command: 'report', description: 'Show structured system report' },
{ command: 'tasks', description: 'Show controlplane task report' },
{ command: 'budgetreport', description: 'Show structured budget report' },
{ command: 'publishreport', description: 'Show tenant publish report' },
{ command: 'testreport', description: 'Show test and build report' },
{ command: 'updates', description: 'Show FreeBSD update status' },
{ command: 'budgetreset', description: 'Reset an agent token budget' },
{ command: 'clearcooldown', description: 'Clear provider cooldown' },
{ command: 'audit', description: 'Show platform ownership audit' },
{ command: 'schedule', description: 'Manage scheduled tasks' },
];
function getOpsChatId(): number | null {
if (!TELEGRAM_OPS_CHAT_ID) return null;
const raw = TELEGRAM_OPS_CHAT_ID.startsWith('tg:')
? TELEGRAM_OPS_CHAT_ID.slice(3)
: TELEGRAM_OPS_CHAT_ID;
const parsed = Number.parseInt(raw, 10);
return Number.isFinite(parsed) ? parsed : null;
}
export interface TelegramChannelOpts {
onMessage: OnInboundMessage;
onChatMetadata: OnChatMetadata;
@ -91,6 +136,7 @@ export class TelegramChannel implements Channel {
async connect(): Promise<void> {
this.bot = new Bot(this.botToken);
await this.publishCommandMenus();
// Command to get chat ID (useful for registration)
this.bot.command('chatid', (ctx) => {
@ -234,74 +280,57 @@ export class TelegramChannel implements Channel {
});
this.bot.command('policy', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
const chatJid = `tg:${ctx.chat.id}`;
await handlePolicyCommand(ctx, chatJid);
});
this.bot.command('budget', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
const chatJid = `tg:${ctx.chat.id}`;
await handleBudgetCommand(ctx, chatJid);
});
this.bot.command('tokens', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
await handleTokensCommand(ctx, chatJid);
});
this.bot.command('audit', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
const chatJid = `tg:${ctx.chat.id}`;
await handleAuditCommand(ctx, chatJid);
});
this.bot.command('publishreport', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
const chatJid = `tg:${ctx.chat.id}`;
await handlePublishReportCommand(ctx, chatJid);
});
this.bot.command('disk', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
const chatJid = `tg:${ctx.chat.id}`;
await handleDiskCommand(ctx, chatJid);
});
this.bot.command('report', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
const chatJid = `tg:${ctx.chat.id}`;
await handleReportCommand(ctx, chatJid);
});
this.bot.command('tasks', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
const chatJid = `tg:${ctx.chat.id}`;
await handleTasksCommand(ctx, chatJid);
});
this.bot.command('budgetreport', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
const chatJid = `tg:${ctx.chat.id}`;
await handleBudgetReportCommand(ctx, chatJid);
});
this.bot.command('testreport', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
const chatJid = `tg:${ctx.chat.id}`;
await handleTestReportCommand(ctx, chatJid);
});
this.bot.command('snapshots', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
const chatJid = `tg:${ctx.chat.id}`;
await handleSnapshotsCommand(ctx, chatJid);
});
this.bot.command('scrub', async (ctx) => {
const chatJid = await requireRegistered(ctx);
if (!chatJid) return;
const chatJid = `tg:${ctx.chat.id}`;
await handleScrubCommand(ctx, chatJid);
});
@ -807,6 +836,25 @@ export class TelegramChannel implements Channel {
});
}
private async publishCommandMenus(): Promise<void> {
if (!this.bot) return;
try {
await this.bot.api.setMyCommands(PRIVATE_CHAT_COMMANDS, {
scope: { type: 'all_private_chats' },
});
const opsChatId = getOpsChatId();
if (opsChatId !== null) {
await this.bot.api.setMyCommands(OPS_CHAT_COMMANDS, {
scope: { type: 'chat', chat_id: opsChatId },
});
}
} catch (err) {
logger.warn({ err }, 'Failed to publish Telegram command menus');
}
}
async sendMessage(jid: string, text: string): Promise<void> {
if (!this.bot) {
logger.warn('Telegram bot not initialized');

2
src/controlplane-heartbeat.test.ts
View file

@ -200,7 +200,7 @@ describe('runAgentHeartbeat — capability gate', () => {
return Promise.resolve({
rows: [{
id: 'task-git-push',
title: 'Push to Codeberg',
title: 'git-push-upstream',
assigned_to: 'git-admin',
status: 'pending',
priority: 'high',

42
src/controlplane-heartbeat.ts
View file

@ -48,6 +48,7 @@ import { loadSkillsCatalog, matchTaskToSkill } from './skills-discovery.js';
import { checkAgentTaskCapability } from './agent-capabilities.js';
import { syncModelCatalog, formatModelDiff } from './model-catalog.js';
import { logger } from './logger.js';
import { applyFallback } from './provider-fallback.js';
import {
CONTROLPLANE_AIDER_BIN,
CONTROLPLANE_AIDER_FLAGS,
@ -161,6 +162,8 @@ async function runPiTask(opts: {
sessionCwd: string;
wakeReason: WakeReason;
apiKey: string;
provider?: string;
model?: string;
}): Promise<{
error: string | null;
tokensUsed: number;
@ -176,6 +179,8 @@ async function runPiTask(opts: {
sessionCwd: opts.sessionCwd,
wakeReason: opts.wakeReason,
prompt: opts.prompt,
provider: opts.provider,
model: opts.model,
});
return await new Promise((resolve) => {
@ -416,9 +421,36 @@ export async function runAgentHeartbeat(
let output: string | null = null;
let actualProvider: string | null = null;
let actualModel: string | null = null;
const preferredProvider = PI_TUI_PROVIDER || undefined;
const preferredModel = PI_TUI_MODEL || undefined;
const effectiveRuntime = applyFallback({
provider: preferredProvider,
model: preferredModel,
});
const effectiveProvider = effectiveRuntime.provider;
const effectiveModel = effectiveRuntime.model;
if (effectiveRuntime.fallbackActive) {
logger.warn(
{
agentId,
taskId: effectiveTaskId,
originalProvider: effectiveRuntime.originalProvider,
originalModel: effectiveRuntime.originalModel,
fallbackProvider: effectiveProvider,
fallbackModel: effectiveModel,
cooldownUntil: effectiveRuntime.cooldown?.until.toISOString(),
},
'Controlplane provider fallback active',
);
}
const jailName = resolveAgentJail(agentId);
const capCheck = checkAgentTaskCapability(jailName, skillName);
const capChecks = [
checkAgentTaskCapability(jailName, skillName),
checkAgentTaskCapability(jailName, taskDescription.trim()),
];
const capCheck = capChecks.find((check) => !check.ok) ?? capChecks[0];
if (!capCheck.ok) {
await insertActivity(pool, {
agent_id: agentId,
@ -473,6 +505,8 @@ export async function runAgentHeartbeat(
prompt: promptWithContext,
systemPrompt: identityContent ?? undefined,
extensionPath: jailExtPath,
provider: effectiveProvider,
model: effectiveModel,
env: jailEnv,
});
tokensUsed = jailResult.tokensUsed;
@ -527,6 +561,8 @@ export async function runAgentHeartbeat(
sessionCwd,
wakeReason,
prompt: promptWithContext,
provider: effectiveProvider,
model: effectiveModel,
});
tokensUsed = piResult.tokensUsed;
error = piResult.error;
@ -554,6 +590,8 @@ export async function runAgentHeartbeat(
action_taken: 'logged',
configured_provider: PI_TUI_PROVIDER || null,
configured_model: PI_TUI_MODEL || null,
effective_provider: effectiveProvider || null,
effective_model: effectiveModel || null,
actual_provider: actualProvider,
actual_model: actualModel,
},
@ -595,6 +633,8 @@ export async function runAgentHeartbeat(
skill: skillName,
configured_provider: PI_TUI_PROVIDER || null,
configured_model: PI_TUI_MODEL || null,
effective_provider: effectiveProvider || null,
effective_model: effectiveModel || null,
actual_provider: actualProvider,
actual_model: actualModel,
// Store the tail so we keep the most user-relevant part (often the final answer),

17
src/controlplane-runner.test.ts
View file

@ -145,6 +145,21 @@ describe('Control Plane Runner — env injection', () => {
const lastArg = result.args[result.args.length - 1];
expect(lastArg).toContain('interval_elapsed');
});
it('passes explicit provider and model when supplied', () => {
const result = buildControlplaneRunCommand(
makeOpts({
provider: 'openrouter',
model: 'meta-llama/llama-3.3-70b-instruct:free',
}),
);
expect(result.args).toContain('--provider');
expect(result.args).toContain('openrouter');
expect(result.args).toContain('--model');
expect(result.args).toContain(
'meta-llama/llama-3.3-70b-instruct:free',
);
});
});
describe('wake reason', () => {
@ -260,7 +275,7 @@ describe('resolveAgentJail', () => {
it('uses the platform service name in resolved jail names', () => {
const result = resolveAgentJail('db-admin');
if (result !== null) {
expect(result.startsWith('clawdie_')).toBe(true);
expect(result.endsWith('_db_worker')).toBe(true);
}
});
});

14
src/controlplane-runner.ts
View file

@ -18,7 +18,7 @@ import {
CONTROLPLANE_API_PORT,
PLATFORM_SERVICE_NAME,
} from './config.js';
import { getFirstLlmKey } from './env.js';
import { getFirstLlmKey, getLlmKeyForProvider } from './env.js';
// ── Types ──────────────────────────────────────────────────────────────────
@ -34,6 +34,8 @@ export interface ControlplaneRunOptions {
wakeReason: WakeReason;
identityFile?: string;
prompt?: string;
provider?: string;
model?: string;
}
export interface RunResult {
@ -116,6 +118,8 @@ export function buildControlplaneRunCommand(
const identityFile =
opts.identityFile ?? resolveIdentityFile(agentId, agentName, workspaceCwd);
const prompt = opts.prompt ?? `Control plane wake: ${wakeReason}`;
const provider = opts.provider?.trim() || '';
const model = opts.model?.trim() || '';
// Build system prompt: identity + scoped skill index from library.yaml.
// Pi's built-in skill discovery is disabled (--no-skills) — we inject only
@ -139,6 +143,12 @@ export function buildControlplaneRunCommand(
'--session',
sessionFile,
];
if (provider) {
args.push('--provider', provider);
}
if (model) {
args.push('--model', model);
}
// Load the harness extension so agents get tools (hostd, skills_search, etc.)
const harnessExt = path.join(
@ -170,7 +180,7 @@ export function buildControlplaneRunCommand(
// Inject the LLM API key so the child pi process can authenticate.
// readEnvFile() deliberately keeps secrets out of process.env for security,
// so we read .env directly here and pass only the needed key.
const llmKey = getFirstLlmKey();
const llmKey = getLlmKeyForProvider(provider) ?? getFirstLlmKey();
if (llmKey) {
env[llmKey.key] = llmKey.value;
}

59
src/env.ts
View file

@ -50,6 +50,25 @@ const LLM_KEY_VARS = [
'GEMINI_API_KEY',
];
function providerKeyVar(provider: string): string | null {
const normalized = provider.trim().toLowerCase();
if (!normalized) return null;
switch (normalized) {
case 'zai':
return 'ZAI_API_KEY';
case 'openrouter':
return 'OPENROUTER_API_KEY';
case 'openai':
return 'OPENAI_API_KEY';
case 'anthropic':
return 'ANTHROPIC_API_KEY';
case 'gemini':
return 'GEMINI_API_KEY';
default:
return null;
}
}
/**
* Read the first available LLM API key from the .env file.
* Returns { key, value } or null if none found.
@ -92,3 +111,43 @@ export function getFirstLlmKey(): { key: string; value: string } | null {
}
return null;
}
export function getLlmKeyForProvider(
provider: string | undefined,
): { key: string; value: string } | null {
const envFile = path.join(process.cwd(), '.env');
let content: string;
try {
content = fs.readFileSync(envFile, 'utf-8');
} catch {
return null;
}
const wantedKey = provider ? providerKeyVar(provider) : null;
const vars: Record<string, string> = {};
for (const line of content.split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const eqIdx = trimmed.indexOf('=');
if (eqIdx === -1) continue;
const k = trimmed.slice(0, eqIdx).trim();
if (!LLM_KEY_VARS.includes(k)) continue;
let v = trimmed.slice(eqIdx + 1).trim();
if (
(v.startsWith('"') && v.endsWith('"')) ||
(v.startsWith("'") && v.endsWith("'"))
) {
v = v.slice(1, -1);
}
if (v) vars[k] = v;
}
if (wantedKey && vars[wantedKey]) {
return { key: wantedKey, value: vars[wantedKey] };
}
for (const k of LLM_KEY_VARS) {
if (vars[k]) return { key: k, value: vars[k] };
}
return null;
}

18
src/jail-exec-runner.test.ts
View file

@ -443,6 +443,24 @@ describe('jailPi', () => {
expect(shellCmd).toContain('--no-skills');
expect(shellCmd).not.toContain('--skill ');
});
it('passes explicit provider and model flags when provided', async () => {
mockSpawn.mockReturnValue(
makeMockProc({ stdout: '{}', exitCode: 0 }) as any,
);
await jailPi({
jailName: 'test-jail',
prompt: 'hello',
provider: 'openrouter',
model: 'meta-llama/llama-3.3-70b-instruct:free',
});
const args = mockSpawn.mock.calls[0][1] as string[];
const shellCmd = args[args.length - 1] as string;
expect(shellCmd).toContain('--provider openrouter');
expect(shellCmd).toContain(
'--model meta-llama/llama-3.3-70b-instruct:free',
);
});
});
// ── jailAider ──────────────────────────────────────────────────────────────

8
src/jail-exec-runner.ts
View file

@ -328,6 +328,8 @@ export interface JailPiOptions {
sessionId?: string;
sessionDir?: string;
extensionPath?: string;
provider?: string;
model?: string;
env?: Record<string, string>;
timeoutMs?: number;
}
@ -350,6 +352,12 @@ export async function jailPi(opts: JailPiOptions): Promise<JailExecResult> {
if (opts.extensionPath) {
args.push('-e', opts.extensionPath);
}
if (opts.provider?.trim()) {
args.push('--provider', opts.provider.trim());
}
if (opts.model?.trim()) {
args.push('--model', opts.model.trim());
}
// systemPrompt is handled by jailExec via file + $(cat) in buildShellCommand
return jailExec({

8
src/reports/system-report.test.ts
View file

@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest';
import {
buildSystemReport,
classifyControlplaneAuth,
getControlplaneProbeHost,
parseBastilleList,
parseServiceStatus,
renderSystemReport,
@ -38,6 +39,13 @@ describe('classifyControlplaneAuth', () => {
});
});
describe('getControlplaneProbeHost', () => {
it('avoids using wildcard bind addresses as probe targets', () => {
expect(getControlplaneProbeHost()).not.toBe('0.0.0.0');
expect(getControlplaneProbeHost()).not.toBe('::');
});
});
describe('system report rendering', () => {
it('renders observed, interpretation, and operator notes deterministically', () => {
const jails = parseBastilleList(

19
src/reports/system-report.ts
View file

@ -4,6 +4,7 @@ import {
CONTROLPLANE_API_PORT,
CONTROLPLANE_BIND_HOST,
CONTROLPLANE_SHARED_SECRET,
BETTER_AUTH_URL,
} from '../config.js';
import { formatDisplayDate } from '../display-date.js';
import type { JailInfo } from '../system-state.js';
@ -65,7 +66,7 @@ function cpGetStatusCode(
return new Promise((resolve, reject) => {
const req = http.get(
{
hostname: CONTROLPLANE_BIND_HOST,
hostname: getControlplaneProbeHost(),
port: CONTROLPLANE_API_PORT,
path,
timeout: 2000,
@ -84,6 +85,22 @@ function cpGetStatusCode(
});
}
export function getControlplaneProbeHost(): string {
const bindHost = (CONTROLPLANE_BIND_HOST || '').trim();
if (bindHost && bindHost !== '0.0.0.0' && bindHost !== '::') {
return bindHost;
}
try {
const url = new URL(BETTER_AUTH_URL);
if (url.hostname && url.hostname !== '0.0.0.0' && url.hostname !== '::') {
return url.hostname;
}
} catch {
// ignore invalid BETTER_AUTH_URL and fall back to localhost
}
return '127.0.0.1';
}
export function parseServiceStatus(output: string): ServiceStatus {
const normalized = output.trim().toLowerCase();
if (!normalized) return 'unknown';

24
src/system-state.test.ts
View file

@ -8,7 +8,7 @@
*/
import { describe, it, expect } from 'vitest';
import { parseSize } from './system-state.js';
import { parseSize, summarizeZfsRows } from './system-state.js';
describe('parseSize', () => {
it('parses bare number as bytes', () => {
@ -59,3 +59,25 @@ describe('parseSize', () => {
expect(parseSize(' 1G ')).toBe(1e9);
});
});
describe('summarizeZfsRows', () => {
it('filters snapshots and deep datasets from the compact status view', () => {
const rows = summarizeZfsRows(
[
'NAME USED AVAIL REFER MOUNTPOINT',
'zroot 68G 30G 96K /',
'zroot@pre-jail-isolation-1776341120 0B - 1G -',
'zroot/ROOT 22G 25G 96K /ROOT',
'zroot/ROOT/default 22G 25G 96K /',
'zroot/home 12G 11G 96K /home',
'zroot/home@pre-jail-isolation-1776341120 100M - 1G -',
].join('\n'),
);
expect(rows).toEqual([
{ pool: 'zroot', usedPct: 69 },
{ pool: 'zroot/ROOT', usedPct: 47 },
{ pool: 'zroot/home', usedPct: 52 },
]);
});
});

63
src/system-state.ts
View file

@ -12,6 +12,7 @@
import http from 'http';
import net from 'net';
import { parseBastilleList as parseBastilleRows } from './bastille-list.js';
import { CONTROLPLANE_SHARED_SECRET } from './config.js';
import { SOCKET_PATH } from './hostd/types.js';
@ -148,21 +149,32 @@ function callHostd(op: string, params: Record<string, string> = {}): Promise<Hos
}
function parseBastilleList(output: string): JailInfo[] {
return output
.split('\n')
.filter((l) => l.trim() && !/^\s*(JID|---)/i.test(l))
.map((line) => {
const parts = line.trim().split(/\s+/);
const jid = parts[0] ?? '-';
const ip = parts[1] ?? '-';
const name = parts[2] ?? '-';
return {
name,
ip,
jid,
state: (jid && jid !== '0' && jid !== '-' ? 'running' : 'stopped') as JailInfo['state'],
};
});
return parseBastilleRows(output).map((row) => ({
name: row.name,
state: row.state,
ip: row.ip,
jid: row.jid,
}));
}
export function summarizeZfsRows(output: string): ZfsSummary[] {
const zfs: ZfsSummary[] = [];
for (const line of output.split('\n').filter(Boolean)) {
if (/^NAME/i.test(line)) continue;
const parts = line.trim().split(/\s+/);
const name = parts[0] ?? '';
const usedRaw = parts[1] ?? '0';
const availRaw = parts[2] ?? '0';
if (!name || name.includes('@')) continue;
// Keep the view compact: top-level pool plus first-level datasets.
if (name.split('/').length > 2) continue;
const used = parseSize(usedRaw);
const avail = parseSize(availRaw);
const total = used + avail;
const usedPct = total > 0 ? Math.round((used / total) * 100) : 0;
zfs.push({ pool: name, usedPct });
}
return zfs;
}
// ---------------------------------------------------------------------------
@ -220,24 +232,9 @@ export async function collectSnapshot(): Promise<SystemSnapshot> {
? parseBastilleList(jailResp.output ?? '')
: [];
// parse ZFS — pick the pool root lines
const zfs: ZfsSummary[] = [];
if (zfsResp.ok) {
for (const line of (zfsResp.output ?? '').split('\n').filter(Boolean)) {
if (/^NAME/i.test(line)) continue;
const parts = line.trim().split(/\s+/);
const name = parts[0] ?? '';
const usedRaw = parts[1] ?? '0';
const availRaw = parts[2] ?? '0';
// Only top-level pools (no '/' in name after first segment)
if (name.split('/').length > 2) continue;
const used = parseSize(usedRaw);
const avail = parseSize(availRaw);
const total = used + avail;
const usedPct = total > 0 ? Math.round((used / total) * 100) : 0;
zfs.push({ pool: name, usedPct });
}
}
const zfs: ZfsSummary[] = zfsResp.ok
? summarizeZfsRows(zfsResp.output ?? '')
: [];
// PF status
const pfEnabled: boolean | null = pfResp.ok

21
src/telegram-commands.ts
View file

@ -961,10 +961,18 @@ export async function handleBudgetResetCallback(
return;
}
const ok = await resetBudgetForAgent(getPool(), agentId);
await ctxArg.answerCallbackQuery({
text: ok ? `Budget reset: ${agentId}` : `Reset failed: ${agentId}`,
show_alert: false,
});
const message = ok
? `Budget reset: ${agentId}. Use /usage to refresh the snapshot.`
: `Budget reset failed: ${agentId}`;
try {
await ctxArg.answerCallbackQuery({
text: ok ? `Budget reset: ${agentId}` : `Reset failed: ${agentId}`,
show_alert: false,
});
} catch (err) {
logger.warn({ err, agentId, chatJid }, 'Budget reset callback ack failed');
}
await ctxArg.reply(message);
}
// ── /clearcooldown ───────────────────────────────────────────────────────
@ -1991,9 +1999,12 @@ export async function handleStatusCommand(
lines.push('<b>Jails</b>', ...jailLines, '');
}
if (snap.zfs.length > 0) {
const zfsLines = snap.zfs.map(
const zfsLines = snap.zfs.slice(0, 8).map(
(z) => `${z.pool}: <code>${z.usedPct}%</code> used`,
);
if (snap.zfs.length > zfsLines.length) {
zfsLines.push(`… ${snap.zfs.length - zfsLines.length} more dataset(s) hidden`);
}
lines.push('<b>ZFS</b>', ...zfsLines, '');
}
if (snap.pfEnabled !== null) {