Add browser clone hostd lifecycle ops
--- Build: pass | Tests: pass — 2395 passed (175 files)
This commit is contained in:
parent
6c549e7ad0
commit
6d662d5d3b
7 changed files with 607 additions and 22 deletions
|
|
@ -417,6 +417,7 @@ export default function registerHarness(pi: ExtensionAPI): void {
|
|||
"Execute a privileged hostd operation on the FreeBSD host. " +
|
||||
"Available ops: bastille-start, bastille-stop, bastille-restart, bastille-list, " +
|
||||
"zfs-snapshot, zfs-list, zfs-create, zfs-rollback, pf-reload, pf-enable, " +
|
||||
"browser-clone-create, browser-clone-destroy, browser-clone-reap, browser-clone-force-unmount, " +
|
||||
"service-start, service-stop, service-restart, service-status, " +
|
||||
"pkg-version, pkg-audit, freebsd-update-status, freebsd-version, " +
|
||||
"bastille-pkg-install, bastille-mount-pkg-cache.",
|
||||
|
|
|
|||
|
|
@ -175,11 +175,11 @@ require the full stack and should not be the first integration target.
|
|||
|
||||
### hostd / lifecycle
|
||||
|
||||
- [ ] Add narrow hostd ops for browser clone/create/destroy/reaper cleanup.
|
||||
- [ ] Use static PF table `browser_tasks` with add/delete per clone IP.
|
||||
- [ ] Implement forced-unmount fallback after busy dataset destroy failure.
|
||||
- [ ] Remove stale epairs before retrying a clone name.
|
||||
- [ ] Use rc.d/PID-file shutdown, not broad `pkill`.
|
||||
- [x] Add narrow hostd ops for browser clone/create/destroy/reaper cleanup.
|
||||
- [x] Use static PF table `browser_tasks` with add/delete per clone IP.
|
||||
- [x] Implement forced-unmount fallback after busy dataset destroy failure.
|
||||
- [x] Remove stale epairs before retrying a clone name.
|
||||
- [x] Use rc.d/PID-file shutdown, not broad `pkill`.
|
||||
|
||||
### Browser backend
|
||||
|
||||
|
|
|
|||
|
|
@ -420,26 +420,32 @@ Expected create path:
|
|||
|
||||
1. ensure `browser` template is stopped/quiescent,
|
||||
2. snapshot or clone from a known-good template state,
|
||||
3. clone dataset to `browsertaskNNN`,
|
||||
4. patch jail name/IP/VNET interface config,
|
||||
5. start jail,
|
||||
6. add IP to PF `browser_tasks`,
|
||||
7. start browser HTTP service / Chromium,
|
||||
8. inject cookies if `credential_mode=operator`,
|
||||
9. run task loop.
|
||||
3. call hostd `browser-clone-create` to clone datasets, patch jail name/IP/VNET config, clear stale epairs, and add the clone IP to PF `browser_tasks`,
|
||||
4. start jail,
|
||||
5. start browser HTTP service / Chromium,
|
||||
6. inject cookies if `credential_mode=operator`,
|
||||
7. run task loop.
|
||||
|
||||
Destroy/reaper order:
|
||||
|
||||
1. stop in-jail browser HTTP service,
|
||||
2. TERM Chromium by PID file,
|
||||
3. KILL by PID only as fallback,
|
||||
4. unmount nullfs/pkg-cache/session mounts,
|
||||
5. `bastille stop <clone>`,
|
||||
6. remove clone IP from PF table,
|
||||
7. release IP,
|
||||
8. `zfs destroy -r <clone_dataset>`,
|
||||
9. on busy dataset: `zfs unmount -f` and retry destroy,
|
||||
10. remove stale epairs before retrying a clone name.
|
||||
1. call hostd `browser-clone-destroy` / `browser-clone-reap`,
|
||||
2. stop in-jail browser HTTP service through rc.d,
|
||||
3. TERM Chromium by PID file,
|
||||
4. KILL by PID only as fallback,
|
||||
5. unmount nullfs/pkg-cache/session mounts,
|
||||
6. `bastille stop <clone>`,
|
||||
7. remove clone IP from PF table,
|
||||
8. release IP,
|
||||
9. `zfs destroy -r <clone_dataset>`,
|
||||
10. on busy dataset: `browser-clone-force-unmount` / `zfs unmount -f` and retry destroy,
|
||||
11. remove stale epairs before retrying a clone name.
|
||||
|
||||
Hostd operations added for Phase 1A:
|
||||
|
||||
- `browser-clone-create` — clone the thick `browser` datasets from a named snapshot, patch Bastille/VNET config, clear stale `btNNN` epairs, and add the task IP to PF table `browser_tasks`.
|
||||
- `browser-clone-destroy` — stop the in-jail browser service, perform PID-file-targeted shutdown, stop the jail, remove PF membership, clear epairs, and destroy clone datasets with forced-unmount retry.
|
||||
- `browser-clone-reap` — idempotent destroy path for orphaned clones.
|
||||
- `browser-clone-force-unmount` — narrow reaper fallback for busy clone datasets.
|
||||
|
||||
Broad `pkill chrome` is not production behavior. Use rc.d service stop or
|
||||
PID-file-targeted shutdown.
|
||||
|
|
|
|||
|
|
@ -244,4 +244,31 @@ describe('authorizeHostdOperation', () => {
|
|||
reason: 'platform-level operation requires operator approval',
|
||||
});
|
||||
});
|
||||
|
||||
it('treats browser clone lifecycle ops as operator-only platform operations', () => {
|
||||
expect(
|
||||
authorizeHostdOperation(
|
||||
'browser-clone-create',
|
||||
{
|
||||
clone: 'browsertask001',
|
||||
ip: '192.168.72.150',
|
||||
suffix: 'bt001',
|
||||
snapshot: 'phase06-injection-11.maj.2026-1603',
|
||||
},
|
||||
{ tenantId: 'alpha', caller: 'operator', registry },
|
||||
),
|
||||
).toEqual({ allowed: true, owner: 'shared-platform' });
|
||||
|
||||
expect(
|
||||
authorizeHostdOperation(
|
||||
'browser-clone-destroy',
|
||||
{ clone: 'browsertask001', ip: '192.168.72.150', suffix: 'bt001' },
|
||||
{ tenantId: 'alpha', caller: 'tenant-agent', registry },
|
||||
),
|
||||
).toEqual({
|
||||
allowed: false,
|
||||
owner: 'shared-platform',
|
||||
reason: 'platform-level operation requires operator approval',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -167,6 +167,10 @@ export function authorizeHostdOperation(
|
|||
});
|
||||
}
|
||||
|
||||
case 'browser-clone-create':
|
||||
case 'browser-clone-destroy':
|
||||
case 'browser-clone-reap':
|
||||
case 'browser-clone-force-unmount':
|
||||
case 'pf-reload':
|
||||
case 'pf-enable':
|
||||
case 'zpool-list':
|
||||
|
|
|
|||
|
|
@ -57,6 +57,10 @@ describe('handleOp — dispatch', () => {
|
|||
'bastille-cmd',
|
||||
'bastille-pkg-install',
|
||||
'bastille-mount-pkg-cache',
|
||||
'browser-clone-create',
|
||||
'browser-clone-destroy',
|
||||
'browser-clone-reap',
|
||||
'browser-clone-force-unmount',
|
||||
'zfs-snapshot',
|
||||
'zfs-list',
|
||||
'zfs-list-usedsnap',
|
||||
|
|
@ -93,6 +97,8 @@ describe('handleOp — dispatch', () => {
|
|||
expect(ops).toContain('bastille-start');
|
||||
expect(ops).toContain('zfs-snapshot');
|
||||
expect(ops).toContain('zpool-list');
|
||||
expect(ops).toContain('browser-clone-create');
|
||||
expect(ops).toContain('browser-clone-destroy');
|
||||
expect(ops).toContain('sanoid-snapshot');
|
||||
});
|
||||
|
||||
|
|
@ -157,6 +163,50 @@ describe('handleOp — jail name validation', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('handleOp — browser clone validation', () => {
|
||||
it.each([
|
||||
['wrong prefix', 'worker001'],
|
||||
['missing numeric suffix', 'browsertask'],
|
||||
['too many digits', 'browsertask0001'],
|
||||
['shell injection', 'browsertask001;rm'],
|
||||
['hyphen', 'browser-task001'],
|
||||
])('browser-clone-create rejects clone name: %s', (_, clone) => {
|
||||
const result = handleOp('browser-clone-create', {
|
||||
clone,
|
||||
ip: '192.168.72.150',
|
||||
suffix: 'bt001',
|
||||
snapshot: 'phase06-injection-11.maj.2026-1603',
|
||||
});
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.error).toMatch(/invalid params/);
|
||||
expect(mockSpawnSync).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('browser-clone-create rejects invalid VNET suffixes', () => {
|
||||
const result = handleOp('browser-clone-create', {
|
||||
clone: 'browsertask001',
|
||||
ip: '192.168.72.150',
|
||||
suffix: 'bt-001',
|
||||
snapshot: 'phase06-injection-11.maj.2026-1603',
|
||||
});
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.error).toMatch(/invalid params/);
|
||||
expect(mockSpawnSync).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('browser-clone-create rejects invalid IPs', () => {
|
||||
const result = handleOp('browser-clone-create', {
|
||||
clone: 'browsertask001',
|
||||
ip: '999.168.72.150',
|
||||
suffix: 'bt001',
|
||||
snapshot: 'phase06-injection-11.maj.2026-1603',
|
||||
});
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.error).toMatch(/invalid params/);
|
||||
expect(mockSpawnSync).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleOp — dataset name validation', () => {
|
||||
it.each([
|
||||
['shell injection', 'zroot/data; rm -rf /'],
|
||||
|
|
@ -357,6 +407,178 @@ describe('handleOp — spawnSync args (Bastille)', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('handleOp — spawnSync args (browser clone lifecycle)', () => {
|
||||
it('browser-clone-create clones both datasets, patches config, and adds PF membership', () => {
|
||||
const result = handleOp('browser-clone-create', {
|
||||
clone: 'browsertask001',
|
||||
ip: '192.168.72.150',
|
||||
suffix: 'bt001',
|
||||
snapshot: 'phase06-injection-11.maj.2026-1603',
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'ifconfig',
|
||||
['e0a_bt001', 'destroy'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
'ifconfig',
|
||||
['e0b_bt001', 'destroy'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
'zfs',
|
||||
[
|
||||
'clone',
|
||||
'-o',
|
||||
'mountpoint=/usr/local/bastille/jails/browsertask001',
|
||||
'zroot/clawdie-runtime/jails/browser@phase06-injection-11.maj.2026-1603',
|
||||
'zroot/clawdie-runtime/jails/browsertask001',
|
||||
],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
4,
|
||||
'zfs',
|
||||
[
|
||||
'clone',
|
||||
'-o',
|
||||
'mountpoint=/usr/local/bastille/jails/browsertask001/root',
|
||||
'zroot/clawdie-runtime/jails/browser/root@phase06-injection-11.maj.2026-1603',
|
||||
'zroot/clawdie-runtime/jails/browsertask001/root',
|
||||
],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync.mock.calls[4][0]).toBe(process.execPath);
|
||||
expect(mockSpawnSync.mock.calls[4][1]).toEqual([
|
||||
'-e',
|
||||
expect.any(String),
|
||||
'browser',
|
||||
'browsertask001',
|
||||
'bt001',
|
||||
'192.168.72.150',
|
||||
'/usr/local/bastille/jails',
|
||||
]);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
6,
|
||||
'pfctl',
|
||||
['-t', 'browser_tasks', '-T', 'add', '192.168.72.150'],
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it('browser-clone-destroy uses service/PID-file shutdown, PF delete, epair cleanup, and ZFS destroy', () => {
|
||||
const result = handleOp('browser-clone-destroy', {
|
||||
clone: 'browsertask001',
|
||||
ip: '192.168.72.150',
|
||||
suffix: 'bt001',
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'bastille',
|
||||
['cmd', 'browsertask001', 'service', 'clawdie_browser', 'stop'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
'bastille',
|
||||
['cmd', 'browsertask001', '/bin/sh', '-lc', expect.stringContaining('/var/run/clawdie-browser-chromium.pid')],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync.mock.calls[1][1][4]).not.toContain('pkill');
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
'bastille',
|
||||
['stop', 'browsertask001'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
4,
|
||||
'pfctl',
|
||||
['-t', 'browser_tasks', '-T', 'delete', '192.168.72.150'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
5,
|
||||
'ifconfig',
|
||||
['e0a_bt001', 'destroy'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
6,
|
||||
'ifconfig',
|
||||
['e0b_bt001', 'destroy'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
7,
|
||||
'zfs',
|
||||
['destroy', '-r', 'zroot/clawdie-runtime/jails/browsertask001'],
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it('browser-clone-force-unmount unmounts root before parent dataset', () => {
|
||||
const result = handleOp('browser-clone-force-unmount', {
|
||||
clone: 'browsertask001',
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
'zfs',
|
||||
['unmount', '-f', 'zroot/clawdie-runtime/jails/browsertask001/root'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
'zfs',
|
||||
['unmount', '-f', 'zroot/clawdie-runtime/jails/browsertask001'],
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it('browser-clone-destroy retries ZFS destroy after forced unmount on busy datasets', () => {
|
||||
mockSpawnSync
|
||||
.mockReturnValueOnce(spawnOk()) // service stop
|
||||
.mockReturnValueOnce(spawnOk()) // PID-file shutdown
|
||||
.mockReturnValueOnce(spawnOk()) // bastille stop
|
||||
.mockReturnValueOnce(spawnOk()) // pf delete
|
||||
.mockReturnValueOnce(spawnOk()) // e0a cleanup
|
||||
.mockReturnValueOnce(spawnOk()) // e0b cleanup
|
||||
.mockReturnValueOnce(spawnFail('dataset is busy'))
|
||||
.mockReturnValueOnce(spawnOk()) // root forced unmount
|
||||
.mockReturnValueOnce(spawnOk()) // parent forced unmount
|
||||
.mockReturnValueOnce(spawnOk()); // retry destroy
|
||||
|
||||
const result = handleOp('browser-clone-destroy', {
|
||||
clone: 'browsertask001',
|
||||
ip: '192.168.72.150',
|
||||
suffix: 'bt001',
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
8,
|
||||
'zfs',
|
||||
['unmount', '-f', 'zroot/clawdie-runtime/jails/browsertask001/root'],
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(mockSpawnSync).toHaveBeenNthCalledWith(
|
||||
10,
|
||||
'zfs',
|
||||
['destroy', '-r', 'zroot/clawdie-runtime/jails/browsertask001'],
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleOp — spawnSync args (ZFS)', () => {
|
||||
it('zfs-snapshot: combines dataset@name', () => {
|
||||
handleOp('zfs-snapshot', { dataset: 'zroot/clawdie', name: 'pre-update' });
|
||||
|
|
|
|||
|
|
@ -17,6 +17,19 @@ import {
|
|||
// ── Param validators ──────────────────────────────────────────────────────────
|
||||
|
||||
const jailName = z.string().regex(/^[a-zA-Z0-9_-]{1,32}$/, 'invalid jail name');
|
||||
const browserCloneName = z
|
||||
.string()
|
||||
.regex(/^browsertask[0-9]{3}$/, 'invalid browser clone name');
|
||||
const browserVnetSuffix = z
|
||||
.string()
|
||||
.regex(/^bt[0-9]{3}$/, 'invalid browser VNET suffix');
|
||||
const ipv4Address = z.string().refine((value) => {
|
||||
const parts = value.split('.');
|
||||
return (
|
||||
parts.length === 4 &&
|
||||
parts.every((part) => /^\d{1,3}$/.test(part) && Number(part) >= 0 && Number(part) <= 255)
|
||||
);
|
||||
}, 'invalid IPv4 address');
|
||||
const datasetName = z
|
||||
.string()
|
||||
.regex(/^[a-zA-Z0-9_\-/.]{1,128}$/, 'invalid dataset name');
|
||||
|
|
@ -147,6 +160,262 @@ function serviceMaybeStop(
|
|||
return exec('service', [name, 'stop'], opts);
|
||||
}
|
||||
|
||||
function browserJailDatasetBase(): string {
|
||||
const pool = (process.env.ZFS_POOL || 'zroot').trim() || 'zroot';
|
||||
const prefix = (process.env.ZFS_PREFIX || 'clawdie-runtime').trim() || 'clawdie-runtime';
|
||||
return `${pool}/${prefix}/jails`;
|
||||
}
|
||||
|
||||
const BROWSER_TEMPLATE_JAIL = 'browser';
|
||||
const BROWSER_TASKS_PF_TABLE = 'browser_tasks';
|
||||
const BASTILLE_JAILS_DIR = '/usr/local/bastille/jails';
|
||||
const BROWSER_SERVICE_NAME = 'clawdie_browser';
|
||||
const BROWSER_PID_SHUTDOWN_SCRIPT = [
|
||||
'for f in /var/run/clawdie-browser.pid /var/run/clawdie-browser-chromium.pid /var/run/phase06-chromium.pid /var/run/browser-cookie-server.pid; do',
|
||||
' [ -f "$f" ] || continue',
|
||||
' pid="$(cat "$f" 2>/dev/null || true)"',
|
||||
' case "$pid" in ""|*[!0-9]*) rm -f "$f"; continue ;; esac',
|
||||
' kill -TERM "$pid" 2>/dev/null || true',
|
||||
' sleep 1',
|
||||
' kill -0 "$pid" 2>/dev/null && kill -KILL "$pid" 2>/dev/null || true',
|
||||
' rm -f "$f"',
|
||||
'done',
|
||||
].join('\n');
|
||||
|
||||
function isIgnorableCleanupFailure(result: OpResult): boolean {
|
||||
const text = `${result.output || ''}\n${result.error || ''}`.toLowerCase();
|
||||
return (
|
||||
text.includes('does not exist') ||
|
||||
text.includes('not found') ||
|
||||
text.includes('no such file') ||
|
||||
text.includes('not running') ||
|
||||
text.includes('not currently mounted') ||
|
||||
text.includes('invalid argument')
|
||||
);
|
||||
}
|
||||
|
||||
function runCleanup(label: string, command: () => OpResult, lines: string[]): OpResult {
|
||||
const result = command();
|
||||
if (result.ok) {
|
||||
lines.push(`${label}: ok`);
|
||||
return result;
|
||||
}
|
||||
if (isIgnorableCleanupFailure(result)) {
|
||||
lines.push(`${label}: skipped (${result.output || result.error || 'not present'})`);
|
||||
return { ...result, ok: true };
|
||||
}
|
||||
lines.push(`${label}: failed (${result.output || result.error || 'unknown error'})`);
|
||||
return result;
|
||||
}
|
||||
|
||||
function cleanupBrowserEpair(suffix: string, lines: string[]): OpResult {
|
||||
let ok = true;
|
||||
for (const iface of [`e0a_${suffix}`, `e0b_${suffix}`]) {
|
||||
const result = exec('ifconfig', [iface, 'destroy']);
|
||||
if (result.ok || isIgnorableCleanupFailure(result)) {
|
||||
lines.push(`epair ${iface}: clear`);
|
||||
} else {
|
||||
ok = false;
|
||||
lines.push(`epair ${iface}: ${result.output || result.error || 'cleanup failed'}`);
|
||||
}
|
||||
}
|
||||
return ok
|
||||
? { ok: true, output: lines.join('\n') }
|
||||
: { ok: false, output: lines.join('\n'), error: 'stale epair cleanup failed' };
|
||||
}
|
||||
|
||||
function pfTable(action: 'add' | 'delete', ip: string): OpResult {
|
||||
return exec('pfctl', ['-t', BROWSER_TASKS_PF_TABLE, '-T', action, ip]);
|
||||
}
|
||||
|
||||
function forceUnmountBrowserCloneDataset(clone: string, lines: string[]): OpResult {
|
||||
const base = browserJailDatasetBase();
|
||||
const rootDataset = `${base}/${clone}/root`;
|
||||
const cloneDataset = `${base}/${clone}`;
|
||||
const root = runCleanup(
|
||||
`zfs unmount ${rootDataset}`,
|
||||
() => exec('zfs', ['unmount', '-f', rootDataset]),
|
||||
lines,
|
||||
);
|
||||
const parent = runCleanup(
|
||||
`zfs unmount ${cloneDataset}`,
|
||||
() => exec('zfs', ['unmount', '-f', cloneDataset]),
|
||||
lines,
|
||||
);
|
||||
return root.ok && parent.ok
|
||||
? { ok: true, output: lines.join('\n') }
|
||||
: { ok: false, output: lines.join('\n'), error: 'forced unmount failed' };
|
||||
}
|
||||
|
||||
function destroyBrowserCloneDataset(clone: string, lines: string[]): OpResult {
|
||||
const dataset = `${browserJailDatasetBase()}/${clone}`;
|
||||
const first = exec('zfs', ['destroy', '-r', dataset]);
|
||||
if (first.ok) {
|
||||
lines.push(`zfs destroy ${dataset}: ok`);
|
||||
return first;
|
||||
}
|
||||
if (isIgnorableCleanupFailure(first)) {
|
||||
lines.push(`zfs destroy ${dataset}: skipped (${first.output || first.error || 'not present'})`);
|
||||
return { ...first, ok: true };
|
||||
}
|
||||
|
||||
lines.push(`zfs destroy ${dataset}: retrying after forced unmount`);
|
||||
const unmount = forceUnmountBrowserCloneDataset(clone, lines);
|
||||
if (!unmount.ok) return unmount;
|
||||
const retry = exec('zfs', ['destroy', '-r', dataset]);
|
||||
if (retry.ok || isIgnorableCleanupFailure(retry)) {
|
||||
lines.push(`zfs destroy ${dataset}: ok after forced unmount`);
|
||||
return { ...retry, ok: true };
|
||||
}
|
||||
lines.push(`zfs destroy ${dataset}: failed (${retry.output || retry.error || 'unknown error'})`);
|
||||
return retry;
|
||||
}
|
||||
|
||||
function patchBrowserCloneConfig(
|
||||
template: string,
|
||||
clone: string,
|
||||
suffix: string,
|
||||
ip: string,
|
||||
): OpResult {
|
||||
const script = String.raw`
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const [template, clone, suffix, ip, jailsDir] = process.argv.slice(1);
|
||||
const base = path.join(jailsDir, clone);
|
||||
function replaceAll(text, from, to) {
|
||||
return text.split(from).join(to);
|
||||
}
|
||||
function patchFile(file, patches) {
|
||||
let text = fs.readFileSync(file, 'utf8');
|
||||
for (const [from, to] of patches) text = replaceAll(text, from, to);
|
||||
fs.writeFileSync(file, text);
|
||||
}
|
||||
patchFile(path.join(base, 'jail.conf'), [
|
||||
[template + ' {', clone + ' {'],
|
||||
['/var/log/bastille/' + template + '_console.log', '/var/log/bastille/' + clone + '_console.log'],
|
||||
['host.hostname = ' + template + ';', 'host.hostname = ' + clone + ';'],
|
||||
[jailsDir + '/' + template + '/fstab', jailsDir + '/' + clone + '/fstab'],
|
||||
[jailsDir + '/' + template + '/root', jailsDir + '/' + clone + '/root'],
|
||||
['e0b_' + template, 'e0b_' + suffix],
|
||||
['e0a_' + template, 'e0a_' + suffix],
|
||||
['jail ' + template, 'jail ' + clone],
|
||||
]);
|
||||
const fstab = path.join(base, 'fstab');
|
||||
if (fs.existsSync(fstab)) {
|
||||
patchFile(fstab, [[jailsDir + '/' + template + '/', jailsDir + '/' + clone + '/']]);
|
||||
}
|
||||
const rc = path.join(base, 'root/etc/rc.conf');
|
||||
let rcText = fs.readFileSync(rc, 'utf8');
|
||||
rcText = replaceAll(rcText, 'ifconfig_e0b_' + template + '_name', 'ifconfig_e0b_' + suffix + '_name');
|
||||
rcText = rcText.replace(/inet [0-9]{1,3}(?:\.[0-9]{1,3}){3}\/24/g, 'inet ' + ip + '/24');
|
||||
fs.writeFileSync(rc, rcText);
|
||||
`;
|
||||
return exec(process.execPath, ['-e', script, template, clone, suffix, ip, BASTILLE_JAILS_DIR]);
|
||||
}
|
||||
|
||||
function createBrowserClone(params: {
|
||||
clone: string;
|
||||
ip: string;
|
||||
suffix: string;
|
||||
snapshot: string;
|
||||
template?: string;
|
||||
}): OpResult {
|
||||
const template = params.template || BROWSER_TEMPLATE_JAIL;
|
||||
const base = browserJailDatasetBase();
|
||||
const lines: string[] = [];
|
||||
const epairCleanup = cleanupBrowserEpair(params.suffix, lines);
|
||||
if (!epairCleanup.ok) return epairCleanup;
|
||||
|
||||
const cloneDataset = `${base}/${params.clone}`;
|
||||
const rootDataset = `${base}/${params.clone}/root`;
|
||||
const templateDataset = `${base}/${template}`;
|
||||
const templateRootDataset = `${base}/${template}/root`;
|
||||
|
||||
const cloneParent = exec('zfs', [
|
||||
'clone',
|
||||
'-o',
|
||||
`mountpoint=${BASTILLE_JAILS_DIR}/${params.clone}`,
|
||||
`${templateDataset}@${params.snapshot}`,
|
||||
cloneDataset,
|
||||
]);
|
||||
if (!cloneParent.ok) return cloneParent;
|
||||
lines.push(`zfs clone ${cloneDataset}: ok`);
|
||||
|
||||
const cloneRoot = exec('zfs', [
|
||||
'clone',
|
||||
'-o',
|
||||
`mountpoint=${BASTILLE_JAILS_DIR}/${params.clone}/root`,
|
||||
`${templateRootDataset}@${params.snapshot}`,
|
||||
rootDataset,
|
||||
]);
|
||||
if (!cloneRoot.ok) {
|
||||
destroyBrowserCloneDataset(params.clone, lines);
|
||||
return { ...cloneRoot, output: [lines.join('\n'), cloneRoot.output].filter(Boolean).join('\n') };
|
||||
}
|
||||
lines.push(`zfs clone ${rootDataset}: ok`);
|
||||
|
||||
const patch = patchBrowserCloneConfig(template, params.clone, params.suffix, params.ip);
|
||||
if (!patch.ok) {
|
||||
destroyBrowserCloneDataset(params.clone, lines);
|
||||
return { ...patch, output: [lines.join('\n'), patch.output].filter(Boolean).join('\n') };
|
||||
}
|
||||
lines.push('clone config patch: ok');
|
||||
|
||||
const pfAdd = pfTable('add', params.ip);
|
||||
if (!pfAdd.ok) {
|
||||
destroyBrowserCloneDataset(params.clone, lines);
|
||||
return { ...pfAdd, output: [lines.join('\n'), pfAdd.output].filter(Boolean).join('\n') };
|
||||
}
|
||||
lines.push(`pf ${BROWSER_TASKS_PF_TABLE} add ${params.ip}: ok`);
|
||||
|
||||
return { ok: true, output: lines.join('\n') };
|
||||
}
|
||||
|
||||
function destroyBrowserClone(params: {
|
||||
clone: string;
|
||||
ip?: string;
|
||||
suffix: string;
|
||||
service?: string;
|
||||
}): OpResult {
|
||||
const lines: string[] = [];
|
||||
const service = params.service || BROWSER_SERVICE_NAME;
|
||||
|
||||
runCleanup(
|
||||
`service ${service} stop`,
|
||||
() => exec('bastille', ['cmd', params.clone, 'service', service, 'stop']),
|
||||
lines,
|
||||
);
|
||||
runCleanup(
|
||||
'PID-file shutdown',
|
||||
() => exec('bastille', ['cmd', params.clone, '/bin/sh', '-lc', BROWSER_PID_SHUTDOWN_SCRIPT]),
|
||||
lines,
|
||||
);
|
||||
runCleanup(
|
||||
`bastille stop ${params.clone}`,
|
||||
() => exec('bastille', ['stop', params.clone]),
|
||||
lines,
|
||||
);
|
||||
if (params.ip) {
|
||||
runCleanup(
|
||||
`pf ${BROWSER_TASKS_PF_TABLE} delete ${params.ip}`,
|
||||
() => pfTable('delete', params.ip as string),
|
||||
lines,
|
||||
);
|
||||
}
|
||||
const epairCleanup = cleanupBrowserEpair(params.suffix, lines);
|
||||
const destroyed = destroyBrowserCloneDataset(params.clone, lines);
|
||||
if (!epairCleanup.ok) {
|
||||
return {
|
||||
ok: false,
|
||||
output: lines.join('\n'),
|
||||
error: epairCleanup.error || 'stale epair cleanup failed',
|
||||
};
|
||||
}
|
||||
return destroyed.ok
|
||||
? { ok: true, output: lines.join('\n') }
|
||||
: { ok: false, output: lines.join('\n'), error: destroyed.error || 'destroy clone failed' };
|
||||
}
|
||||
|
||||
function runMaintenanceReboot(delayMinutes: number): OpResult {
|
||||
const timeoutMs = 30_000;
|
||||
|
||||
|
|
@ -294,6 +563,62 @@ const OPS: Record<string, OpEntry> = {
|
|||
},
|
||||
},
|
||||
|
||||
// ── Browser clone lifecycle ──────────────────────────────────────────────
|
||||
'browser-clone-create': {
|
||||
schema: z.object({
|
||||
clone: browserCloneName,
|
||||
ip: ipv4Address,
|
||||
suffix: browserVnetSuffix,
|
||||
snapshot: snapshotTag,
|
||||
template: z.literal(BROWSER_TEMPLATE_JAIL).optional(),
|
||||
}),
|
||||
handler: (p) =>
|
||||
createBrowserClone({
|
||||
clone: String(p.clone),
|
||||
ip: String(p.ip),
|
||||
suffix: String(p.suffix),
|
||||
snapshot: String(p.snapshot),
|
||||
template: typeof p.template === 'string' ? p.template : undefined,
|
||||
}),
|
||||
},
|
||||
'browser-clone-destroy': {
|
||||
schema: z.object({
|
||||
clone: browserCloneName,
|
||||
ip: ipv4Address.optional(),
|
||||
suffix: browserVnetSuffix,
|
||||
service: serviceName.optional(),
|
||||
}),
|
||||
handler: (p) =>
|
||||
destroyBrowserClone({
|
||||
clone: String(p.clone),
|
||||
ip: typeof p.ip === 'string' ? p.ip : undefined,
|
||||
suffix: String(p.suffix),
|
||||
service: typeof p.service === 'string' ? p.service : undefined,
|
||||
}),
|
||||
},
|
||||
'browser-clone-reap': {
|
||||
schema: z.object({
|
||||
clone: browserCloneName,
|
||||
ip: ipv4Address.optional(),
|
||||
suffix: browserVnetSuffix,
|
||||
service: serviceName.optional(),
|
||||
}),
|
||||
handler: (p) =>
|
||||
destroyBrowserClone({
|
||||
clone: String(p.clone),
|
||||
ip: typeof p.ip === 'string' ? p.ip : undefined,
|
||||
suffix: String(p.suffix),
|
||||
service: typeof p.service === 'string' ? p.service : undefined,
|
||||
}),
|
||||
},
|
||||
'browser-clone-force-unmount': {
|
||||
schema: z.object({ clone: browserCloneName }),
|
||||
handler: (p) => {
|
||||
const lines: string[] = [];
|
||||
return forceUnmountBrowserCloneDataset(String(p.clone), lines);
|
||||
},
|
||||
},
|
||||
|
||||
// ── ZFS ──────────────────────────────────────────────────────────────────
|
||||
'zfs-snapshot': {
|
||||
schema: z.object({ dataset: datasetName, name: snapshotTag }),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue