385 lines
15 KiB
TypeScript
385 lines
15 KiB
TypeScript
/**
|
|
* setup/db.ts — Provision the PostgreSQL db jail (skills + memory).
|
|
*
|
|
* Creates a thin Bastille jail at `${SUBNET_BASE}.3`, installs PostgreSQL 17 +
|
|
* pgvector, enables the service, then creates the split-brain databases and
|
|
* applies the shipped schema SQL.
|
|
*/
|
|
import { execSync, spawnSync } from 'child_process';
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
|
|
import { AGENT_NAME, DB_HOST, GIT_JAIL_IP, SUBNET_BASE } from '../src/config.js';
|
|
import { readEnvFile } from '../src/env.js';
|
|
import { logger } from '../src/logger.js';
|
|
import { ensureSplitBrainSecrets } from './secrets.js';
|
|
import { loadPackageList, mountPkgCacheInJail } from './packages.js';
|
|
import { commandExists, getPlatform, isRoot } from './platform.js';
|
|
import { emitStatus } from './status.js';
|
|
import { maybeEnableTailscaleInJail } from './tailscale.js';
|
|
|
|
const LOG = 'logs/setup.log';
|
|
|
|
function bastille(...args: string[]): { ok: boolean; output: string } {
|
|
const result = spawnSync('bastille', args, { encoding: 'utf-8', env: process.env });
|
|
const output = [result.stdout || '', result.stderr || ''].filter(Boolean).join('\n').trim();
|
|
return { ok: (result.status ?? 1) === 0, output };
|
|
}
|
|
|
|
function detectFreeBSDRelease(): string {
|
|
const output = execSync('freebsd-version -u', { encoding: 'utf-8' }).trim();
|
|
const match = output.match(/^(\d+\.\d+-\w+)/);
|
|
return match?.[1] ?? output;
|
|
}
|
|
|
|
function jailExists(name: string): boolean {
|
|
const { output } = bastille('list');
|
|
return output.split('\n').some((line) => {
|
|
const trimmed = line.trim();
|
|
if (!trimmed || trimmed.startsWith('JID')) return false;
|
|
const cols = trimmed.split(/\s+/u);
|
|
return cols.length > 1 && cols[1] === name;
|
|
});
|
|
}
|
|
|
|
function jailRoot(jailName: string): string {
|
|
return `/usr/local/bastille/jails/${jailName}/root`;
|
|
}
|
|
|
|
function ensureLine(filePath: string, line: string): void {
|
|
const content = fs.existsSync(filePath) ? fs.readFileSync(filePath, 'utf-8') : '';
|
|
if (!content.includes(line)) {
|
|
fs.appendFileSync(filePath, (content.endsWith('\n') || !content ? '' : '\n') + line + '\n');
|
|
}
|
|
}
|
|
|
|
function setOrAppendConfLine(filePath: string, key: string, value: string): void {
|
|
const content = fs.existsSync(filePath) ? fs.readFileSync(filePath, 'utf-8') : '';
|
|
const re = new RegExp(`^#?\\s*${key}\\s*=.*$`, 'm');
|
|
const nextLine = `${key} = '${value}'`;
|
|
const next = re.test(content) ? content.replace(re, nextLine) : `${content.trimEnd()}\n${nextLine}\n`;
|
|
fs.writeFileSync(filePath, next);
|
|
}
|
|
|
|
function runPsql(jailName: string, sql: string, database = 'postgres'): void {
|
|
const projectRoot = process.cwd();
|
|
const tmpDir = path.join(projectRoot, 'tmp');
|
|
fs.mkdirSync(tmpDir, { recursive: true });
|
|
|
|
const stamp = Date.now();
|
|
const hostTmp = path.join(tmpDir, `db-sql-${stamp}.sql`);
|
|
fs.writeFileSync(hostTmp, sql);
|
|
|
|
const jailTmpRel = path.join('var', 'tmp', `db-sql-${stamp}.sql`);
|
|
const jailTmpHostPath = path.join(jailRoot(jailName), jailTmpRel);
|
|
fs.mkdirSync(path.dirname(jailTmpHostPath), { recursive: true });
|
|
fs.copyFileSync(hostTmp, jailTmpHostPath);
|
|
|
|
try {
|
|
runPsqlFile(jailName, database, `/${jailTmpRel}`);
|
|
} finally {
|
|
fs.rmSync(hostTmp, { force: true });
|
|
fs.rmSync(jailTmpHostPath, { force: true });
|
|
}
|
|
}
|
|
|
|
function runPsqlFile(jailName: string, database: string, jailFilePath: string): void {
|
|
const res = bastille(
|
|
'cmd',
|
|
jailName,
|
|
'su',
|
|
'-m',
|
|
'postgres',
|
|
'-c',
|
|
`psql -v ON_ERROR_STOP=1 -d ${JSON.stringify(database)} -f ${JSON.stringify(jailFilePath)}`,
|
|
);
|
|
if (!res.ok) {
|
|
throw new Error(`psql -f failed: ${res.output}`);
|
|
}
|
|
}
|
|
|
|
function ensureDatabase(jailName: string, database: string, owner: string): void {
|
|
const query = `SELECT 1 FROM pg_database WHERE datname='${database}'`;
|
|
const check = bastille(
|
|
'cmd',
|
|
jailName,
|
|
'su',
|
|
'-m',
|
|
'postgres',
|
|
'-c',
|
|
`psql -tAc ${JSON.stringify(query)}`,
|
|
);
|
|
if (!check.ok) {
|
|
throw new Error(`psql failed: ${check.output}`);
|
|
}
|
|
const lines = check.output
|
|
.split('\n')
|
|
.map((line) => line.trim())
|
|
.filter((line) => line && !line.startsWith('['));
|
|
if (lines.includes('1')) return;
|
|
|
|
runPsql(jailName, `CREATE DATABASE ${database} OWNER ${owner};`);
|
|
}
|
|
|
|
export async function run(_args: string[]): Promise<void> {
|
|
const projectRoot = process.cwd();
|
|
const envOverrides = readEnvFile(['DB_JAIL_NAME']);
|
|
|
|
if (getPlatform() !== 'freebsd') {
|
|
emitStatus('SETUP_DB', { STATUS: 'failed', ERROR: 'unsupported_platform', LOG });
|
|
process.exit(1);
|
|
}
|
|
if (!isRoot()) {
|
|
emitStatus('SETUP_DB', { STATUS: 'failed', ERROR: 'requires_root', LOG });
|
|
throw new Error('setup_db_requires_root');
|
|
}
|
|
if (!commandExists('bastille')) {
|
|
emitStatus('SETUP_DB', { STATUS: 'failed', ERROR: 'missing_bastille', LOG });
|
|
throw new Error('missing_bastille');
|
|
}
|
|
|
|
const explicitJailName = (
|
|
process.env.DB_JAIL_NAME || envOverrides.DB_JAIL_NAME || ''
|
|
).trim();
|
|
const safeAgentName = AGENT_NAME.replace(/[-_]/g, '');
|
|
const preferredJailName = `${safeAgentName}db`;
|
|
const legacyHyphenName = `${AGENT_NAME}-db`;
|
|
const legacyJailName = 'db';
|
|
let jailName = explicitJailName || preferredJailName;
|
|
if (!explicitJailName) {
|
|
if (jailExists(legacyHyphenName) && !jailExists(preferredJailName)) {
|
|
jailName = legacyHyphenName;
|
|
} else if (jailExists(legacyJailName) && !jailExists(preferredJailName)) {
|
|
jailName = legacyJailName;
|
|
}
|
|
}
|
|
const dbIp = process.env.WARDEN_DB_IP || DB_HOST || `${SUBNET_BASE}.3`;
|
|
const hostname = `db.${AGENT_NAME}.home.arpa`;
|
|
const gateway = process.env.WARDEN_GATEWAY || `${SUBNET_BASE}.1`;
|
|
const bridge = process.env.WARDEN_BRIDGE || 'warden0';
|
|
const release = detectFreeBSDRelease();
|
|
|
|
// Ensure secrets + derived DB identifiers exist in .env
|
|
const secrets = ensureSplitBrainSecrets(projectRoot);
|
|
|
|
const postgresConf = path.join(jailRoot(jailName), 'var/db/postgres/data17/postgresql.conf');
|
|
const pgHbaConf = path.join(jailRoot(jailName), 'var/db/postgres/data17/pg_hba.conf');
|
|
const runBastille = (args: string[]) => bastille(...args);
|
|
|
|
try {
|
|
// ── Create jail ─────────────────────────────────────────────────────────
|
|
const exists = jailExists(jailName);
|
|
if (!exists) {
|
|
logger.info({ jailName, ip: dbIp, release }, 'Creating db jail');
|
|
const create = bastille(
|
|
'create',
|
|
'-T',
|
|
'-B',
|
|
'-g',
|
|
gateway,
|
|
jailName,
|
|
release,
|
|
`${dbIp}/24`,
|
|
bridge,
|
|
);
|
|
if (!create.ok) {
|
|
throw new Error(`bastille create failed: ${create.output}`);
|
|
}
|
|
bastille('config', jailName, 'set', 'host.hostname', hostname);
|
|
bastille('config', jailName, 'set', 'allow.sysvipc', '1');
|
|
bastille('restart', jailName);
|
|
} else {
|
|
logger.info({ jailName }, 'db jail already exists, skipping creation');
|
|
}
|
|
|
|
if (exists) {
|
|
bastille('config', jailName, 'set', 'allow.sysvipc', '1');
|
|
bastille('restart', jailName);
|
|
}
|
|
|
|
// ── Packages ────────────────────────────────────────────────────────────
|
|
mountPkgCacheInJail(jailName);
|
|
const packages = loadPackageList('db-jail.txt');
|
|
const pkg = bastille('pkg', jailName, 'install', '-y', ...packages);
|
|
if (!pkg.ok) {
|
|
logger.warn({ output: pkg.output }, 'db jail package install had warnings');
|
|
}
|
|
|
|
maybeEnableTailscaleInJail(runBastille, jailName, jailName);
|
|
|
|
// ── Postgres init + start ───────────────────────────────────────────────
|
|
bastille('cmd', jailName, 'sysrc', 'postgresql_enable=YES');
|
|
|
|
const dataDir = path.join(jailRoot(jailName), 'var/db/postgres/data17');
|
|
const pgVersion = path.join(dataDir, 'PG_VERSION');
|
|
if (!fs.existsSync(pgVersion)) {
|
|
logger.info({ jailName }, 'Initializing PostgreSQL data directory');
|
|
const init = bastille('cmd', jailName, 'service', 'postgresql', 'initdb');
|
|
if (!init.ok) throw new Error(`postgresql initdb failed: ${init.output}`);
|
|
}
|
|
|
|
// Ensure the db listens on loopback + jail IP for host access.
|
|
if (fs.existsSync(postgresConf)) {
|
|
setOrAppendConfLine(postgresConf, 'listen_addresses', `127.0.0.1,${dbIp}`);
|
|
}
|
|
|
|
// Allow host (warden gateway) to connect to both DBs.
|
|
// Keep this minimal: only these users/databases, scram-sha-256.
|
|
if (fs.existsSync(pgHbaConf)) {
|
|
ensureLine(
|
|
pgHbaConf,
|
|
`host ${secrets.skillsDbName} ${secrets.skillsDbUser} ${gateway}/32 scram-sha-256`,
|
|
);
|
|
ensureLine(
|
|
pgHbaConf,
|
|
`host ${secrets.memoryDbName} ${secrets.memoryDbUser} ${gateway}/32 scram-sha-256`,
|
|
);
|
|
ensureLine(
|
|
pgHbaConf,
|
|
`host ${secrets.forgejoDbName} ${secrets.forgejoDbUser} ${GIT_JAIL_IP}/32 scram-sha-256`,
|
|
);
|
|
ensureLine(
|
|
pgHbaConf,
|
|
`host strapi_cms strapi_cms ${process.env.WARDEN_CMS_IP || `${SUBNET_BASE}.4`}/32 scram-sha-256`,
|
|
);
|
|
}
|
|
|
|
const status = bastille('cmd', jailName, 'service', 'postgresql', 'onestatus');
|
|
const action = status.ok ? 'restart' : 'start';
|
|
const start = bastille('cmd', jailName, 'service', 'postgresql', action);
|
|
if (!start.ok) throw new Error(`postgresql start failed: ${start.output}`);
|
|
|
|
// ── Roles + DBs ─────────────────────────────────────────────────────────
|
|
runPsql(jailName, `ALTER USER postgres WITH PASSWORD '${secrets.postgresAdminPassword}';`);
|
|
|
|
runPsql(
|
|
jailName,
|
|
`DO $$ BEGIN
|
|
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${secrets.skillsDbUser}') THEN
|
|
CREATE ROLE ${secrets.skillsDbUser} WITH LOGIN PASSWORD '${secrets.skillsDbPassword}';
|
|
END IF;
|
|
END $$;`,
|
|
);
|
|
runPsql(
|
|
jailName,
|
|
`DO $$ BEGIN
|
|
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${secrets.memoryDbUser}') THEN
|
|
CREATE ROLE ${secrets.memoryDbUser} WITH LOGIN PASSWORD '${secrets.memoryDbPassword}';
|
|
END IF;
|
|
END $$;`,
|
|
);
|
|
runPsql(
|
|
jailName,
|
|
`DO $$ BEGIN
|
|
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${secrets.forgejoDbUser}') THEN
|
|
CREATE ROLE ${secrets.forgejoDbUser} WITH LOGIN PASSWORD '${secrets.forgejoDbPassword}';
|
|
END IF;
|
|
END $$;`,
|
|
);
|
|
runPsql(
|
|
jailName,
|
|
`DO $$ BEGIN
|
|
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'strapi_cms') THEN
|
|
CREATE ROLE strapi_cms WITH LOGIN PASSWORD '${secrets.strapiDbPassword}';
|
|
END IF;
|
|
END $$;`,
|
|
);
|
|
|
|
ensureDatabase(jailName, secrets.skillsDbName, secrets.skillsDbUser);
|
|
ensureDatabase(jailName, secrets.memoryDbName, secrets.memoryDbUser);
|
|
ensureDatabase(jailName, secrets.forgejoDbName, secrets.forgejoDbUser);
|
|
ensureDatabase(jailName, 'strapi_cms', 'strapi_cms');
|
|
|
|
// ── Extensions ──────────────────────────────────────────────────────────
|
|
for (const db of [secrets.skillsDbName, secrets.memoryDbName]) {
|
|
runPsql(jailName, 'CREATE EXTENSION IF NOT EXISTS pgcrypto;', db);
|
|
runPsql(jailName, 'CREATE EXTENSION IF NOT EXISTS vector;', db);
|
|
runPsql(jailName, 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";', db);
|
|
}
|
|
|
|
// ── Schema SQL ──────────────────────────────────────────────────────────
|
|
const sqlDirInJail = '/var/db/postgres/clawdie-sql';
|
|
bastille(
|
|
'cmd',
|
|
jailName,
|
|
'install',
|
|
'-d',
|
|
'-m',
|
|
'755',
|
|
'-o',
|
|
'postgres',
|
|
'-g',
|
|
'postgres',
|
|
sqlDirInJail,
|
|
);
|
|
|
|
const schemas: Array<{ src: string; dest: string; db: string }> = [
|
|
{
|
|
src: path.join(projectRoot, 'docs', 'internal', 'sql', 'builtin-knowledge-base.sql'),
|
|
dest: `${sqlDirInJail}/builtin-knowledge-base.sql`,
|
|
db: secrets.skillsDbName,
|
|
},
|
|
{
|
|
src: path.join(projectRoot, 'docs', 'internal', 'sql', 'ai-brain-base.sql'),
|
|
dest: `${sqlDirInJail}/ai-brain-base.sql`,
|
|
db: secrets.memoryDbName,
|
|
},
|
|
{
|
|
src: path.join(projectRoot, 'docs', 'internal', 'sql', 'clawdie-brain-hybrid-upgrade.sql'),
|
|
dest: `${sqlDirInJail}/clawdie-brain-hybrid-upgrade.sql`,
|
|
db: secrets.memoryDbName,
|
|
},
|
|
{
|
|
src: path.join(projectRoot, 'docs', 'internal', 'sql', 'search-memories.sql'),
|
|
dest: `${sqlDirInJail}/search-memories.sql`,
|
|
db: secrets.memoryDbName,
|
|
},
|
|
];
|
|
|
|
for (const item of schemas) {
|
|
if (!fs.existsSync(item.src)) {
|
|
throw new Error(`missing_schema_file: ${item.src}`);
|
|
}
|
|
const hostDest = path.join(jailRoot(jailName), item.dest.replace(/^\//, ''));
|
|
fs.mkdirSync(path.dirname(hostDest), { recursive: true });
|
|
fs.copyFileSync(item.src, hostDest);
|
|
runPsqlFile(jailName, item.db, item.dest);
|
|
}
|
|
|
|
runPsql(
|
|
jailName,
|
|
[
|
|
`GRANT USAGE ON SCHEMA public TO ${secrets.skillsDbUser};`,
|
|
`GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${secrets.skillsDbUser};`,
|
|
`GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO ${secrets.skillsDbUser};`,
|
|
`ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${secrets.skillsDbUser};`,
|
|
`ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO ${secrets.skillsDbUser};`,
|
|
].join(' '),
|
|
secrets.skillsDbName,
|
|
);
|
|
runPsql(
|
|
jailName,
|
|
[
|
|
`GRANT USAGE ON SCHEMA public TO ${secrets.memoryDbUser};`,
|
|
`GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO ${secrets.memoryDbUser};`,
|
|
`GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ${secrets.memoryDbUser};`,
|
|
`ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ${secrets.memoryDbUser};`,
|
|
`ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO ${secrets.memoryDbUser};`,
|
|
].join(' '),
|
|
secrets.memoryDbName,
|
|
);
|
|
|
|
emitStatus('SETUP_DB', {
|
|
STATUS: 'success',
|
|
JAIL_NAME: jailName,
|
|
JAIL_IP: dbIp,
|
|
SKILLS_DB: secrets.skillsDbName,
|
|
MEMORY_DB: secrets.memoryDbName,
|
|
LOG,
|
|
});
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
emitStatus('SETUP_DB', { STATUS: 'failed', ERROR: message, LOG });
|
|
throw err;
|
|
}
|
|
}
|