clawdie-ai/setup/skills-memory.ts

/**
 * Step: skills-memory — bootstrap built-in knowledge into the skills DB.
 *
 * Goal: reduce install/onboarding LLM calls by importing a checked-in artifact
 * instead of generating knowledge live during first setup.
 */
import { execFileSync } from 'child_process';
import fs from 'fs';
import path from 'path';
import pg from 'pg';

import {
  DB_RUNTIME,
  SKILLS_DB_NAME,
  SKILLS_DB_URL,
  SKILLS_DB_USER,
} from '../src/config.js';
import {
  BUILTIN_KNOWLEDGE_ARTIFACT_SQL as ARTIFACT_SQL,
  BUILTIN_KNOWLEDGE_BOOTSTRAP_DIR as BOOTSTRAP_DIR,
  BUILTIN_KNOWLEDGE_METADATA_JSON as METADATA_JSON,
  readBuiltinKnowledgeMetadata as readMetadata,
} from '../src/builtin-knowledge-artifact.js';
import { logger } from '../src/logger.js';
import {
  collectSplitBrainStatus,
  type SplitBrainStatus,
} from '../src/split-brain-status.js';
import { readEnvFile } from '../src/env.js';
import { commandExists, getPlatform } from './platform.js';
import { emitStatus } from './status.js';
import { jailExists, jailRoot } from './bastille-helpers.js';

function getSkillsDbUrl(): string {
  return SKILLS_DB_URL;
}

export function redactDatabaseUrl(raw: string): string {
  try {
    const url = new URL(raw);
    if (url.password) url.password = 'REDACTED';
    return url.toString();
  } catch {
    return raw.replace(/(postgres(?:ql)?:\/\/[^:\s/@]+:)[^@\s]+(@)/iu, '$1REDACTED$2');
  }
}

export interface SkillsMemoryPlan {
  status:
    | 'artifact_missing'
    | 'db_unavailable'
    | 'already_current'
    | 'import_needed';
  reason:
    | 'artifact_files_missing'
    | 'skills_db_unavailable'
    | 'artifact_ready'
    | 'artifact_not_loaded'
    | 'artifact_outdated'
    | 'artifact_incomplete'
    | 'artifact_unknown';
  importNeeded: boolean;
  canImport: boolean;
}

export function deriveSkillsMemoryPlan(
  splitBrain: SplitBrainStatus,
): SkillsMemoryPlan {
  if (
    splitBrain.skillsArtifactSql !== 'present' ||
    splitBrain.skillsArtifactMetadata !== 'present'
  ) {
    return {
      status: 'artifact_missing',
      reason: 'artifact_files_missing',
      importNeeded: false,
      canImport: false,
    };
  }

  if (splitBrain.skillsDb !== 'available') {
    return {
      status: 'db_unavailable',
      reason: 'skills_db_unavailable',
      importNeeded: true,
      canImport: false,
    };
  }

  if (splitBrain.skillsArtifact === 'ready') {
    return {
      status: 'already_current',
      reason: 'artifact_ready',
      importNeeded: false,
      canImport: true,
    };
  }

  if (splitBrain.skillsArtifact === 'missing') {
    return {
      status: 'import_needed',
      reason: 'artifact_not_loaded',
      importNeeded: true,
      canImport: true,
    };
  }

  if (
    splitBrain.skillsArtifact === 'incomplete' &&
    splitBrain.skillsArtifactVersion !== 'unknown' &&
    splitBrain.skillsArtifactDbVersion !== 'unknown' &&
    splitBrain.skillsArtifactVersion !== splitBrain.skillsArtifactDbVersion
  ) {
    return {
      status: 'import_needed',
      reason: 'artifact_outdated',
      importNeeded: true,
      canImport: true,
    };
  }

  if (splitBrain.skillsArtifact === 'incomplete') {
    return {
      status: 'import_needed',
      reason: 'artifact_incomplete',
      importNeeded: true,
      canImport: true,
    };
  }

  return {
    status: 'import_needed',
    reason: 'artifact_unknown',
    importNeeded: true,
    canImport: true,
  };
}

function statusReport(
  status: string,
  extra: Record<string, string | number | boolean> = {},
): void {
  const metadata = readMetadata();
  emitStatus('SETUP_SKILLS_MEMORY', {
    STATUS: status,
    PSQL: commandExists('psql'),
    ARTIFACT_SQL: fs.existsSync(ARTIFACT_SQL),
    METADATA_JSON: fs.existsSync(METADATA_JSON),
    BOOTSTRAP_DIR: BOOTSTRAP_DIR,
    EMBEDDING_MODEL: String(metadata?.embedding_model ?? 'unknown'),
    ARTIFACT_VERSION: String(metadata?.artifact_version ?? 'unknown'),
    CHUNKING_VERSION: String(
      metadata?.chunker_version ??
        (metadata as Record<string, unknown> | undefined)?.[
          'chunking_version'
        ] ??
        'unknown',
    ),
    ...extra,
    LOG: 'logs/setup.log',
  });
}

function jailRootLocal(jailName: string): string {
  return `/usr/local/bastille/jails/${jailName}/root`;
}

function detectDbJailName(): string {
  const envOverrides = readEnvFile(['DB_JAIL_NAME']);
  const explicit = (
    process.env.DB_JAIL_NAME ||
    envOverrides.DB_JAIL_NAME ||
    ''
  ).trim();
  if (explicit) return explicit;

  return 'db';
}

function importArtifactViaBastille(jailName: string, dbName: string): void {
  const projectRoot = process.cwd();
  const tmpDir = path.join(projectRoot, 'tmp');
  fs.mkdirSync(tmpDir, { recursive: true });

  const stamp = Date.now();
  const hostTmp = path.join(tmpDir, `skills-artifact-${stamp}.sql`);
  fs.copyFileSync(ARTIFACT_SQL, hostTmp);

  const jailTmpRel = path.join('var', 'tmp', `skills-artifact-${stamp}.sql`);
  const jailTmpHostPath = path.join(jailRootLocal(jailName), jailTmpRel);
  fs.mkdirSync(path.dirname(jailTmpHostPath), { recursive: true });
  fs.copyFileSync(hostTmp, jailTmpHostPath);

  try {
    logger.info(
      { artifact: ARTIFACT_SQL, jailName },
      'Importing built-in knowledge artifact',
    );
    execFileSync(
      'bastille',
      [
        'cmd',
        jailName,
        'su',
        '-m',
        'postgres',
        '-c',
        `psql -v ON_ERROR_STOP=1 -d ${dbName} -f /${jailTmpRel}`,
      ],
      { stdio: 'inherit' },
    );
  } finally {
    fs.rmSync(hostTmp, { force: true });
    fs.rmSync(jailTmpHostPath, { force: true });
  }
}

function importArtifactViaHost(database: string): void {
  const projectRoot = process.cwd();
  const tmpDir = path.join(projectRoot, 'tmp');
  fs.mkdirSync(tmpDir, { recursive: true });

  const stamp = Date.now();
  const tmpSql = path.join(tmpDir, `skills-artifact-host-${stamp}.sql`);
  fs.copyFileSync(ARTIFACT_SQL, tmpSql);

  try {
    logger.info(
      { artifact: ARTIFACT_SQL, database },
      'Importing built-in knowledge artifact via host PostgreSQL',
    );
    const cmd = `psql -v ON_ERROR_STOP=1 -d ${JSON.stringify(database)} -f ${JSON.stringify(tmpSql)}`;
    if (commandExists('sudo')) {
      execFileSync('sudo', ['-n', '-u', 'postgres', 'sh', '-c', cmd], {
        stdio: 'inherit',
      });
      return;
    }
    execFileSync('su', ['-m', 'postgres', '-c', cmd], { stdio: 'inherit' });
  } finally {
    fs.rmSync(tmpSql, { force: true });
  }
}

function grantSkillsReadAccess(
  jailName: string,
  dbName: string,
  dbUser: string,
): void {
  const grantSql = [
    `GRANT USAGE ON SCHEMA public TO ${dbUser};`,
    `GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${dbUser};`,
    `GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO ${dbUser};`,
    `ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${dbUser};`,
    `ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO ${dbUser};`,
  ].join(' ');

  execFileSync(
    'bastille',
    [
      'cmd',
      jailName,
      'su',
      '-m',
      'postgres',
      '-c',
      `psql -v ON_ERROR_STOP=1 -d ${dbName} -c ${JSON.stringify(grantSql)}`,
    ],
    { stdio: 'inherit' },
  );
}

export function importArtifact(): void {
  if (!fs.existsSync(ARTIFACT_SQL)) {
    throw new Error('missing_skills_memory_artifact');
  }

  if (DB_RUNTIME === 'host' && getPlatform() === 'freebsd') {
    importArtifactViaHost(SKILLS_DB_NAME);
    return;
  }

  if (
    DB_RUNTIME !== 'host' &&
    getPlatform() === 'freebsd' &&
    commandExists('bastille')
  ) {
    const jailName = detectDbJailName();
    importArtifactViaBastille(jailName, SKILLS_DB_NAME);
    grantSkillsReadAccess(jailName, SKILLS_DB_NAME, SKILLS_DB_USER);
    return;
  }

  if (!commandExists('psql')) {
    throw new Error('missing_psql');
  }

  const dbUrl = getSkillsDbUrl();
  logger.info({ artifact: ARTIFACT_SQL }, 'Importing built-in knowledge artifact');
  execFileSync('psql', ['-v', 'ON_ERROR_STOP=1', dbUrl, '-f', ARTIFACT_SQL], {
    stdio: 'inherit',
  });
}

async function verifyImport(): Promise<Record<string, number>> {
  const { Client } = pg;
  const client = new Client({
    connectionString: getSkillsDbUrl(),
    connectionTimeoutMillis: 5000,
  });
  await client.connect();
  try {
    const [artifacts, documents, chunks] = await Promise.all([
      client.query<{ count: string }>(
        'SELECT count(*) FROM builtin_knowledge_artifacts',
      ),
      client.query<{ count: string }>(
        'SELECT count(*) FROM builtin_knowledge_documents',
      ),
      client.query<{ count: string }>(
        'SELECT count(*) FROM builtin_knowledge_chunks',
      ),
    ]);

    return {
      artifacts: Number(artifacts.rows[0]?.count ?? 0),
      documents: Number(documents.rows[0]?.count ?? 0),
      chunks: Number(chunks.rows[0]?.count ?? 0),
    };
  } finally {
    await client.end();
  }
}

export async function run(args: string[]): Promise<void> {
  const doEnsure = args.includes('--ensure');
  const doImport = args.includes('--import');
  const doStatus = args.includes('--status') || (!doImport && !doEnsure);
  const splitBrain = await collectSplitBrainStatus();
  const plan = deriveSkillsMemoryPlan(splitBrain);
  const commonStatus = {
    SKILLS_DB: splitBrain.skillsDb,
    SKILLS_ARTIFACT: splitBrain.skillsArtifact,
    SKILLS_ARTIFACT_DB_VERSION: splitBrain.skillsArtifactDbVersion,
    IMPORT_NEEDED: plan.importNeeded,
    IMPORT_REASON: plan.reason,
  };

  if (doStatus && !doImport && !doEnsure) {
    statusReport(plan.status, {
      ARTIFACT_ROWS: splitBrain.skillsArtifactRows,
      DOCUMENT_ROWS: splitBrain.skillsDocumentRows,
      CHUNK_ROWS: splitBrain.skillsChunkRows,
      ...commonStatus,
    });
    return;
  }

  if (doEnsure) {
    if (plan.status === 'artifact_missing') {
      statusReport('artifact_missing', {
        ARTIFACT_ROWS: splitBrain.skillsArtifactRows,
        DOCUMENT_ROWS: splitBrain.skillsDocumentRows,
        CHUNK_ROWS: splitBrain.skillsChunkRows,
        ...commonStatus,
      });
      return;
    }
    if (plan.status === 'already_current') {
      statusReport('already_current', {
        ARTIFACT_ROWS: splitBrain.skillsArtifactRows,
        DOCUMENT_ROWS: splitBrain.skillsDocumentRows,
        CHUNK_ROWS: splitBrain.skillsChunkRows,
        ...commonStatus,
      });
      return;
    }
    if (!plan.canImport) {
      throw new Error('skills_memory_import_unavailable');
    }
  }

  importArtifact();
  const counts = await verifyImport();
  statusReport('imported', {
    DB_URL_TARGET: redactDatabaseUrl(getSkillsDbUrl()),
    ARTIFACT_ROWS: counts.artifacts,
    DOCUMENT_ROWS: counts.documents,
    CHUNK_ROWS: counts.chunks,
    IMPORT_NEEDED: false,
    IMPORT_REASON: 'artifact_ready',
  });
}