clawdie-ai/scripts/memory/refresh-skills-artifact.mjs

#!/usr/bin/env node
/**
 * Regenerate bootstrap/skills-memory artifacts only when knowledge sources changed.
 *
 * Default mode checks working tree + untracked files against HEAD. Pass
 * --base <ref> to compare committed changes from <ref>...HEAD as well.
 */
import { execFileSync, spawnSync } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
import { pathToFileURL } from 'node:url';

const PROJECT_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..', '..');
const DEFAULT_MIN_REMAINING_USD = 0.25;

export function parseArgs(argv) {
  const args = {
    base: undefined,
    artifactVersion: undefined,
    force: false,
    checkOnly: false,
    committedOnly: false,
    skipBudgetCheck: false,
    skipOnMissingKey: false,
    minRemainingUsd: Number(process.env.SKILLS_ARTIFACT_MIN_OPENROUTER_REMAINING_USD || DEFAULT_MIN_REMAINING_USD),
  };

  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];
    if (arg === '--base') {
      args.base = argv[++i];
    } else if (arg === '--artifact-version') {
      args.artifactVersion = argv[++i];
    } else if (arg === '--min-remaining-usd') {
      args.minRemainingUsd = Number(argv[++i]);
    } else if (arg === '--force') {
      args.force = true;
    } else if (arg === '--check-only') {
      args.checkOnly = true;
    } else if (arg === '--committed-only') {
      args.committedOnly = true;
    } else if (arg === '--skip-budget-check') {
      args.skipBudgetCheck = true;
    } else if (arg === '--skip-on-missing-key') {
      args.skipOnMissingKey = true;
    } else if (arg === '-h' || arg === '--help') {
      printHelp();
      process.exit(0);
    } else {
      throw new Error(`Unknown option: ${arg}`);
    }
  }

  if (args.base && !args.base.trim()) throw new Error('--base requires a value');
  if (args.artifactVersion && !args.artifactVersion.trim()) throw new Error('--artifact-version requires a value');
  if (!Number.isFinite(args.minRemainingUsd) || args.minRemainingUsd < 0) {
    throw new Error('--min-remaining-usd must be a non-negative number');
  }
  return args;
}

function printHelp() {
  console.log(`Usage: node scripts/memory/refresh-skills-artifact.mjs [options]

Regenerate bootstrap/skills-memory/artifact.sql only when knowledge-source files changed.

Options:
  --base <ref>                  Include committed changes from <ref>...HEAD
  --artifact-version <version>  Override artifact version (default: current metadata version)
  --force                       Regenerate even when no relevant source changed
  --check-only                  Report whether regeneration is needed, then exit
  --committed-only              Ignore working tree and untracked files
  --skip-budget-check           Do not query OpenRouter key status before regenerating
  --skip-on-missing-key         Exit 0 instead of failing when OPENROUTER_API_KEY is missing
  --min-remaining-usd <amount>  Minimum reported OpenRouter remaining budget (default: ${DEFAULT_MIN_REMAINING_USD})
`);
}

function git(args) {
  return execFileSync('git', args, {
    cwd: PROJECT_ROOT,
    encoding: 'utf8',
    stdio: ['ignore', 'pipe', 'pipe'],
  }).trim();
}

function gitLines(args) {
  const out = git(args);
  return out ? out.split(/\r?\n/u).filter(Boolean) : [];
}

function refExists(ref) {
  const result = spawnSync('git', ['rev-parse', '--verify', '--quiet', ref], {
    cwd: PROJECT_ROOT,
    stdio: 'ignore',
  });
  return result.status === 0;
}

export function isKnowledgeSource(file) {
  const p = file.replace(/\\/gu, '/');
  if (p === 'SOUL.md') return true;
  if (p === 'IDENTITY.md') return true;
  if (p === 'USER.md') return true;
  if (p === 'AGENTS.md') return true;
  if (p === 'MEMORY.md') return true;
  if (p === 'CLAWDIE-ISO.md') return true;
  if (p === 'scripts/memory/embed-builtin-knowledge.py') return true;

  if (p.startsWith('docs/internal/sessions/')) return false;
  if (/^docs\/internal\/BUILD-TEST-REPORT-.*\.md$/u.test(p)) return false;
  if (p === 'docs/internal/test-results.md') return false;
  if (p.startsWith('docs/public/') && p.endsWith('.md')) return true;
  if (p.startsWith('docs/internal/') && p.endsWith('.md')) return true;
  if (/^\.agent\/skills\/[^/]+\/SKILL\.md$/u.test(p)) return true;
  return false;
}

export function collectChangedFiles(options) {
  const files = new Set();
  const diffFilter = '--diff-filter=ACMRTUXBD';
  const listGitLines = options.gitLines ?? gitLines;

  if (options.base) {
    if (refExists(options.base)) {
      for (const file of listGitLines(['diff', '--name-only', diffFilter, `${options.base}...HEAD`])) {
        files.add(file);
      }
    } else {
      console.warn(`Warning: base ref not found, ignoring --base ${options.base}`);
    }
  }

  if (!options.committedOnly) {
    for (const file of listGitLines(['diff', '--name-only', diffFilter, 'HEAD'])) {
      files.add(file);
    }
    for (const file of listGitLines(['ls-files', '--others', '--exclude-standard'])) {
      files.add(file);
    }
  }

  return [...files].sort();
}

function currentArtifactVersion() {
  const metadataPath = path.join(PROJECT_ROOT, 'bootstrap/skills-memory/metadata.json');
  try {
    const metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8'));
    if (typeof metadata.artifact_version === 'string' && metadata.artifact_version.trim()) {
      return metadata.artifact_version.trim();
    }
  } catch {
    // fall through
  }
  return 'v1-auto';
}

function safeNumber(value) {
  if (typeof value === 'number' && Number.isFinite(value)) return value;
  if (typeof value === 'string' && value.trim() && Number.isFinite(Number(value))) return Number(value);
  return undefined;
}

async function checkOpenRouterBudget(options) {
  const apiKey = process.env.OPENROUTER_API_KEY;
  if (!apiKey) {
    if (options.skipOnMissingKey) {
      console.log('OpenRouter key missing; skipping artifact refresh.');
      process.exit(0);
    }
    throw new Error('OPENROUTER_API_KEY is required for embedding generation');
  }

  const response = await fetch('https://openrouter.ai/api/v1/auth/key', {
    headers: { Authorization: `Bearer ${apiKey}` },
    signal: AbortSignal.timeout(10_000),
  });
  if (!response.ok) throw new Error(`OpenRouter key status failed: HTTP ${response.status}`);
  const json = await response.json();
  const data = json?.data ?? json;
  const usage = safeNumber(data?.usage);
  const limit = safeNumber(data?.limit);
  const remaining = safeNumber(data?.remaining) ?? (usage !== undefined && limit !== undefined ? limit - usage : undefined);

  const usageText = usage === undefined ? 'unknown' : `$${usage.toFixed(4)} used`;
  const remainingText = remaining === undefined ? 'unknown remaining' : `$${remaining.toFixed(4)} remaining`;
  console.log(`OpenRouter budget: ${usageText}; ${remainingText}`);
  console.log('Last observed full artifact refresh cost was about $0.003.');

  if (remaining !== undefined && remaining < options.minRemainingUsd) {
    throw new Error(
      `OpenRouter remaining budget ${remaining.toFixed(4)} is below minimum ${options.minRemainingUsd.toFixed(4)}`,
    );
  }
}

function runGenerator(artifactVersion) {
  const result = spawnSync(
    'python3',
    [
      'scripts/memory/embed-builtin-knowledge.py',
      '--output-sql',
      'bootstrap/skills-memory/artifact.sql',
      '--output-metadata',
      'bootstrap/skills-memory/metadata.json',
      '--artifact-version',
      artifactVersion,
    ],
    { cwd: PROJECT_ROOT, stdio: 'inherit' },
  );

  if (result.status !== 0) {
    throw new Error(`embed-builtin-knowledge.py failed with exit code ${result.status ?? 'unknown'}`);
  }
}

export async function main() {
  const options = parseArgs(process.argv.slice(2));
  const changedFiles = collectChangedFiles(options);
  const relevantFiles = changedFiles.filter(isKnowledgeSource);

  if (relevantFiles.length === 0 && !options.force) {
    console.log('Skills artifact refresh not needed; no knowledge-source files changed.');
    return;
  }

  if (relevantFiles.length > 0) {
    console.log('Knowledge-source changes detected:');
    for (const file of relevantFiles) console.log(`  ${file}`);
  } else {
    console.log('Forced skills artifact refresh requested.');
  }

  if (options.checkOnly) {
    console.log('Check only; not regenerating artifacts.');
    return;
  }

  if (!options.skipBudgetCheck) {
    await checkOpenRouterBudget(options);
  }

  const artifactVersion = options.artifactVersion ?? currentArtifactVersion();
  console.log(`Regenerating skills artifact as ${artifactVersion}...`);
  runGenerator(artifactVersion);
}

const isDirectRun =
  process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href;

if (isDirectRun) {
  main().catch((error) => {
    console.error(error instanceof Error ? error.message : String(error));
    process.exit(1);
  });
}