clawdie-iso/firstboot/shell-deploy.sh

#!/bin/sh
# Clawdie Shell — Deployment Module
# Purpose: Extract Clawdie-AI tarball and run installation
# POSIX-compliant (no bash-isms)

set -eu

# Configuration (can be overridden for testing)
CLAWDIE_HOME="${CLAWDIE_HOME:-/home/clawdie}"
CLAWDIE_AI_DIR="${CLAWDIE_AI_DIR:-$CLAWDIE_HOME/clawdie-ai}"
CLAWDIE_TARBALL="${CLAWDIE_TARBALL:-/usr/local/share/clawdie-iso/clawdie-ai.tar.gz}"
ENV_FILE="${ENV_FILE:-$CLAWDIE_HOME/.env}"
LOG_FILE="${LOG_FILE:-/var/log/clawdie-firstboot.log}"
PROGRESS_FILE="${PROGRESS_FILE:-/var/log/clawdie-firstboot.progress}"
USB_LLM_MODELS_PATH="${USB_LLM_MODELS_PATH:-/mnt/media/llm-models}"
LLAMA_CPP_JAIL_NAME="${LLAMA_CPP_JAIL_NAME:-llamacpp}"
LLAMA_CPP_MODELS_DIR="${LLAMA_CPP_MODELS_DIR:-/var/db/llm-models}"

# ============================================================================
# MAIN ENTRY POINT
# ============================================================================

clawdie_shell_deploy() {
  # Main orchestrator: fresh install or upgrade, then run install-all

  log_msg "[deploy] Starting Clawdie-AI deployment (mode: ${CLAWDIE_BOOT_MODE:-install})"

  if [ "${CLAWDIE_BOOT_MODE:-install}" = "upgrade" ]; then
    clawdie_shell_deploy_upgrade
  else
    clawdie_shell_deploy_fresh
  fi

  # Common post-deploy steps
  cd "$CLAWDIE_AI_DIR" || {
    log_msg "[deploy] ERROR: Failed to cd to $CLAWDIE_AI_DIR"
    return 1
  }

  # Run npm install-all (handles db migrations, secrets, services)
  log_msg "[deploy] Running npm run install-all..."
  clawdie_shell_deploy_run_install_all || {
    log_msg "[deploy] ERROR: npm run install-all failed"
    return 1
  }
  log_msg "[deploy] npm install-all completed successfully"

  if [ -f "${CLAWDIE_HOME}/.env.upgrade-backup" ]; then
    rm -f "${CLAWDIE_HOME}/.env.upgrade-backup" 2>/dev/null || true
    log_msg "[deploy] Cleared upgrade .env backup"
  fi

  # Seed llama-cpp models if selected and available on USB
  clawdie_shell_deploy_seed_llama_cpp_models || {
    log_msg "[deploy] WARNING: Failed to seed llama-cpp models"
  }

  # Configure Aider venv for FreeBSD tree-sitter compatibility
  clawdie_shell_deploy_setup_aider_venv || {
    log_msg "[deploy] WARNING: Aider venv setup failed"
  }

  # Verify
  clawdie_shell_deploy_verify || {
    log_msg "[deploy] WARNING: Post-install verification found issues"
  }

  echo "[DEPLOY] SUCCESS" >> "$PROGRESS_FILE"
  log_msg "[deploy] Clawdie-AI deployment complete"
}

# ============================================================================
# FRESH INSTALL PATH
# ============================================================================

clawdie_shell_deploy_fresh() {
  # Validate inputs
  if [ -z "${ASSISTANT_NAME:-}" ]; then
    log_msg "[deploy] ERROR: ASSISTANT_NAME not set"
    return 1
  fi
  if [ -z "${AGENT_DOMAIN:-}" ]; then
    log_msg "[deploy] ERROR: AGENT_DOMAIN not set"
    return 1
  fi

  # Extract tarball
  clawdie_shell_deploy_extract_tarball || {
    log_msg "[deploy] ERROR: Failed to extract tarball"
    return 1
  }

  # Validate extracted directory
  if [ ! -d "$CLAWDIE_AI_DIR" ] || [ ! -f "$CLAWDIE_AI_DIR/package.json" ]; then
    log_msg "[deploy] ERROR: Tarball extraction failed or invalid"
    return 1
  fi
  log_msg "[deploy] Package.json verified"

  # Copy firstboot .env seed into the repo root for install-all
  if [ -f "$ENV_FILE" ]; then
    cp "$ENV_FILE" "$CLAWDIE_AI_DIR/.env" 2>/dev/null || {
      log_msg "[deploy] WARNING: Failed to copy $ENV_FILE to $CLAWDIE_AI_DIR/.env"
    }
    chmod 600 "$CLAWDIE_AI_DIR/.env" 2>/dev/null || true
    chown clawdie:clawdie "$CLAWDIE_AI_DIR/.env" 2>/dev/null || true
    log_msg "[deploy] Seeded $CLAWDIE_AI_DIR/.env from firstboot"
  else
    log_msg "[deploy] WARNING: ENV_FILE not found at $ENV_FILE (install-all will generate defaults)"
  fi
}

# ============================================================================
# UPGRADE PATH
# ============================================================================

clawdie_shell_deploy_upgrade() {
  log_msg "[deploy] Upgrade mode — preserving existing config and customizations"

  if [ ! -d "$CLAWDIE_AI_DIR" ] || [ ! -f "$CLAWDIE_AI_DIR/package.json" ]; then
    log_msg "[deploy] WARNING: No existing install found — falling back to fresh install"
    clawdie_shell_deploy_fresh
    return
  fi

  if [ ! -f "$CLAWDIE_TARBALL" ]; then
    log_msg "[deploy] ERROR: Tarball not found: $CLAWDIE_TARBALL"
    return 1
  fi

  # Save existing version
  _old_version=$(cd "$CLAWDIE_AI_DIR" && node -p "require('./package.json').version" 2>/dev/null || echo "unknown")
  log_msg "[deploy] Existing version: $_old_version"

  # Step 1: Extract new tarball to a staging directory
  STAGING_DIR="${CLAWDIE_HOME}/clawdie-ai-upgrade-staging"
  rm -rf "$STAGING_DIR"
  mkdir -p "$STAGING_DIR"

  if ! tar -xzf "$CLAWDIE_TARBALL" -C "$STAGING_DIR" --strip-components=1 2>/dev/null; then
    # Try without --strip-components (Codeberg archives have a top-level dir)
    tar -xzf "$CLAWDIE_TARBALL" -C "$(dirname "$STAGING_DIR")" 2>/dev/null || {
      log_msg "[deploy] ERROR: Failed to extract upgrade tarball"
      return 1
    }
    # Rename Clawdie-AI → staging
    if [ -d "$(dirname "$STAGING_DIR")/Clawdie-AI" ]; then
      rm -rf "$STAGING_DIR"
      mv "$(dirname "$STAGING_DIR")/Clawdie-AI" "$STAGING_DIR"
    fi
  fi

  _new_version=$(cd "$STAGING_DIR" && node -p "require('./package.json').version" 2>/dev/null || echo "unknown")
  log_msg "[deploy] New version: $_new_version"

  if [ ! -f "$STAGING_DIR/package.json" ]; then
    log_msg "[deploy] ERROR: Staging directory missing package.json"
    return 1
  fi

  # Step 2: Preserve .env (never overwrite with seed)
  if [ -f "$CLAWDIE_AI_DIR/.env" ]; then
    cp "$CLAWDIE_AI_DIR/.env" "${CLAWDIE_HOME}/.env.upgrade-backup"
    log_msg "[deploy] Backed up existing .env"
  fi

  # Step 3: Apply update via skills-engine (three-way merge)
  if [ -d "$CLAWDIE_AI_DIR/.nanoclaw" ] && [ -f "$CLAWDIE_AI_DIR/.nanoclaw/state.yaml" ]; then
    log_msg "[deploy] Skills engine found — running three-way merge update"

    _tsx="$CLAWDIE_AI_DIR/node_modules/.bin/tsx"
    [ -x "$_tsx" ] || _tsx="tsx"

    # Run applyUpdate with the staging dir as the new core
    _update_script='
import { applyUpdate } from "./skills-engine/update.js";
const result = await applyUpdate(process.argv[1]);
console.log(JSON.stringify(result, null, 2));
if (!result.success) process.exit(1);
'
    if [ "$(id -u)" -eq 0 ]; then
      su - clawdie -c "cd $CLAWDIE_AI_DIR && $_tsx --eval '$_update_script' '$STAGING_DIR'" 2>&1 | tee -a "$LOG_FILE" || {
        log_msg "[deploy] WARNING: Skills-engine update failed — falling back to overwrite"
        clawdie_shell_deploy_upgrade_overwrite "$STAGING_DIR" || return 1
      }
    else
      cd "$CLAWDIE_AI_DIR" && "$_tsx" --eval "$_update_script" "$STAGING_DIR" 2>&1 | tee -a "$LOG_FILE" || {
        log_msg "[deploy] WARNING: Skills-engine update failed — falling back to overwrite"
        clawdie_shell_deploy_upgrade_overwrite "$STAGING_DIR" || return 1
      }
    fi
  else
    # No skills engine — overwrite and migrate
    log_msg "[deploy] No skills engine — overwriting and migrating"
    clawdie_shell_deploy_upgrade_overwrite "$STAGING_DIR" || return 1
  fi

  # Step 4: Restore .env
  if [ -f "${CLAWDIE_HOME}/.env.upgrade-backup" ]; then
    cp "${CLAWDIE_HOME}/.env.upgrade-backup" "$CLAWDIE_AI_DIR/.env"
    chmod 600 "$CLAWDIE_AI_DIR/.env" 2>/dev/null || true
    chown clawdie:clawdie "$CLAWDIE_AI_DIR/.env" 2>/dev/null || true
    log_msg "[deploy] Restored existing .env"
  fi

  # Cleanup staging
  rm -rf "$STAGING_DIR"

  chown -R clawdie:clawdie "$CLAWDIE_AI_DIR" 2>/dev/null || true
  log_msg "[deploy] Upgrade complete: $_old_version → $_new_version"
}

# Fallback: overwrite existing files with new tarball, then migrate
clawdie_shell_deploy_upgrade_overwrite() {
  _staging="$1"
  log_msg "[deploy] Overwriting existing install with new version"

  # Preserve data directories that shouldn't be replaced
  for _dir in data store logs groups .nanoclaw node_modules; do
    if [ -e "${CLAWDIE_HOME}/_upgrade_${_dir}" ]; then
      rm -rf "${CLAWDIE_HOME}/_upgrade_${_dir}.stale" 2>/dev/null || true
      mv "${CLAWDIE_HOME}/_upgrade_${_dir}" "${CLAWDIE_HOME}/_upgrade_${_dir}.stale" 2>/dev/null || true
      log_msg "[deploy] Moved existing backup for $_dir to _upgrade_${_dir}.stale"
    fi
    if [ -d "$CLAWDIE_AI_DIR/$_dir" ]; then
      mv "$CLAWDIE_AI_DIR/$_dir" "${CLAWDIE_HOME}/_upgrade_${_dir}" 2>/dev/null || true
    fi
  done

  # Copy new files over existing
  if ! (cd "$_staging" && tar -cf - .) | (cd "$CLAWDIE_AI_DIR" && tar -xf -); then
    log_msg "[deploy] ERROR: Failed to copy new files from staging"
    return 1
  fi

  # Restore preserved directories
  for _dir in data store logs groups .nanoclaw node_modules; do
    if [ -d "${CLAWDIE_HOME}/_upgrade_${_dir}" ]; then
      rm -rf "$CLAWDIE_AI_DIR/$_dir" 2>/dev/null || true
      mv "${CLAWDIE_HOME}/_upgrade_${_dir}" "$CLAWDIE_AI_DIR/$_dir"
    fi
  done

  # Run migrateExisting if skills engine is available but wasn't initialized
  if [ ! -d "$CLAWDIE_AI_DIR/.nanoclaw" ]; then
    _tsx="$CLAWDIE_AI_DIR/node_modules/.bin/tsx"
    [ -x "$_tsx" ] || _tsx="tsx"
    _migrate_script='import { migrateExisting } from "./skills-engine/migrate.js"; migrateExisting();'
    if [ "$(id -u)" -eq 0 ]; then
      su - clawdie -c "cd $CLAWDIE_AI_DIR && $_tsx --eval '$_migrate_script'" 2>&1 | tee -a "$LOG_FILE" || {
        log_msg "[deploy] WARNING: migrateExisting failed — skills tracking unavailable"
      }
    else
      cd "$CLAWDIE_AI_DIR" && "$_tsx" --eval "$_migrate_script" 2>&1 | tee -a "$LOG_FILE" || {
        log_msg "[deploy] WARNING: migrateExisting failed — skills tracking unavailable"
      }
    fi
  fi
}

# ============================================================================
# TARBALL EXTRACTION
# ============================================================================

clawdie_shell_deploy_extract_tarball() {
  # Extract Clawdie-AI tarball to CLAWDIE_HOME

  if [ ! -f "$CLAWDIE_TARBALL" ]; then
    log_msg "[deploy] ERROR: Tarball not found: $CLAWDIE_TARBALL"
    return 1
  fi

  log_msg "[deploy] Extracting $CLAWDIE_TARBALL"

  # Create parent directory if needed
  mkdir -p "$CLAWDIE_HOME"

  # Extract tarball
  # Note: tarball structure is Clawdie-AI/... so extract to CLAWDIE_HOME parent
  # and rename to clawdie-ai
  if ! tar -xzf "$CLAWDIE_TARBALL" -C "$(dirname "$CLAWDIE_HOME")" 2>/dev/null; then
    log_msg "[deploy] ERROR: tar extraction failed"
    return 1
  fi

  # Rename if extraction created Clawdie-AI directory
  if [ -d "$(dirname "$CLAWDIE_HOME")/Clawdie-AI" ] && [ ! -d "$CLAWDIE_AI_DIR" ]; then
    mv "$(dirname "$CLAWDIE_HOME")/Clawdie-AI" "$CLAWDIE_AI_DIR"
    log_msg "[deploy] Renamed Clawdie-AI to clawdie-ai"
  fi

  # Fix ownership
  chown -R clawdie:clawdie "$CLAWDIE_HOME" 2>/dev/null || true

  log_msg "[deploy] Tarball extraction complete"
  return 0
}

# ============================================================================
# NPM INSTALL-ALL
# ============================================================================

clawdie_shell_deploy_run_install_all() {
  # Run npm run install-all with error handling

  if [ ! -f "$CLAWDIE_AI_DIR/package.json" ]; then
    log_msg "[deploy] ERROR: package.json not found in $CLAWDIE_AI_DIR"
    return 1
  fi

  # Run npm install-all as clawdie user if not already clawdie
  local run_cmd="npm run install-all"

  if [ "$(id -u)" -eq 0 ]; then
    # Running as root, drop to clawdie user
    su - clawdie -c "cd $CLAWDIE_AI_DIR && $run_cmd" 2>&1 | tee -a "$LOG_FILE" || return 1
  else
    # Running as clawdie user directly
    $run_cmd 2>&1 | tee -a "$LOG_FILE" || return 1
  fi

  return 0
}

# ============================================================================
# AIDER VENV SETUP (FREEBSD)
# ============================================================================

clawdie_shell_deploy_setup_aider_venv() {
  local venv_dir="/opt/clawdie/venv/aider"
  local tmp_dir="${CLAWDIE_AI_DIR}/tmp"
  local python_bin="${venv_dir}/bin/python"
  local pip_bin="${venv_dir}/bin/pip"

  if [ ! -d "$CLAWDIE_AI_DIR" ]; then
    log_msg "[deploy] WARNING: $CLAWDIE_AI_DIR missing — skipping Aider venv"
    return 0
  fi

  mkdir -p "$tmp_dir" 2>/dev/null || true

  mkdir -p "/opt/clawdie/venv" 2>/dev/null || true
  if id clawdie >/dev/null 2>&1; then
    chown -R clawdie:clawdie "/opt/clawdie" 2>/dev/null || true
  fi

  if [ ! -x "$python_bin" ]; then
    log_msg "[deploy] Creating Aider venv at $venv_dir"
    if [ "$(id -u)" -eq 0 ]; then
      su - clawdie -c "python3.11 -m venv --system-site-packages '$venv_dir'" || return 1
    else
      python3.11 -m venv --system-site-packages "$venv_dir" || return 1
    fi
  fi

  # Convenience symlink for operator tooling.
  if [ ! -L "/home/clawdie/.venv/aider" ]; then
    mkdir -p "/home/clawdie/.venv" 2>/dev/null || true
    rm -rf "/home/clawdie/.venv/aider" 2>/dev/null || true
    ln -s "$venv_dir" "/home/clawdie/.venv/aider" 2>/dev/null || true
    chown -h clawdie:clawdie "/home/clawdie/.venv/aider" 2>/dev/null || true
  fi

  if [ ! -x "$pip_bin" ]; then
    log_msg "[deploy] WARNING: Aider venv pip missing — skipping"
    return 1
  fi

  log_msg "[deploy] Pinning Aider venv deps (litellm + tree_sitter)"
  if [ "$(id -u)" -eq 0 ]; then
    su - clawdie -c "TMPDIR='$tmp_dir' '$pip_bin' install --no-user --upgrade --ignore-installed aider-chat==0.86.2" || return 1
    su - clawdie -c "TMPDIR='$tmp_dir' '$pip_bin' install --no-user --upgrade --ignore-installed litellm==1.81.10" || return 1
    su - clawdie -c "TMPDIR='$tmp_dir' '$pip_bin' install --no-user --upgrade --force-reinstall tree_sitter==0.20.4" || return 1
  else
    TMPDIR="$tmp_dir" "$pip_bin" install --no-user --upgrade --ignore-installed aider-chat==0.86.2 || return 1
    TMPDIR="$tmp_dir" "$pip_bin" install --no-user --upgrade --ignore-installed litellm==1.81.10 || return 1
    TMPDIR="$tmp_dir" "$pip_bin" install --no-user --upgrade --force-reinstall tree_sitter==0.20.4 || return 1
  fi

  return 0
}

# ============================================================================
# POST-INSTALL VERIFICATION
# ============================================================================

clawdie_shell_deploy_verify() {
  # Verify basic services are running

  log_msg "[deploy] Running post-install verification"

  local failed=0

  # Check if jails are running (optional, may not be ready yet)
  if command -v jls >/dev/null 2>&1; then
    local jail_count=$(jls -N 2>/dev/null | wc -l || echo "0")
    if [ "$jail_count" -gt 1 ]; then
      log_msg "[deploy] ✓ Jails active: $((jail_count - 1)) jails"
    else
      log_msg "[deploy] ⚠ No jails detected (may still be provisioning)"
    fi
  fi

  # Check if clawdie service is enabled
  if grep -q "clawdie_enable" /etc/rc.conf 2>/dev/null; then
    log_msg "[deploy] ✓ clawdie service configured in rc.conf"
  else
    log_msg "[deploy] ⚠ clawdie service not in rc.conf"
    failed=1
  fi

  # Check if .env file was created
  if [ -f "$ENV_FILE" ]; then
    log_msg "[deploy] ✓ .env file created"
  else
    log_msg "[deploy] ⚠ .env file not found"
    failed=1
  fi

  # Check if skills engine was initialized
  if [ -d "$CLAWDIE_AI_DIR/.nanoclaw" ] && [ -f "$CLAWDIE_AI_DIR/.nanoclaw/state.yaml" ]; then
    log_msg "[deploy] ✓ Skills engine initialized (.nanoclaw/)"
  else
    log_msg "[deploy] ⚠ Skills engine not initialized (.nanoclaw/ missing)"
  fi

  # Check if bootstrap knowledge was imported
  if [ -f "$CLAWDIE_AI_DIR/bootstrap/skills-memory/artifact.sql" ]; then
    log_msg "[deploy] ✓ Bootstrap knowledge artifact present"
  fi

  # Check if Aider CLI is available (controlplane harness dependency)
  if command -v aider >/dev/null 2>&1; then
    local aider_version
    aider_version=$(aider --version 2>/dev/null | head -n 1 || true)
    if [ -n "$aider_version" ]; then
      log_msg "[deploy] ✓ Aider available: ${aider_version}"
    else
      log_msg "[deploy] ✓ Aider available"
    fi
  else
    log_msg "[deploy] ⚠ Aider CLI not found (py311-aider_chat missing)"
  fi

  return $failed
}

# ============================================================================
# LLAMA-CPP MODEL SEEDING
# ============================================================================

clawdie_shell_deploy_seed_llama_cpp_models() {
  if [ "${LOCAL_LLM_PROVIDER:-none}" != "llama_cpp" ]; then
    return 0
  fi

  if [ ! -d "$USB_LLM_MODELS_PATH" ]; then
    log_msg "[deploy] No USB model pack found at $USB_LLM_MODELS_PATH"
    return 0
  fi

  if ! find "$USB_LLM_MODELS_PATH" -name "*.gguf" -type f 2>/dev/null | head -1 >/dev/null; then
    log_msg "[deploy] No GGUF models found in $USB_LLM_MODELS_PATH"
    return 0
  fi

  if ! command -v bastille >/dev/null 2>&1; then
    log_msg "[deploy] bastille not available — cannot seed llama-cpp models"
    return 1
  fi

  log_msg "[deploy] Seeding llama-cpp models into ${LLAMA_CPP_JAIL_NAME}:${LLAMA_CPP_MODELS_DIR}"
  bastille cmd "$LLAMA_CPP_JAIL_NAME" install -d -m 755 "$LLAMA_CPP_MODELS_DIR" || return 1

  for model in "$USB_LLM_MODELS_PATH"/*.gguf; do
    [ -f "$model" ] || continue
    dest="/usr/local/bastille/jails/${LLAMA_CPP_JAIL_NAME}/root${LLAMA_CPP_MODELS_DIR}/$(basename "$model")"
    if [ -f "$dest" ]; then
      log_msg "[deploy] Model already present: $(basename "$model")"
      continue
    fi
    cp "$model" "$dest" 2>/dev/null || return 1
    log_msg "[deploy] Copied model: $(basename "$model")"
  done

  return 0
}

# ============================================================================
# LOGGING HELPER
# ============================================================================

log_msg() {
  echo "$(date '+%H:%M:%S') $1" | tee -a "$LOG_FILE" 2>/dev/null || true
}

# Only run if sourced directly (not during test)
if [ "${SHELL_DEPLOY_TEST:-0}" -eq 0 ]; then
  clawdie_shell_deploy
fi