Clawdie AI
0357ddb789
- Add embed-builtin-knowledge.py script for OpenRouter bge-m3 embeddings - Update artifact.sql with pre-generated 1024-dim embeddings for 6 chunks - Update metadata: openrouter/baai/bge-m3, embedding_count=6, search_mode=hybrid - Change locale from en-US to sl-SI per project language setting Author: Clawdie AI <hello@clawdie.si> Co-Authored-By: Samo Blatnik, Zed.ai
105 lines
4.4 KiB
Python
105 lines
4.4 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
scripts/memory/embed-builtin-knowledge.py — Generate embedding SQL for built-in knowledge.
|
|
|
|
Reads chunk texts from artifact.sql, generates embeddings via OpenRouter (baai/bge-m3),
|
|
and outputs SQL INSERT statements for builtin_knowledge_embeddings table.
|
|
|
|
Usage:
|
|
OPENROUTER_API_KEY=sk-or-... python3 scripts/memory/embed-builtin-knowledge.py
|
|
|
|
Output: SQL INSERT statements ready to merge into artifact.sql
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
|
|
|
CHUNKS = [
|
|
{
|
|
"content_hash": "chunk-readme-quick-start-0-v1",
|
|
"text": "Current main setup order on FreeBSD is: run ./setup.sh, install pi if it is still missing, run npm run wizard, then run environment, pi-config, jails --create, db, git, cms, mounts, telegram-auth, service, and verify. The db jail is mandatory for split-brain, and built-in skills are imported during db setup.",
|
|
},
|
|
{
|
|
"content_hash": "chunk-readme-naming-dns-0-v1",
|
|
"text": "Use AGENT_DOMAIN for public addresses and AGENT_INTERNAL_DOMAIN for internal jail resolution. Fresh installs should use home.arpa for internal names, not .local, because .local collides with mDNS. The canonical bridge is warden0.",
|
|
},
|
|
{
|
|
"content_hash": "chunk-postgres-memory-split-brain-0-v1",
|
|
"text": "Split-brain means two PostgreSQL databases inside the db jail. Agent System Skills is preloaded knowledge with a separate lifecycle. User or Agent Memory is dynamic, grows with conversations, and uses hybrid search over summaries, chunks, and embeddings.",
|
|
},
|
|
{
|
|
"content_hash": "chunk-setup-skill-host-baseline-0-v1",
|
|
"text": "Recommended FreeBSD host baseline packages are bash, git, bsddialog, bastille, node24, npm, tmux, python311, uv, ripgrep, fd, rsync, postgresql17-client, py311-pillow, and dejavu. If pkg reports that fd replaces fd-find, that replacement is expected.",
|
|
},
|
|
{
|
|
"content_hash": "chunk-builtin-knowledge-import-contract-0-v1",
|
|
"text": "Built-in knowledge bootstrap must create the skills schema, import the shipped SQL package, read artifact metadata, and verify row counts. Install must not silently fall back to live embedding generation if the artifact is missing or invalid.",
|
|
},
|
|
{
|
|
"content_hash": "chunk-builtin-knowledge-import-contract-1-v1",
|
|
"text": "Built-in knowledge should stay separate from user memory. The db jail stores and serves both, but the skills side must remain inspectable, versioned, and independently updatable from dynamic user or agent memory.",
|
|
},
|
|
]
|
|
|
|
OPENROUTER_URL = "https://openrouter.ai/api/v1/embeddings"
|
|
MODEL = "baai/bge-m3"
|
|
|
|
|
|
def embed(texts: list[str]) -> list[list[float]]:
|
|
api_key = os.environ.get("OPENROUTER_API_KEY", "")
|
|
if not api_key:
|
|
print("ERROR: OPENROUTER_API_KEY not set", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
body = {"input": texts, "model": MODEL}
|
|
payload = json.dumps(body).encode()
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
"Authorization": f"Bearer {api_key}",
|
|
}
|
|
|
|
req = urllib.request.Request(OPENROUTER_URL, data=payload, headers=headers)
|
|
with urllib.request.urlopen(req, timeout=120) as resp:
|
|
data = json.load(resp)
|
|
|
|
return [d["embedding"] for d in sorted(data["data"], key=lambda d: d["index"])]
|
|
|
|
|
|
def format_vector_sql(vec: list[float]) -> str:
|
|
return "[" + ",".join(f"{v:.8f}" for v in vec) + "]"
|
|
|
|
|
|
def main():
|
|
print("-- Generated embeddings for built-in knowledge")
|
|
print("-- Model: baai/bge-m3 via OpenRouter")
|
|
print("-- Dimensions: 1024")
|
|
print()
|
|
|
|
texts = [c["text"] for c in CHUNKS]
|
|
print(f"-- Embedding {len(texts)} chunks...", file=sys.stderr)
|
|
|
|
vectors = embed(texts)
|
|
print(f"-- Got {len(vectors)} embeddings", file=sys.stderr)
|
|
|
|
print("-- Embeddings for builtin_knowledge_embeddings")
|
|
print()
|
|
|
|
for chunk, vector in zip(CHUNKS, vectors):
|
|
content_hash = chunk["content_hash"]
|
|
vec_sql = format_vector_sql(vector)
|
|
print(f"INSERT INTO builtin_knowledge_embeddings (chunk_id, embedding, embedding_provider, embedding_model)")
|
|
print(f"SELECT c.id, '{vec_sql}'::vector, 'openrouter', 'baai/bge-m3'")
|
|
print(f"FROM builtin_knowledge_chunks c")
|
|
print(f"WHERE c.content_hash = '{content_hash}';")
|
|
print()
|
|
|
|
print("-- End of embeddings", file=sys.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|