Add daily runtime spend tracking
--- Build: pass | Tests: FAIL — Tests 1 failed | 2025 passed (2026)
This commit is contained in:
parent
155ab02c49
commit
a1634cbfbc
11 changed files with 466 additions and 154 deletions
|
|
@ -285,7 +285,16 @@ describe('extractRuntimeFromPiSession', () => {
|
|||
expect(extractRuntimeFromPiSession(sessionFile)).toEqual({
|
||||
actualProvider: 'zai',
|
||||
actualModel: 'glm-5',
|
||||
tokensUsed: 1234,
|
||||
totalTokens: 1234,
|
||||
inputTokens: null,
|
||||
outputTokens: null,
|
||||
cacheReadTokens: null,
|
||||
cacheWriteTokens: null,
|
||||
costInputUsd: null,
|
||||
costOutputUsd: null,
|
||||
costCacheReadUsd: null,
|
||||
costCacheWriteUsd: null,
|
||||
costTotalUsd: null,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -304,7 +313,16 @@ describe('extractRuntimeFromPiSession', () => {
|
|||
expect(extractRuntimeFromPiSession(sessionFile)).toEqual({
|
||||
actualProvider: 'openrouter',
|
||||
actualModel: 'openai/gpt-5.1-codex',
|
||||
tokensUsed: null,
|
||||
totalTokens: null,
|
||||
inputTokens: null,
|
||||
outputTokens: null,
|
||||
cacheReadTokens: null,
|
||||
cacheWriteTokens: null,
|
||||
costInputUsd: null,
|
||||
costOutputUsd: null,
|
||||
costCacheReadUsd: null,
|
||||
costCacheWriteUsd: null,
|
||||
costTotalUsd: null,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -312,7 +330,16 @@ describe('extractRuntimeFromPiSession', () => {
|
|||
expect(extractRuntimeFromPiSession(path.join(tmpDir, 'missing.jsonl'))).toEqual({
|
||||
actualProvider: null,
|
||||
actualModel: null,
|
||||
tokensUsed: null,
|
||||
totalTokens: null,
|
||||
inputTokens: null,
|
||||
outputTokens: null,
|
||||
cacheReadTokens: null,
|
||||
cacheWriteTokens: null,
|
||||
costInputUsd: null,
|
||||
costOutputUsd: null,
|
||||
costCacheReadUsd: null,
|
||||
costCacheWriteUsd: null,
|
||||
costTotalUsd: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -42,6 +42,11 @@ import { incCounter, incLabeledCounter, registerGauge } from './metrics.js';
|
|||
import { compactSession } from './session-compaction.js';
|
||||
import { getImportantMemories, searchMemories } from './memory-pg.js';
|
||||
import { buildRuntimeManifest, renderRuntimeManifestSummary } from './runtime-manifest.js';
|
||||
import {
|
||||
emptyPiRuntimeUsage,
|
||||
extractPiRuntimeUsageFromJsonLines,
|
||||
type PiRuntimeUsage,
|
||||
} from './pi-usage.js';
|
||||
|
||||
const METRICS_PREFIX = `${TENANT_ID}_`;
|
||||
import { logger } from './logger.js';
|
||||
|
|
@ -88,6 +93,7 @@ export interface AgentOutput {
|
|||
actualProvider?: string | null;
|
||||
actualModel?: string | null;
|
||||
tokensUsed?: number | null;
|
||||
runtimeUsage?: PiRuntimeUsage | null;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
|
|
@ -176,84 +182,12 @@ function createFreshSessionFile(
|
|||
}
|
||||
}
|
||||
|
||||
export function extractRuntimeFromPiSession(sessionFile: string): {
|
||||
actualProvider: string | null;
|
||||
actualModel: string | null;
|
||||
tokensUsed: number | null;
|
||||
} {
|
||||
export function extractRuntimeFromPiSession(sessionFile: string): PiRuntimeUsage {
|
||||
try {
|
||||
const raw = fs.readFileSync(sessionFile, 'utf-8');
|
||||
if (!raw.trim()) {
|
||||
return {
|
||||
actualProvider: null,
|
||||
actualModel: null,
|
||||
tokensUsed: null,
|
||||
};
|
||||
}
|
||||
|
||||
const lines = raw.split('\n').filter(Boolean);
|
||||
let actualProvider: string | null = null;
|
||||
let actualModel: string | null = null;
|
||||
let tokensUsed: number | null = null;
|
||||
|
||||
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
||||
try {
|
||||
const parsed = JSON.parse(lines[i]) as {
|
||||
type?: string;
|
||||
provider?: unknown;
|
||||
modelId?: unknown;
|
||||
message?: {
|
||||
role?: unknown;
|
||||
provider?: unknown;
|
||||
model?: unknown;
|
||||
usage?: { totalTokens?: unknown };
|
||||
};
|
||||
};
|
||||
|
||||
const message = parsed.message;
|
||||
if (
|
||||
message?.role === 'assistant' &&
|
||||
(typeof message.provider === 'string' ||
|
||||
typeof message.model === 'string' ||
|
||||
typeof message.usage?.totalTokens === 'number')
|
||||
) {
|
||||
actualProvider =
|
||||
typeof message.provider === 'string' ? message.provider : null;
|
||||
actualModel = typeof message.model === 'string' ? message.model : null;
|
||||
tokensUsed =
|
||||
typeof message.usage?.totalTokens === 'number'
|
||||
? message.usage.totalTokens
|
||||
: null;
|
||||
break;
|
||||
}
|
||||
|
||||
if (
|
||||
parsed.type === 'model_change' &&
|
||||
(!actualProvider || !actualModel)
|
||||
) {
|
||||
if (!actualProvider && typeof parsed.provider === 'string') {
|
||||
actualProvider = parsed.provider;
|
||||
}
|
||||
if (!actualModel && typeof parsed.modelId === 'string') {
|
||||
actualModel = parsed.modelId;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
actualProvider,
|
||||
actualModel,
|
||||
tokensUsed,
|
||||
};
|
||||
return extractPiRuntimeUsageFromJsonLines(raw);
|
||||
} catch {
|
||||
return {
|
||||
actualProvider: null,
|
||||
actualModel: null,
|
||||
tokensUsed: null,
|
||||
};
|
||||
return emptyPiRuntimeUsage();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -745,11 +679,7 @@ export async function runJailAgent(
|
|||
: newestSessionFile(sessionDir);
|
||||
const runtime = sessionFile
|
||||
? extractRuntimeFromPiSession(path.join(sessionDir, sessionFile))
|
||||
: {
|
||||
actualProvider: null,
|
||||
actualModel: null,
|
||||
tokensUsed: null,
|
||||
};
|
||||
: emptyPiRuntimeUsage();
|
||||
|
||||
// Detect provider cap errors and trip the cooldown so subsequent runs
|
||||
// (this process or the next) skip the capped provider until reset.
|
||||
|
|
@ -786,7 +716,8 @@ export async function runJailAgent(
|
|||
newSessionId: sessionFile,
|
||||
actualProvider: runtime.actualProvider,
|
||||
actualModel: runtime.actualModel,
|
||||
tokensUsed: runtime.tokensUsed,
|
||||
tokensUsed: runtime.totalTokens,
|
||||
runtimeUsage: runtime,
|
||||
},
|
||||
code,
|
||||
);
|
||||
|
|
@ -799,7 +730,8 @@ export async function runJailAgent(
|
|||
newSessionId: sessionFile,
|
||||
actualProvider: runtime.actualProvider,
|
||||
actualModel: runtime.actualModel,
|
||||
tokensUsed: runtime.tokensUsed,
|
||||
tokensUsed: runtime.totalTokens,
|
||||
runtimeUsage: runtime,
|
||||
},
|
||||
code,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import {
|
|||
copySkills,
|
||||
CONTROLPLANE_SCHEMA_SQL,
|
||||
VALID_EVENT_TYPES,
|
||||
getAgentSpendAnalytics,
|
||||
} from './controlplane-db.js';
|
||||
|
||||
// Re-imported for new-function tests only — vi.fn() mocks the pool
|
||||
|
|
@ -254,6 +255,38 @@ describe('VALID_EVENT_TYPES', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('getAgentSpendAnalytics', () => {
|
||||
it('maps summed cost rows into numbers', async () => {
|
||||
const pool = {
|
||||
query: vi.fn().mockResolvedValue({
|
||||
rows: [
|
||||
{
|
||||
agent_id: 'mevy',
|
||||
cost_today_usd: '0.02471',
|
||||
max_cost_usd: '0.0124',
|
||||
priced_runs_today: '4',
|
||||
unpriced_runs_today: '1',
|
||||
last_provider: 'deepseek',
|
||||
last_model: 'deepseek-chat',
|
||||
},
|
||||
],
|
||||
}),
|
||||
} as any;
|
||||
|
||||
await expect(getAgentSpendAnalytics(pool)).resolves.toEqual([
|
||||
{
|
||||
agent_id: 'mevy',
|
||||
cost_today_usd: 0.02471,
|
||||
max_cost_usd: 0.0124,
|
||||
priced_runs_today: 4,
|
||||
unpriced_runs_today: 1,
|
||||
last_provider: 'deepseek',
|
||||
last_model: 'deepseek-chat',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CONTROLPLANE_SCHEMA_SQL — Phase 6 additions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -95,6 +95,16 @@ export interface AgentTokenAnalytics {
|
|||
last_model: string | null;
|
||||
}
|
||||
|
||||
export interface AgentSpendAnalytics {
|
||||
agent_id: string;
|
||||
cost_today_usd: number;
|
||||
max_cost_usd: number;
|
||||
priced_runs_today: number;
|
||||
unpriced_runs_today: number;
|
||||
last_provider: string | null;
|
||||
last_model: string | null;
|
||||
}
|
||||
|
||||
// ── Default agent definitions ──────────────────────────────────────────────
|
||||
|
||||
export function getDefaultAgents(
|
||||
|
|
@ -814,6 +824,78 @@ export async function getAgentTokenAnalytics(
|
|||
}));
|
||||
}
|
||||
|
||||
export async function getAgentSpendAnalytics(
|
||||
pool: pg.Pool,
|
||||
): Promise<AgentSpendAnalytics[]> {
|
||||
const result = await pool.query<{
|
||||
agent_id: string;
|
||||
cost_today_usd: string;
|
||||
max_cost_usd: string;
|
||||
priced_runs_today: string;
|
||||
unpriced_runs_today: string;
|
||||
last_provider: string | null;
|
||||
last_model: string | null;
|
||||
}>(`
|
||||
WITH recent AS (
|
||||
SELECT
|
||||
agent_id,
|
||||
created_at,
|
||||
NULLIF(
|
||||
COALESCE(
|
||||
payload->>'actual_provider',
|
||||
payload->>'provider',
|
||||
payload->>'configured_provider',
|
||||
''
|
||||
),
|
||||
''
|
||||
) AS provider,
|
||||
NULLIF(
|
||||
COALESCE(
|
||||
payload->>'actual_model',
|
||||
payload->>'model',
|
||||
payload->>'configured_model',
|
||||
''
|
||||
),
|
||||
''
|
||||
) AS model,
|
||||
CASE
|
||||
WHEN COALESCE(payload->>'cost_total_usd', '') = '' THEN NULL
|
||||
ELSE (payload->>'cost_total_usd')::double precision
|
||||
END AS cost_total_usd,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY agent_id
|
||||
ORDER BY created_at DESC, id DESC
|
||||
) AS rn
|
||||
FROM agent_activity
|
||||
WHERE created_at >= date_trunc('day', now())
|
||||
AND agent_id IS NOT NULL
|
||||
)
|
||||
SELECT
|
||||
agent_id,
|
||||
COALESCE(SUM(cost_total_usd), 0)::text AS cost_today_usd,
|
||||
COALESCE(MAX(cost_total_usd), 0)::text AS max_cost_usd,
|
||||
SUM(CASE WHEN cost_total_usd IS NOT NULL THEN 1 ELSE 0 END)::text AS priced_runs_today,
|
||||
SUM(CASE WHEN cost_total_usd IS NULL THEN 1 ELSE 0 END)::text AS unpriced_runs_today,
|
||||
MAX(CASE WHEN rn = 1 THEN provider END) AS last_provider,
|
||||
MAX(CASE WHEN rn = 1 THEN model END) AS last_model
|
||||
FROM recent
|
||||
GROUP BY agent_id
|
||||
HAVING COALESCE(SUM(cost_total_usd), 0) > 0
|
||||
OR SUM(CASE WHEN cost_total_usd IS NOT NULL THEN 1 ELSE 0 END) > 0
|
||||
ORDER BY COALESCE(SUM(cost_total_usd), 0) DESC, agent_id ASC
|
||||
`);
|
||||
|
||||
return result.rows.map((row) => ({
|
||||
agent_id: row.agent_id,
|
||||
cost_today_usd: Number(row.cost_today_usd || 0),
|
||||
max_cost_usd: Number(row.max_cost_usd || 0),
|
||||
priced_runs_today: Number(row.priced_runs_today || 0),
|
||||
unpriced_runs_today: Number(row.unpriced_runs_today || 0),
|
||||
last_provider: row.last_provider,
|
||||
last_model: row.last_model,
|
||||
}));
|
||||
}
|
||||
|
||||
// ── Task mutation queries ────────────────────────────────────────────────────
|
||||
|
||||
export async function updateTaskStatus(
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ import {
|
|||
formatSessionContext,
|
||||
pruneOldEntries,
|
||||
} from './agent-session.js';
|
||||
import { extractPiRuntimeUsageFromJsonLines } from './pi-usage.js';
|
||||
import { loadSkillsCatalog, matchTaskToSkill } from './skills-discovery.js';
|
||||
import { checkAgentTaskCapability } from './agent-capabilities.js';
|
||||
import { syncModelCatalog, formatModelDiff } from './model-catalog.js';
|
||||
|
|
@ -273,6 +274,7 @@ async function runPiTask(opts: {
|
|||
tokensUsed: number;
|
||||
actualProvider: string | null;
|
||||
actualModel: string | null;
|
||||
costTotalUsd: number | null;
|
||||
}> {
|
||||
const runConfig = buildControlplaneRunCommand({
|
||||
agentId: opts.agentId,
|
||||
|
|
@ -318,6 +320,7 @@ async function runPiTask(opts: {
|
|||
tokensUsed,
|
||||
actualProvider: runtime.actualProvider,
|
||||
actualModel: runtime.actualModel,
|
||||
costTotalUsd: runtime.costTotalUsd,
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -328,6 +331,7 @@ async function runPiTask(opts: {
|
|||
tokensUsed: 1,
|
||||
actualProvider: null,
|
||||
actualModel: null,
|
||||
costTotalUsd: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -336,37 +340,14 @@ async function runPiTask(opts: {
|
|||
function extractActualRuntimeFromPiOutput(stdout: string): {
|
||||
actualProvider: string | null;
|
||||
actualModel: string | null;
|
||||
costTotalUsd: number | null;
|
||||
} {
|
||||
const lines = stdout.split('\n').filter(Boolean);
|
||||
let actualProvider: string | null = null;
|
||||
let actualModel: string | null = null;
|
||||
for (let i = lines.length - 1; i >= 0; i--) {
|
||||
try {
|
||||
const parsed = JSON.parse(lines[i]);
|
||||
const message = parsed?.message;
|
||||
if (!actualProvider && typeof message?.provider === 'string') {
|
||||
actualProvider = message.provider;
|
||||
}
|
||||
if (!actualModel && typeof message?.model === 'string') {
|
||||
actualModel = message.model;
|
||||
}
|
||||
if (
|
||||
(!actualProvider || !actualModel) &&
|
||||
parsed?.type === 'model_change'
|
||||
) {
|
||||
if (!actualProvider && typeof parsed.provider === 'string') {
|
||||
actualProvider = parsed.provider;
|
||||
}
|
||||
if (!actualModel && typeof parsed.modelId === 'string') {
|
||||
actualModel = parsed.modelId;
|
||||
}
|
||||
}
|
||||
if (actualProvider && actualModel) break;
|
||||
} catch {
|
||||
// Not valid JSON, skip
|
||||
}
|
||||
}
|
||||
return { actualProvider, actualModel };
|
||||
const runtime = extractPiRuntimeUsageFromJsonLines(stdout);
|
||||
return {
|
||||
actualProvider: runtime.actualProvider,
|
||||
actualModel: runtime.actualModel,
|
||||
costTotalUsd: runtime.costTotalUsd,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Codex runner ──────────────────────────────────────────────────────────
|
||||
|
|
@ -581,6 +562,7 @@ export async function runAgentHeartbeat(
|
|||
let output: string | null = null;
|
||||
let actualProvider: string | null = null;
|
||||
let actualModel: string | null = null;
|
||||
let costTotalUsd: number | null = null;
|
||||
const { configuredProvider, configuredModel } =
|
||||
resolveHeartbeatRuntimePreference();
|
||||
const effectiveRuntime = applyFallback({
|
||||
|
|
@ -728,6 +710,7 @@ export async function runAgentHeartbeat(
|
|||
error = piResult.error;
|
||||
actualProvider = piResult.actualProvider;
|
||||
actualModel = piResult.actualModel;
|
||||
costTotalUsd = piResult.costTotalUsd;
|
||||
}
|
||||
|
||||
await recordTokenSpend(pool, agentId, tokensUsed);
|
||||
|
|
@ -754,6 +737,7 @@ export async function runAgentHeartbeat(
|
|||
effective_model: effectiveModel || null,
|
||||
actual_provider: actualProvider,
|
||||
actual_model: actualModel,
|
||||
cost_total_usd: costTotalUsd,
|
||||
},
|
||||
tokens_used: tokensUsed,
|
||||
});
|
||||
|
|
@ -797,6 +781,7 @@ export async function runAgentHeartbeat(
|
|||
effective_model: effectiveModel || null,
|
||||
actual_provider: actualProvider,
|
||||
actual_model: actualModel,
|
||||
cost_total_usd: costTotalUsd,
|
||||
// Store the tail so we keep the most user-relevant part (often the final answer),
|
||||
// without bloating the activity log with large runner transcripts.
|
||||
output: output ? output.slice(-2000) : null,
|
||||
|
|
|
|||
|
|
@ -781,6 +781,11 @@ async function runAgent(
|
|||
override_model: group.jailConfig?.model || null,
|
||||
actual_provider: output.actualProvider ?? null,
|
||||
actual_model: output.actualModel ?? null,
|
||||
usage_input_tokens: output.runtimeUsage?.inputTokens ?? null,
|
||||
usage_output_tokens: output.runtimeUsage?.outputTokens ?? null,
|
||||
usage_cache_read_tokens: output.runtimeUsage?.cacheReadTokens ?? null,
|
||||
usage_cache_write_tokens: output.runtimeUsage?.cacheWriteTokens ?? null,
|
||||
cost_total_usd: output.runtimeUsage?.costTotalUsd ?? null,
|
||||
},
|
||||
tokens_used: tokensUsed,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import fs from 'fs';
|
|||
import path from 'path';
|
||||
|
||||
import { logger } from './logger.js';
|
||||
import { extractPiRuntimeUsageFromJsonLines } from './pi-usage.js';
|
||||
import { TMP_DIR } from './config.js';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
|
@ -380,47 +381,23 @@ function extractPiRuntimeFromOutput(output: string | null): {
|
|||
tokensUsed: number;
|
||||
actualProvider: string | null;
|
||||
actualModel: string | null;
|
||||
costTotalUsd: number | null;
|
||||
} {
|
||||
if (!output) {
|
||||
return { tokensUsed: 1, actualProvider: null, actualModel: null };
|
||||
return {
|
||||
tokensUsed: 1,
|
||||
actualProvider: null,
|
||||
actualModel: null,
|
||||
costTotalUsd: null,
|
||||
};
|
||||
}
|
||||
const lines = output.split('\n').filter(Boolean);
|
||||
let actualProvider: string | null = null;
|
||||
let actualModel: string | null = null;
|
||||
let tokensUsed = Math.max(1, Math.ceil(output.length / 4));
|
||||
for (let i = lines.length - 1; i >= 0; i--) {
|
||||
try {
|
||||
const parsed = JSON.parse(lines[i]);
|
||||
const message = parsed?.message;
|
||||
const usage = message?.usage;
|
||||
if (!actualProvider && typeof message?.provider === 'string') {
|
||||
actualProvider = message.provider;
|
||||
}
|
||||
if (!actualModel && typeof message?.model === 'string') {
|
||||
actualModel = message.model;
|
||||
}
|
||||
if (
|
||||
(!actualProvider || !actualModel) &&
|
||||
parsed?.type === 'model_change'
|
||||
) {
|
||||
if (!actualProvider && typeof parsed.provider === 'string') {
|
||||
actualProvider = parsed.provider;
|
||||
}
|
||||
if (!actualModel && typeof parsed.modelId === 'string') {
|
||||
actualModel = parsed.modelId;
|
||||
}
|
||||
}
|
||||
if (usage?.totalTokens && typeof usage.totalTokens === 'number') {
|
||||
tokensUsed = usage.totalTokens;
|
||||
}
|
||||
if (actualProvider && actualModel && usage?.totalTokens) {
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// Not valid JSON
|
||||
}
|
||||
}
|
||||
return { tokensUsed, actualProvider, actualModel };
|
||||
const runtime = extractPiRuntimeUsageFromJsonLines(output);
|
||||
return {
|
||||
tokensUsed: runtime.totalTokens ?? Math.max(1, Math.ceil(output.length / 4)),
|
||||
actualProvider: runtime.actualProvider,
|
||||
actualModel: runtime.actualModel,
|
||||
costTotalUsd: runtime.costTotalUsd,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Convenience: run aider inside a jail ───────────────────────────────────
|
||||
|
|
|
|||
75
src/pi-usage.test.ts
Normal file
75
src/pi-usage.test.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { extractPiRuntimeUsageFromJsonLines } from './pi-usage.js';
|
||||
|
||||
describe('extractPiRuntimeUsageFromJsonLines', () => {
|
||||
it('parses provider, model, tokens, and cost from assistant usage', () => {
|
||||
const raw = [
|
||||
JSON.stringify({
|
||||
type: 'model_change',
|
||||
provider: 'deepseek',
|
||||
modelId: 'deepseek-chat',
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'message',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
provider: 'deepseek',
|
||||
model: 'deepseek-chat',
|
||||
usage: {
|
||||
input: 15,
|
||||
output: 24,
|
||||
cacheRead: 12416,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 12455,
|
||||
cost: {
|
||||
input: 0.0000021,
|
||||
output: 0.00000672,
|
||||
cacheRead: 0.000347648,
|
||||
cacheWrite: 0,
|
||||
total: 0.000356468,
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
].join('\n');
|
||||
|
||||
expect(extractPiRuntimeUsageFromJsonLines(raw)).toEqual({
|
||||
actualProvider: 'deepseek',
|
||||
actualModel: 'deepseek-chat',
|
||||
totalTokens: 12455,
|
||||
inputTokens: 15,
|
||||
outputTokens: 24,
|
||||
cacheReadTokens: 12416,
|
||||
cacheWriteTokens: 0,
|
||||
costInputUsd: 0.0000021,
|
||||
costOutputUsd: 0.00000672,
|
||||
costCacheReadUsd: 0.000347648,
|
||||
costCacheWriteUsd: 0,
|
||||
costTotalUsd: 0.000356468,
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to model_change metadata when assistant usage is absent', () => {
|
||||
const raw = JSON.stringify({
|
||||
type: 'model_change',
|
||||
provider: 'openrouter',
|
||||
modelId: 'openai/o3',
|
||||
});
|
||||
|
||||
expect(extractPiRuntimeUsageFromJsonLines(raw)).toEqual({
|
||||
actualProvider: 'openrouter',
|
||||
actualModel: 'openai/o3',
|
||||
totalTokens: null,
|
||||
inputTokens: null,
|
||||
outputTokens: null,
|
||||
cacheReadTokens: null,
|
||||
cacheWriteTokens: null,
|
||||
costInputUsd: null,
|
||||
costOutputUsd: null,
|
||||
costCacheReadUsd: null,
|
||||
costCacheWriteUsd: null,
|
||||
costTotalUsd: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
111
src/pi-usage.ts
Normal file
111
src/pi-usage.ts
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
export interface PiRuntimeUsage {
|
||||
actualProvider: string | null;
|
||||
actualModel: string | null;
|
||||
totalTokens: number | null;
|
||||
inputTokens: number | null;
|
||||
outputTokens: number | null;
|
||||
cacheReadTokens: number | null;
|
||||
cacheWriteTokens: number | null;
|
||||
costInputUsd: number | null;
|
||||
costOutputUsd: number | null;
|
||||
costCacheReadUsd: number | null;
|
||||
costCacheWriteUsd: number | null;
|
||||
costTotalUsd: number | null;
|
||||
}
|
||||
|
||||
function readNumber(value: unknown): number | null {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
export function emptyPiRuntimeUsage(): PiRuntimeUsage {
|
||||
return {
|
||||
actualProvider: null,
|
||||
actualModel: null,
|
||||
totalTokens: null,
|
||||
inputTokens: null,
|
||||
outputTokens: null,
|
||||
cacheReadTokens: null,
|
||||
cacheWriteTokens: null,
|
||||
costInputUsd: null,
|
||||
costOutputUsd: null,
|
||||
costCacheReadUsd: null,
|
||||
costCacheWriteUsd: null,
|
||||
costTotalUsd: null,
|
||||
};
|
||||
}
|
||||
|
||||
export function extractPiRuntimeUsageFromJsonLines(raw: string): PiRuntimeUsage {
|
||||
if (!raw.trim()) return emptyPiRuntimeUsage();
|
||||
|
||||
const lines = raw.split('\n').filter(Boolean);
|
||||
const usage = emptyPiRuntimeUsage();
|
||||
|
||||
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
||||
try {
|
||||
const parsed = JSON.parse(lines[i]) as {
|
||||
type?: string;
|
||||
provider?: unknown;
|
||||
modelId?: unknown;
|
||||
message?: {
|
||||
role?: unknown;
|
||||
provider?: unknown;
|
||||
model?: unknown;
|
||||
usage?: {
|
||||
input?: unknown;
|
||||
output?: unknown;
|
||||
cacheRead?: unknown;
|
||||
cacheWrite?: unknown;
|
||||
totalTokens?: unknown;
|
||||
cost?: {
|
||||
input?: unknown;
|
||||
output?: unknown;
|
||||
cacheRead?: unknown;
|
||||
cacheWrite?: unknown;
|
||||
total?: unknown;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const message = parsed.message;
|
||||
if (
|
||||
message?.role === 'assistant' &&
|
||||
(typeof message.provider === 'string' ||
|
||||
typeof message.model === 'string' ||
|
||||
typeof message.usage?.totalTokens === 'number')
|
||||
) {
|
||||
usage.actualProvider =
|
||||
typeof message.provider === 'string' ? message.provider : null;
|
||||
usage.actualModel =
|
||||
typeof message.model === 'string' ? message.model : null;
|
||||
usage.totalTokens = readNumber(message.usage?.totalTokens);
|
||||
usage.inputTokens = readNumber(message.usage?.input);
|
||||
usage.outputTokens = readNumber(message.usage?.output);
|
||||
usage.cacheReadTokens = readNumber(message.usage?.cacheRead);
|
||||
usage.cacheWriteTokens = readNumber(message.usage?.cacheWrite);
|
||||
usage.costInputUsd = readNumber(message.usage?.cost?.input);
|
||||
usage.costOutputUsd = readNumber(message.usage?.cost?.output);
|
||||
usage.costCacheReadUsd = readNumber(message.usage?.cost?.cacheRead);
|
||||
usage.costCacheWriteUsd = readNumber(message.usage?.cost?.cacheWrite);
|
||||
usage.costTotalUsd = readNumber(message.usage?.cost?.total);
|
||||
break;
|
||||
}
|
||||
|
||||
if (
|
||||
parsed.type === 'model_change' &&
|
||||
(!usage.actualProvider || !usage.actualModel)
|
||||
) {
|
||||
if (!usage.actualProvider && typeof parsed.provider === 'string') {
|
||||
usage.actualProvider = parsed.provider;
|
||||
}
|
||||
if (!usage.actualModel && typeof parsed.modelId === 'string') {
|
||||
usage.actualModel = parsed.modelId;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return usage;
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import path from 'path';
|
|||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
buildAiTokenBriefLines,
|
||||
buildAiSpendBriefLines,
|
||||
buildOwnershipSectionLines,
|
||||
readCurrentCommitHash,
|
||||
formatTimestamp,
|
||||
|
|
@ -16,6 +17,7 @@ import {
|
|||
formatMemorySectionLines,
|
||||
tenantContextWarning,
|
||||
} from './startup-report.js';
|
||||
import type { AgentSpendAnalytics } from './controlplane-db.js';
|
||||
|
||||
describe('tenantContextWarning', () => {
|
||||
it('returns null when tenantId is a registered tenant distinct from the platform', () => {
|
||||
|
|
@ -88,6 +90,37 @@ describe('buildAiTokenBriefLines', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('buildAiSpendBriefLines', () => {
|
||||
it('summarizes recorded spend and gaps compactly', () => {
|
||||
const lines = buildAiSpendBriefLines([
|
||||
{
|
||||
agent_id: 'mevy',
|
||||
cost_today_usd: 0.02471,
|
||||
max_cost_usd: 0.0124,
|
||||
priced_runs_today: 4,
|
||||
unpriced_runs_today: 1,
|
||||
last_provider: 'deepseek',
|
||||
last_model: 'deepseek-chat',
|
||||
},
|
||||
{
|
||||
agent_id: 'sysadmin',
|
||||
cost_today_usd: 0.0012,
|
||||
max_cost_usd: 0.0012,
|
||||
priced_runs_today: 1,
|
||||
unpriced_runs_today: 0,
|
||||
last_provider: 'deepseek',
|
||||
last_model: 'deepseek-chat',
|
||||
},
|
||||
] satisfies AgentSpendAnalytics[]);
|
||||
|
||||
expect(lines).toContain('Recorded spend today: $0.0259');
|
||||
expect(lines).toContain('- mevy: $0.0247 (deepseek · deepseek-chat)');
|
||||
expect(lines).toContain('- sysadmin: $0.00120 (deepseek · deepseek-chat)');
|
||||
expect(lines).toContain('Top spend spike: mevy · $0.0124');
|
||||
expect(lines).toContain('Spend gaps: 1 run(s) missing explicit cost');
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildOwnershipSectionLines', () => {
|
||||
it('includes controlplane addresses and tenant sites', () => {
|
||||
const lines = buildOwnershipSectionLines({
|
||||
|
|
|
|||
|
|
@ -36,7 +36,9 @@ import { getOperatorDashboardUrl } from './controlplane-links.js';
|
|||
import { getPool as getMemoryPool } from './memory-pg.js';
|
||||
import {
|
||||
getAgentTokenAnalytics,
|
||||
getAgentSpendAnalytics,
|
||||
type AgentTokenAnalytics,
|
||||
type AgentSpendAnalytics,
|
||||
} from './controlplane-db.js';
|
||||
import { formatSttGuardLine } from './stt-guard.js';
|
||||
import { buildSurfaceInventory } from './surface-inventory.js';
|
||||
|
|
@ -352,6 +354,49 @@ export function buildAiTokenBriefLines(
|
|||
return lines;
|
||||
}
|
||||
|
||||
function formatUsd(amount: number): string {
|
||||
if (amount >= 1) return `$${amount.toFixed(2)}`;
|
||||
if (amount >= 0.1) return `$${amount.toFixed(3)}`;
|
||||
if (amount >= 0.01) return `$${amount.toFixed(4)}`;
|
||||
return `$${amount.toFixed(5)}`;
|
||||
}
|
||||
|
||||
export function buildAiSpendBriefLines(
|
||||
analytics: AgentSpendAnalytics[],
|
||||
): string[] {
|
||||
if (analytics.length === 0) return [];
|
||||
|
||||
const total = analytics.reduce((sum, row) => sum + row.cost_today_usd, 0);
|
||||
const totalUnpricedRuns = analytics.reduce(
|
||||
(sum, row) => sum + row.unpriced_runs_today,
|
||||
0,
|
||||
);
|
||||
|
||||
const lines = [`Recorded spend today: ${formatUsd(total)}`];
|
||||
for (const row of analytics.slice(0, 3)) {
|
||||
const runtime =
|
||||
row.last_provider && row.last_model
|
||||
? ` (${row.last_provider} · ${row.last_model})`
|
||||
: '';
|
||||
lines.push(`- ${row.agent_id}: ${formatUsd(row.cost_today_usd)}${runtime}`);
|
||||
}
|
||||
|
||||
const topRun = analytics.reduce<AgentSpendAnalytics | null>((best, row) => {
|
||||
if (!best || row.max_cost_usd > best.max_cost_usd) return row;
|
||||
return best;
|
||||
}, null);
|
||||
if (topRun && topRun.max_cost_usd > 0) {
|
||||
lines.push(
|
||||
`Top spend spike: ${topRun.agent_id} · ${formatUsd(topRun.max_cost_usd)}`,
|
||||
);
|
||||
}
|
||||
if (totalUnpricedRuns > 0) {
|
||||
lines.push(`Spend gaps: ${totalUnpricedRuns} run(s) missing explicit cost`);
|
||||
}
|
||||
|
||||
return lines;
|
||||
}
|
||||
|
||||
|
||||
export function formatBytesGb(bytes: number, digits: number): string {
|
||||
const gb = bytes / 1024 / 1024 / 1024;
|
||||
|
|
@ -759,11 +804,18 @@ export async function buildStartupReportWithDiagnostics(): Promise<string> {
|
|||
report.splice(insertAt, 0, line);
|
||||
report.splice(insertAt + 1, 0, formatSttGuardLine('tg:ops'));
|
||||
try {
|
||||
const analytics = await getAgentTokenAnalytics(getMemoryPool());
|
||||
const [analytics, spendAnalytics] = await Promise.all([
|
||||
getAgentTokenAnalytics(getMemoryPool()),
|
||||
getAgentSpendAnalytics(getMemoryPool()),
|
||||
]);
|
||||
const tokenLines = buildAiTokenBriefLines(analytics);
|
||||
if (tokenLines.length > 0) {
|
||||
report.splice(insertAt + 2, 0, ...tokenLines);
|
||||
}
|
||||
const spendLines = buildAiSpendBriefLines(spendAnalytics);
|
||||
if (spendLines.length > 0) {
|
||||
report.splice(insertAt + 2 + tokenLines.length, 0, ...spendLines);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue