Source index

scripts/evaluate.mjs

import {runAgent} from '../src/agent.mjs';
import {readFile, mkdir, writeFile} from 'node:fs/promises';
const cases = JSON.parse(await readFile(new URL('../data/golden-questions.json', import.meta.url), 'utf8'));
const root = new URL('../public/runs/', import.meta.url);
await mkdir(root, {recursive: true});
const records = [], manifest = [];
for (const item of cases) {
  const result = await runAgent(item.question);
  const file = `${item.id}.json`;
  await writeFile(new URL(file, root), JSON.stringify(result, null, 2) + '\n');
  const retrieval = JSON.stringify(result.trace.filter(step => step.kind === 'context-tool' && !step.result?.isError).map(step => step.result));
  const expectedDisposition = item.expectedDisposition === 'abstain' ? 'abstained' : 'complete';
  const checks = {
    completedExpectedDisposition: result.status === expectedDisposition,
    actualModelCalls: result.usage.modelCalls > 0,
    actualContextCalls: result.trace.some(step => step.kind === 'context-tool'),
    expectedGuidesRetrieved: (item.expectedGuideIds || []).every(id => retrieval.includes(id)),
    requiredSourcesCited: (item.requiredCitationUrls || []).every(url => result.citations.cited.includes(url)),
    noUnretrievedURLs: result.citations.unsupported.length === 0,
  };
  records.push({id: item.id, question: item.question, status: result.status, checks, structuralPass: Object.values(checks).every(Boolean), conceptsForManualReview: item.expectedConcepts, forbiddenClaimsForManualReview: item.forbiddenClaims, sourceFile: file});
  manifest.push({file, label: item.id.replace(/[-_]/g, ' ')});
  console.log(`${item.id}: ${result.status}; structural=${Object.values(checks).every(Boolean)}; ${result.usage.modelCalls} model calls.`);
}
const summary = {evaluatedAt: new Date().toISOString(), cases: records.length, structuralPasses: records.filter(item => item.structuralPass).length, method: 'Actual fresh local model + live Context MCP per case. Structural checks measure tool retrieval/document IDs/citation URL membership/disposition, not factual entailment. Concepts and forbidden claims need manual trace review.', records};
await writeFile(new URL('../data/evaluation.json', import.meta.url), JSON.stringify(summary, null, 2) + '\n');
await writeFile(new URL('index.json', root), JSON.stringify({recordedRuns: true, runs: manifest}, null, 2) + '\n');