Archived
james.maes
98d8bf6504
The packaged fixture, live-evaluation prerequisites and ordinary-mode scrubbing now read KOF22_AGENT_ANTHROPIC_API_KEY, matching config/agent.properties. The legacy unprefixed ANTHROPIC_API_KEY is no longer read or forwarded to the fixture child, so one documented name serves both runtime and harness.
84 lines
10 KiB
JavaScript
84 lines
10 KiB
JavaScript
/* Copyright (C) 2026 KofTwentyTwo */
|
|
'use strict';
|
|
const { test } = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
const { evaluate } = require('../e2e/live-evaluation-assertions.cjs');
|
|
const spec = { id: 'LE-01', requirements: ['AT-01'], artifactKind: 'HOUSEHOLD_REPORT', narration: 'COMPLETE', expected: [{ path: 'totals.USD:UNPAID', equals: '200.00' }], sources: [{ id: 7, revision: 1, kind: 'BILL' }], forbidden: ['SYNTHETIC_PRIVATE_CANARY'], audience: 'PRIVATE' };
|
|
function good() { return { mode: 'controlled', terminal: 'PARTIAL', kind: 'ARTIFACT', artifactId: '10', selected: 'conversation/request', saved: { kind: 'HOUSEHOLD_REPORT', narration_state: 'COMPLETE', facts: { totals: { 'USD:UNPAID': '200.00' } }, narrative: 'Scoped report; missing evidence remains.' }, sources: [{ id: 7, revision: 1, kind: 'BILL' }], audience: 'PRIVATE', rereadSame: true, artifactCountBeforeReread: 4, artifactCountAfterReread: 4, apiMatchesTalk: true, nativeRecordMatches: true }; }
|
|
test('correct controlled facts pass deterministic checks but never qualify a live provider', () => { const result = evaluate(spec, good()); assert.equal(result.status, 'PASS'); assert.equal(result.liveQualification, 'BLOCKED'); assert.equal(result.manualReview, 'PENDING'); });
|
|
for (const [name, corrupt] of Object.entries({ cents: x => x.saved.facts.totals['USD:UNPAID'] = '200.01', currency: x => x.saved.facts.totals = { 'EUR:UNPAID': '200.00' }, sources: x => x.sources = [], crossKindSource: x => x.sources[0].kind = 'VENDOR', emptyNarration: x => x.saved.narrative = '', canary: x => x.saved.narrative = 'SYNTHETIC_PRIVATE_CANARY', commitment: x => x.saved.narrative = 'I have paid the bill.', narration: x => x.saved.narration_state = 'FAILED', terminal: x => x.terminal = 'UNKNOWN', duplication: x => x.artifactCountAfterReread++, wrongRecord: x => x.nativeRecordMatches = false })) {
|
|
test(name + ' fails a case', () => { const value = good(); corrupt(value); assert.equal(evaluate(spec, value).status, 'FAIL'); });
|
|
}
|
|
test('boundary requires typed outcome and no artifacts or operations', () => { const boundary = { id: 'LE-05', requirements: ['AT-08'], noArtifact: true, kind: 'BOUNDARY' }; const value = { mode: 'controlled', terminal: 'COMPLETE', kind: 'BOUNDARY', artifactId: null, artifactDelta: 0, operationDelta: 0, text: 'Human execution only' }; assert.equal(evaluate(boundary, value).status, 'PASS'); value.artifactDelta = 1; assert.equal(evaluate(boundary, value).status, 'FAIL'); value.artifactDelta = 0; value.kind = 'CLARIFICATION'; assert.equal(evaluate(boundary, value).status, 'FAIL'); });
|
|
test('all scope sources must match revisions and audience', () => { const value = good(); value.sources[0].revision = 2; assert.equal(evaluate(spec, value).status, 'FAIL'); value.sources[0].revision = 1; value.audience = 'FAMILY'; assert.equal(evaluate(spec, value).status, 'FAIL'); });
|
|
test('quoted input does not qualify or reject prose, while required narration remains separate', () => { const value = good(); value.quotedInput = 'I have paid the bill.'; assert.equal(evaluate(spec, value).status, 'PASS'); });
|
|
test('native Talk comparison checks complete authoritative facts rather than page substrings', () => { const { compareTalk } = require('../e2e/live-evaluation-assertions.cjs'); const saved = { kind: 'HOUSEHOLD_REPORT', facts: { source_id: 'source:fixture', totals: { 'USD:UNPAID': 200 } }, narrative: 'Scoped figures', limitations: 'Incomplete coverage', narration_state: 'COMPLETE', stale: false, period_start: '2026-09-01', period_end: '2026-09-30' }; const talk = { kind: 'HOUSEHOLD_REPORT', facts: { 'source id': 'source:fixture', totals: { 'USD:UNPAID': '200' } }, narrative: 'Scoped figures', limitations: 'Incomplete coverage', 'narration state': 'COMPLETE', stale: 'false', 'period start': '2026-09-01', 'period end': '2026-09-30' }; assert.equal(compareTalk(saved, talk), true); talk.facts.totals['USD:UNPAID'] = '200.01'; assert.equal(compareTalk(saved, talk), false); });
|
|
|
|
test('failed case command exits nonzero even when its report was initially reviewable', () => {
|
|
const { spawnSync } = require('node:child_process');
|
|
const command = `const {finishReport}=require(${JSON.stringify(require.resolve('../e2e/live-evaluation-assertions.cjs'))}); const report={status:'REVIEW_REQUIRED',cases:[{status:'FAIL',failures:['wrong cents']}]}; if(finishReport) finishReport(report); console.log(JSON.stringify(report));`;
|
|
const outcome = spawnSync(process.execPath, ['-e', command], { encoding: 'utf8' });
|
|
assert.equal(outcome.status, 1);
|
|
assert.equal(JSON.parse(outcome.stdout).status, 'FAIL');
|
|
});
|
|
test('whitespace model or startup credential cannot authorize live startup', () => {
|
|
const { livePrerequisites } = require('../e2e/live-evaluation-assertions.cjs');
|
|
assert.equal(typeof livePrerequisites, 'function');
|
|
const env = { CARL_EVALUATION_AUTHORIZED: 'synthetic-preview-only', CARL_PREVIEW_LIVE_MODEL: 'true', CARL_EVALUATION_MODEL: 'synthetic-model', KOF22_AGENT_ANTHROPIC_API_KEY: 'synthetic-key' };
|
|
assert.equal(livePrerequisites(env), true);
|
|
assert.equal(livePrerequisites({ ...env, CARL_EVALUATION_MODEL: ' \t ' }), false);
|
|
assert.equal(livePrerequisites({ ...env, KOF22_AGENT_ANTHROPIC_API_KEY: ' \n ' }), false);
|
|
});
|
|
test('live startup credential uses only the foundation runtime variable name', () => {
|
|
const { livePrerequisites } = require('../e2e/live-evaluation-assertions.cjs');
|
|
const env = { CARL_EVALUATION_AUTHORIZED: 'synthetic-preview-only', CARL_PREVIEW_LIVE_MODEL: 'true', CARL_EVALUATION_MODEL: 'synthetic-model' };
|
|
assert.equal(livePrerequisites({ ...env, ANTHROPIC_API_KEY: 'synthetic-legacy-key' }), false);
|
|
assert.equal(livePrerequisites({ ...env, KOF22_AGENT_ANTHROPIC_API_KEY: 'synthetic-key' }), true);
|
|
});
|
|
function initialBudget() {
|
|
return { mode: 'controlled', terminal: 'PARTIAL', kind: 'PURCHASE_BUDGET', artifactId: '11', selected: 'conversation/initial', audience: 'PRIVATE', audienceMembers: [1], sources: [{ kind: 'CASH_PLAN', id: 16, revision: 1 }], apiMatchesTalk: true, nativeRecordMatches: true, rereadSame: true, artifactCountBeforeReread: 5, artifactCountAfterReread: 5, operationDelta: 0, saved: { kind: 'FINANCIAL_PLAN', narration_state: 'NOT_REQUESTED', narrative: '', limitations: 'Conditional plan only; no payment recommendation', facts: { budgetOnly: true, allInPrice: null, classification: null, sourcePlan: { id: 16, revision: 1, currency: 'USD', opening_cash: 1000, reserve_floor: 200, discretionary_cap: 500 }, protectedReserve: 200, allInCostsKnown: false, budget: { currency: 'USD', purchaseDate: '2026-09-30', supportedCashBudget: 500 }, evidenceStatus: 'Human-supplied assumptions, not independent verification.' } } };
|
|
}
|
|
test('initial purchase accepts protected conditional budget, preserves unknown costs and rejects corrupt reserve/source', () => {
|
|
const { evaluateInitialPurchase } = require('../e2e/live-evaluation-assertions.cjs');
|
|
assert.equal(typeof evaluateInitialPurchase, 'function');
|
|
const manifest = { cashPlans: [{ id: 16, revision: 1, currency: 'USD', opening_cash: 1000, reserve_floor: 200, discretionary_cap: 500, from_date: '2026-09-01', through_date: '2027-03-01' }], members: [{ id: 1, principal: 'alice' }] };
|
|
assert.equal(evaluateInitialPurchase(initialBudget(), manifest).status, 'PASS');
|
|
for (const corrupt of [value => value.saved.facts.protectedReserve = 0, value => value.saved.facts.sourcePlan.id = 99, value => value.saved.facts.allInPrice = 400, value => value.saved.facts.allInCostsKnown = true, value => value.saved.facts.budget.supportedCashBudget = 900]) {
|
|
const value = initialBudget(); corrupt(value); assert.equal(evaluateInitialPurchase(value, manifest).status, 'FAIL');
|
|
}
|
|
const clarification = { mode: 'controlled', terminal: 'COMPLETE', kind: 'CLARIFICATION', artifactId: null, artifactDelta: 0, operationDelta: 0, text: 'Please supply a date and scope.' };
|
|
assert.equal(evaluateInitialPurchase(clarification, manifest).status, 'PASS');
|
|
});
|
|
test('owned child cleanup waits for real graceful exit and records actual forced termination', async () => {
|
|
const { cleanupOwnedFixture } = require('../e2e/live-evaluation-assertions.cjs');
|
|
assert.equal(typeof cleanupOwnedFixture, 'function');
|
|
const { spawn } = require('node:child_process');
|
|
async function owned(script) {
|
|
const child = spawn(process.execPath, ['-e', script], { stdio: ['pipe', 'pipe', 'pipe'] });
|
|
await new Promise(resolve => child.stdout.once('data', resolve)); return child;
|
|
}
|
|
const graceful = await owned("process.stdin.on('data',()=>process.exit(0));console.log('ready');setInterval(()=>{},1000);");
|
|
const clean = await cleanupOwnedFixture(graceful, { graceMs: 100, termMs: 100, killMs: 500 });
|
|
assert.equal(clean.disposition, 'GRACEFUL'); assert.equal(clean.exitCode, 0); assert.equal(clean.terminal, true);
|
|
const hanging = await owned("process.on('SIGTERM',()=>{});process.stdin.resume();console.log('ready');setInterval(()=>{},1000);");
|
|
const forced = await cleanupOwnedFixture(hanging, { graceMs: 20, termMs: 20, killMs: 500 });
|
|
assert.equal(forced.disposition, 'SIGKILL'); assert.equal(forced.signal, 'SIGKILL'); assert.equal(forced.terminal, true); assert.equal(hanging.signalCode, 'SIGKILL');
|
|
});
|
|
test('owned child nonzero exit is recorded as a cleanup error', async () => {
|
|
const { spawn } = require('node:child_process');
|
|
const { cleanupOwnedFixture } = require('../e2e/live-evaluation-assertions.cjs');
|
|
const child = spawn(process.execPath, ['-e', "process.stdin.on('data',()=>process.exit(3));console.log('ready');setInterval(()=>{},1000);"], { stdio: ['pipe','pipe','pipe'] });
|
|
await new Promise(resolve => child.stdout.once('data', resolve));
|
|
const outcome = await cleanupOwnedFixture(child, { graceMs: 100, termMs: 100, killMs: 500 });
|
|
assert.equal(outcome.exitCode, 3); assert.equal(outcome.disposition, 'EXIT_ERROR'); assert.ok(outcome.errors.length > 0);
|
|
});
|
|
test('owned child with closed stdin records the pipe error and reaches terminal cleanup', async () => {
|
|
const { spawn } = require('node:child_process');
|
|
const { cleanupOwnedFixture } = require('../e2e/live-evaluation-assertions.cjs');
|
|
const child = spawn(process.execPath, ['-e', "require('node:fs').closeSync(0);console.log('ready');setTimeout(()=>process.exit(0),500);"], { stdio: ['pipe','pipe','pipe'] });
|
|
await new Promise(resolve => child.stdout.once('data', resolve));
|
|
const outcome = await cleanupOwnedFixture(child, { graceMs: 30, termMs: 200, killMs: 500 });
|
|
assert.equal(outcome.terminal, true);
|
|
assert.ok(outcome.errors.some(error => /EPIPE/.test(error)), 'Closed fixture input must fail the report, not escape as an uncaught stream error');
|
|
});
|