test(10-02b): add AvailableToolsWiringTests regression for B-01

The 2026-05-07 live verification of Plan 10-02 surfaced that
App/AppDelegate.swift:1274 constructed AgentOrchestrator with a
hardcoded availableTools: [] empty array. The model saw the system
prompt naming tools but had no schemas in the API tools[] field, so
it hallucinated JSON tool definitions into chat instead of dispatching
real tool_use blocks.

These tests assert at the orchestrator → provider boundary: schemas
passed to AgentOrchestrator(availableTools:) MUST reach the
LLMProvider.stream(... tools:) call on the first turn. The previous
in-process registry test asserted at the wrong layer (registry was
correctly populated, but its catalog never reached the orchestrator).

Both tests pass at HEAD because the orchestrator's wiring of
availableToolsList → toolsForCall → provider.stream is sound. They
exist to catch any future regression at this layer.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
james.maesandClaude Opus 4.7 committed 2026-05-07 07:18:45 -05:00
1 parent a75ae0a08b
commit 260350b3dc
1 file changed
+133
@@ -0,0 +1,133 @@
import XCTest
import AgentCore
import Config
import Replay
@testable import AgentOrchestrator
/// Plan 10-02b / B-01 regression. The 2026-05-07 live-launch verification
/// surfaced that `App/AppDelegate.swift:1274` constructed
/// `AgentOrchestrator(... availableTools: [], ...)` with a hardcoded empty
/// array, silencing every tool call across the agent (including the four
/// Phase 10-01 self-knowledge tools, the Phase 7 memory tools, and the
/// Phase 5 stdio helper tools). The model saw the system-prompt preamble
/// naming tools but had no schemas in the API `tools[]` array, so it
/// hallucinated JSON tool definitions into chat instead of dispatching
/// real `tool_use` blocks.
///
/// This regression test asserts at the orchestrator → provider boundary:
/// when `AgentOrchestrator` is constructed with a non-empty
/// `availableTools:` list, the `LLMProvider.stream(...)` call MUST receive
/// that same list of `ToolSchema`s in its `tools:` argument. The previous
/// in-process registry test (`registersFourSelfKnowledgeTools`) asserted
/// at the wrong layer — the registry was correctly populated, but its
/// catalog never reached the orchestrator constructor. This test catches
/// the substrate gap.
@MainActor
final class AvailableToolsWiringTests: XCTestCase {
var tempHome: TempReplayHome!
override func setUp() async throws {
try await super.setUp()
tempHome = TempReplayHome.make()
}
override func tearDown() async throws {
tempHome?.cleanup()
try await super.tearDown()
}
/// B-01-REGRESSION: passing N schemas to `availableTools:` MUST result
/// in the same N schemas reaching the provider's `stream(... tools:)`
/// call on the first turn.
func test_availableToolsReachProviderFirstCall() async throws {
// Arrange: build an orchestrator with three tool schemas mirroring
// the production catalog shape — name, description, and a JSON
// Schema bytes blob.
let schemas: [ToolSchema] = [
ToolSchema(
name: "get_active_audio_route",
description: "Returns the active mic + speaker route.",
inputSchema: Data(#"{"type":"object","properties":{}}"#.utf8)
),
ToolSchema(
name: "list_audio_devices",
description: "Lists audio I/O devices on the host.",
inputSchema: Data(#"{"type":"object","properties":{}}"#.utf8)
),
ToolSchema(
name: "get_self_state",
description: "Returns the runtime self-state snapshot.",
inputSchema: Data(#"{"type":"object","properties":{}}"#.utf8)
),
]
let mock = MockLLMProvider(script: .init(events: [
.messageStart(LLMMessageStart(messageId: "m", model: "claude-opus-4-7", usagePrefix: nil)),
.stopReason(.endTurn),
.messageStop,
]))
let replay = try ReplayLog(databaseURL: tempHome.dbURL)
let session = try await replay.beginSession(appVersion: "test", buildSHA: "deadbeef")
let orch = AgentOrchestrator(
configStore: makeConfigStore(),
providerFactory: { _ in mock },
toolDispatcher: StubToolDispatcher(),
replayLog: replay,
sessionId: session,
systemPrompt: "you are jarvis",
availableTools: schemas
)
// Act: submit a turn so the orchestrator opens a stream.
let outcome = await orch.submit(.text("hello"))
guard case .ran = outcome else {
return XCTFail("expected .ran, got \(outcome)")
}
// Drain to .idle so the provider's recorded calls are stable.
for await event in orch.events {
if case .stateChange(.idle) = event { break }
}
// Assert: the provider received exactly the schemas we passed.
let recorded = await mock.getRecordedCalls()
XCTAssertGreaterThanOrEqual(recorded.count, 1, "provider was never called")
let firstTools = recorded[0].tools
XCTAssertEqual(firstTools.count, schemas.count,
"B-01: orchestrator's first stream() call carried \(firstTools.count) tools, expected \(schemas.count)")
XCTAssertEqual(Set(firstTools.map(\.name)), Set(schemas.map(\.name)),
"B-01: tool names reaching the provider differ from the input catalog")
}
/// B-01-NEGATIVE: explicitly verifies that a default-init (omitting
/// `availableTools:`) yields an empty `tools:` array at the provider.
/// This pins down the precondition that the BUG was about hardcoding
/// `[]` at a CONSTRUCTION site, not about a default-arg behavior
/// change.
func test_defaultAvailableToolsYieldsEmptyToolsAtProvider() async throws {
let mock = MockLLMProvider(script: .init(events: [
.messageStart(LLMMessageStart(messageId: "m", model: "claude-opus-4-7", usagePrefix: nil)),
.stopReason(.endTurn),
.messageStop,
]))
let replay = try ReplayLog(databaseURL: tempHome.dbURL)
let session = try await replay.beginSession(appVersion: "test", buildSHA: "deadbeef")
let orch = AgentOrchestrator(
configStore: makeConfigStore(),
providerFactory: { _ in mock },
toolDispatcher: StubToolDispatcher(),
replayLog: replay,
sessionId: session,
systemPrompt: "you are jarvis"
// availableTools omitted → defaults to [].
)
_ = await orch.submit(.text("hello"))
for await event in orch.events {
if case .stateChange(.idle) = event { break }
}
let recorded = await mock.getRecordedCalls()
XCTAssertGreaterThanOrEqual(recorded.count, 1)
XCTAssertTrue(recorded[0].tools.isEmpty,
"default-init must yield empty tools (the bug is in the construction site, not the default)")
}
}