test(track-d 4): end-to-end remember-Brutus regression scenario with fakes

Adds `RememberBrutusEndToEndTests` proving the extraction -> store ->
search loop holds end-to-end. Without this test, Tracks D-1..D-3 each
move a single component but no test covers their composition; a future
regression that severs the loop at the extraction/search boundary
would slip past per-component tests.

Scenario:
  1. User: "My dog Brutus is a Bernese Mountain Dog."
  2. Stub LLM provider returns `apply_memory_ops(ADD subject=Brutus
     predicate=breed object="Bernese Mountain Dog")`.
  3. Real `MemoryExtractionOrchestrator` + `MemoryExtractor` drive the
     ApplyOp callback into a `FakeStore` actor.
  4. User: "What breed is Brutus?"
  5. Real `HybridSearch` actor + stub embedder + `FakeStore`'s
     `MemoryReadStore` conformance return one hit; assert
     `summary == "Brutus breed Bernese Mountain Dog"` and the trigger
     turnId propagated to retrieval recording.

Fakes used:
- `FakeStore` actor — captures applyOp via custom `applyAdd`, conforms
  to `MemoryReadStore` so `HybridSearch.searchFacts` runs unmodified.
  No real SQLite, no vec0.dylib needed.
- `StubEmbedder` — constant 768-dim vector (real path is
  `OllamaEmbeddingClient` -> `nomic-embed-text`).
- `BrutusProvider` — single-shot stub `LLMProvider` emitting one
  `toolUseRequested` then `messageStop`.

Boundary: stops at `HybridSearch.searchFacts`. The MCP-side dispatch
through `SearchMemoryTool` is covered by
`InProcessMemoryToolsTests.testSearchMemoryToolDispatch` in the MCP
package. Together those two tests cover the path the agent takes:
prompt -> extractor -> store -> search tool -> agent context.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
james.maesandClaude Opus 4.7 committed 2026-05-04 14:26:37 -05:00
1 parent 51750c161b
commit c3bd0a5728
1 file changed
+175
@@ -0,0 +1,175 @@
import XCTest
import Foundation
import AgentCore
@testable import Memory
/// Track-D D-4 — end-to-end "remember Brutus" regression scenario.
///
/// Drives the extraction -> store -> search loop with fakes for the
/// vec-dependent + Ollama-dependent surfaces:
///
/// - Real `MemoryExtractionOrchestrator` + `MemoryExtractor`.
/// - Stub `LLMProvider` that turns "My dog Brutus is a Bernese Mountain
/// Dog." into an `apply_memory_ops` ADD call.
/// - In-memory `FakeStore` actor that captures `applyOp` calls and serves
/// the `MemoryReadStore` surface for `HybridSearch` (no real SQLite, no
/// vec0.dylib needed).
/// - Stub `EmbeddingProviding` (returns a constant 768-dim vector).
/// - Real `HybridSearch` actor — what `SearchMemoryTool` calls through
/// `HybridSearchDispatching`. The MCP-side dispatch test lives in
/// `InProcessMemoryToolsTests`.
///
/// **What this protects against:** silent regressions where the
/// extraction pipeline writes facts but the search path can't see them
/// (or vice versa) — the loop break that produced this audit's bullet
/// "memory not actually queryable end-to-end".
final class RememberBrutusEndToEndTests: XCTestCase {
// MARK: - Fakes
/// In-memory store that:
/// 1. Captures `applyOp` calls and assigns auto-incrementing fact ids.
/// 2. Serves `MemoryReadStore` for `HybridSearch` — returns its
/// captured facts as `(Fact, Double)` rows in insertion order with
/// flat scores. No real RRF; this isn't a search-quality test.
actor FakeStore: MemoryReadStore {
private var facts: [Fact] = []
private var nextId: Int64 = 1
private(set) var retrievals: [(FactRef, Int64)] = []
func applyAdd(subject: String, predicate: String, object: String, sourceTurnId: Int64) -> Fact {
let now = Int64(Date().timeIntervalSince1970 * 1000)
let fact = Fact(
id: nextId, subject: subject, predicate: predicate, object: object,
sourceTurnId: sourceTurnId, validFrom: now, validTo: nil,
supersededBy: nil, forgottenAt: nil, createdAt: now
)
nextId += 1
facts.append(fact)
return fact
}
func snapshot() -> [Fact] { facts }
// MemoryReadStore conformance.
func runHybridSearchSQL(query: String, embedding: [Float], k: Int) async throws -> [(Fact, Double)] {
// Filter active rows; in real SQL this is `valid_to IS NULL
// AND forgotten_at IS NULL`.
let active = facts.filter { $0.validTo == nil && $0.forgottenAt == nil }
return Array(active.prefix(k)).enumerated().map { (i, f) in
(f, 1.0 / Double(i + 1)) // RRF-like flat scoring
}
}
func recordRetrieval(_ ref: FactRef, triggerTurnId: Int64) async {
retrievals.append((ref, triggerTurnId))
}
}
/// Stub embedder — constant 768-dim vector. The real
/// `nomic-embed-text` runs through `OllamaEmbeddingClient`; we don't
/// exercise that here.
actor StubEmbedder: EmbeddingProviding {
func embed(_ input: String) async throws -> [Float] {
Array(repeating: Float(0.1), count: MemoryConstants.embeddingDim)
}
}
/// Stub provider that emits exactly one ADD op for the Brutus turn,
/// then `messageStop`. Independent of the system prompt — the
/// extractor's prompt construction is exercised separately in
/// `MemoryExtractorTests`.
final class BrutusProvider: LLMProvider, @unchecked Sendable {
func stream(
messages: [LLMMessage],
tools: [ToolSchema],
toolChoice: ToolChoice,
model: ModelID,
maxOutputTokens: Int,
cacheHints: CacheHints?
) -> AsyncThrowingStream<LLMEvent, Error> {
AsyncThrowingStream { cont in
Task {
let payload: [String: Any] = ["ops": [
[
"op": "ADD",
"subject": "Brutus",
"predicate": "breed",
"object": "Bernese Mountain Dog",
]
]]
let json = try! JSONSerialization.data(withJSONObject: payload)
cont.yield(.toolUseRequested(
ToolUseRequest(id: "t-brutus", name: "apply_memory_ops", argsJSON: json)
))
cont.yield(.messageStop)
cont.finish()
}
}
}
}
// MARK: - Test
func testExtractionWriteThenSearchFindsBrutus() async throws {
let store = FakeStore()
// 1. Extraction wiring.
let extractor = MemoryExtractor(provider: BrutusProvider())
let orchestrator = MemoryExtractionOrchestrator(
extractor: extractor,
applyOp: { op, turnId in
guard case let .add(subject, predicate, object, _) = op else {
XCTFail("D-4: stub provider should produce only ADD; got \(op)")
return
}
_ = await store.applyAdd(
subject: subject, predicate: predicate, object: object,
sourceTurnId: turnId
)
}
)
await orchestrator.start()
// 2. User turn 1 — "remember Brutus".
await orchestrator.enqueue(ExtractionJob(
turnId: TurnID(rawValue: "turn-brutus-1"),
userText: "My dog Brutus is a Bernese Mountain Dog.",
assistantText: "Got it — I'll remember Brutus."
))
// 3. Wait for extraction to drain.
let deadline = Date().addingTimeInterval(2.0)
while await store.snapshot().isEmpty && Date() < deadline {
try await Task.sleep(nanoseconds: 10_000_000)
}
let written = await store.snapshot()
XCTAssertEqual(written.count, 1, "extraction should have written exactly one fact")
XCTAssertEqual(written.first?.subject, "Brutus")
XCTAssertEqual(written.first?.predicate, "breed")
XCTAssertEqual(written.first?.object, "Bernese Mountain Dog")
// 4. Search wiring — same store, stub embedder.
let search = HybridSearch(store: store, embedder: StubEmbedder())
// 5. User turn 2 — "What breed is Brutus?". Search must surface
// the fact written in turn 1.
let hits = try await search.searchFacts(
query: "What breed is Brutus?",
k: 5,
triggerTurnId: 2
)
XCTAssertEqual(hits.count, 1, "D-4: search must find the fact written by extraction")
XCTAssertEqual(hits.first?.summary, "Brutus breed Bernese Mountain Dog",
"FactRef.summary is `subject predicate object` per HybridSearch")
// 6. Retrieval was recorded for DevOverlay correlation (D-05 single
// emission site).
let retrievals = await store.retrievals
XCTAssertEqual(retrievals.count, 1)
XCTAssertEqual(retrievals.first?.1, 2)
await orchestrator.shutdown()
}
}