Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  james.maesandClaude Opus 4.7
|
dced4cb9a3 | ||
|
james.maes
|
00f1075ea5 | ||
|
james.maes
|
774e60a020 | ||
|
james.maes
|
a26f49fdbd | ||
|
james.maes
|
4cdb3560e9 | ||
|
james.maes
|
81ce64c3f7 | ||
|
james.maes
|
5fc2d3fabf |
No files matched your search
@@ -154,6 +154,34 @@ jobs:
|
||||
- Vision
|
||||
- Voice
|
||||
steps:
|
||||
# Grogu is a shared workstation, not an ephemeral VM. Interactive
|
||||
# Xcode test sessions, debugging, or aborted `swift test` runs can
|
||||
# leave orphan xctest/swift-test/swift-driver processes around for
|
||||
# hours, holding Mach services + sockets + ports that a new
|
||||
# WKWebView mount or test-target launch then blocks on (observed:
|
||||
# AgentCore hung 34min on develop CI 25824607025 with 6 zombie
|
||||
# xctests aged 22-24h from prior interactive use).
|
||||
#
|
||||
# Age filter (>= 1 hour): Grogu now has SIBLING runners
|
||||
# (Grogu-jarvis, -2, -3) so concurrent CI jobs can be in flight
|
||||
# simultaneously. CI jobs complete in <10 min, so a 1-hour cutoff
|
||||
# safely reaps interactive-use orphans without killing concurrent
|
||||
# CI work on sibling runners. `ps -o etime` formats: `MM:SS`
|
||||
# (<1h, skip), `HH:MM:SS` (>=1h, kill), `DD-HH:MM:SS` (days, kill).
|
||||
# github-hosted VMs start clean, so this step is gated to self-hosted.
|
||||
- name: Reap orphan xctest/swift-test processes older than 1h (self-hosted only)
|
||||
if: runner.environment == 'self-hosted'
|
||||
run: |
|
||||
age_filter='/(xctest|swift-test|swift-driver)$/ && !/awk/ && (match($2, /-/) || match($2, /[0-9]+:[0-9]+:[0-9]+/)) {print $1}'
|
||||
/bin/ps -A -o pid,etime,comm \
|
||||
| awk "$age_filter" \
|
||||
| xargs -r -I{} kill -TERM {} 2>/dev/null || true
|
||||
sleep 2
|
||||
/bin/ps -A -o pid,etime,comm \
|
||||
| awk "$age_filter" \
|
||||
| xargs -r -I{} kill -9 {} 2>/dev/null || true
|
||||
echo "Surviving xctest/swift-test processes (any age — includes concurrent CI work on sibling runners):"
|
||||
/bin/ps -A -o pid,etime,comm | grep -E '(xctest|swift-test|swift-driver)$' || echo " (none)"
|
||||
- uses: actions/checkout@v4
|
||||
# Project targets macOS 26 Tahoe + Swift 6.2/6.3. Xcode 16 (Swift 6.0)
|
||||
# cannot parse `mlx-audio-swift` v0.1.2's `swift-tools-version:6.2`,
|
||||
@@ -403,3 +431,51 @@ jobs:
|
||||
fi
|
||||
git tag -a "$TAG" -m "Automated tag from CI for $GH_REF"
|
||||
git push origin "$TAG"
|
||||
# GitHub Actions GITHUB_TOKEN cannot trigger downstream workflows. The
|
||||
# separate `release.yml` workflow listens for `push: tags:` events but
|
||||
# never sees a CI-pushed tag — observed on v0.1.0-dev.50 (tag created,
|
||||
# Release object missing, had to be manually created). Create the
|
||||
# Release object directly here in the same job so the Releases page
|
||||
# reflects CI-pushed tags without a manual step. `release.yml` stays
|
||||
# as a fallback for manually-pushed tags (which CAN trigger workflows).
|
||||
- name: Create GitHub Release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ steps.version.outputs.tag }}
|
||||
PRERELEASE: ${{ steps.version.outputs.prerelease }}
|
||||
RUN_URL: ${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! git rev-parse "refs/tags/$TAG" >/dev/null 2>&1; then
|
||||
echo "Tag $TAG not present locally; nothing to release." >&2
|
||||
exit 0
|
||||
fi
|
||||
if gh release view "$TAG" >/dev/null 2>&1; then
|
||||
echo "Release $TAG already exists — skipping create (idempotent)."
|
||||
exit 0
|
||||
fi
|
||||
prev=$(git tag --list 'v*' --sort=-v:refname | grep -v "^${TAG}$" | head -n 1 || true)
|
||||
notes_file=$(mktemp)
|
||||
{
|
||||
echo "## Changes"
|
||||
echo
|
||||
if [ -n "$prev" ]; then
|
||||
git log --pretty='format:- %s (%h)' "${prev}..${TAG}"
|
||||
else
|
||||
git log --pretty='format:- %s (%h)' -20
|
||||
fi
|
||||
echo
|
||||
echo
|
||||
echo "## Verification"
|
||||
echo
|
||||
echo "- CI build: $RUN_URL"
|
||||
echo "- Boundary gates, webview, app-build, 13/13 SPM packages: all green on this commit"
|
||||
} > "$notes_file"
|
||||
flag=()
|
||||
if [ "$PRERELEASE" = "true" ]; then
|
||||
flag+=("--prerelease")
|
||||
fi
|
||||
gh release create "$TAG" \
|
||||
--title "$TAG" \
|
||||
--notes-file "$notes_file" \
|
||||
"${flag[@]}"
|
||||
@@ -64,7 +64,7 @@ The top-level shape is fixed and agreed. Push back with specific reasoning only
|
||||
- New tokenizer produces ~35% more tokens than Opus 3.x for the same text. Instrument token budgets accordingly; cap `tool_result` content at 8 KB.
|
||||
- Cache TTL default is 5 minutes ephemeral. For 1-hour TTL, pass `ttl: "1h"` explicitly on `cache_control` blocks **AND** include the request header `anthropic-beta: extended-cache-ttl-2025-04-11`. Without the beta header, 1h is silently ignored. Verify via DevOverlay watching `cache_creation_input_tokens` vs `cache_read_input_tokens` per turn.
|
||||
- Tool-use and extended thinking both stream; the SSE parser must handle `content_block_start` with `input_json_delta` for tool args. Close on `message_stop`, not `message_delta`; swallow `ping`; route `thinking_delta` to its own case; treat `stop_reason: "refusal"` as first-class; emit `partial_tool_use_at_disconnect` on mid-delta termination.
|
||||
- **Ollama caveat:** as of April 2026, Qwen 3 / 3.5 / Gemma 4 tool-calling is broken in Ollama — confirmed via live issue tracker (ollama/ollama#14493 Qwen 3.5 27B non-functional; #14601 Qwen3 malformed tool defs via `/api/chat`; #14745 qwen3.5:9b print-not-execute; #15315 Gemma 4 tool parser). Root cause: Ollama's renderer/parser maps Qwen 3.5 through a Hermes-style JSON pipeline, but the model was trained on Qwen3-Coder XML format; unclosed `<think>` tags corrupt multi-turn. **Known-good local tool-calling baseline: `qwen2.5-coder:32b`.** Qwen3 stays disabled as opt-in until upstream closes these issues. Llama 4 with the `llama4_pythonic` parser is worth testing but not yet trusted. Document the local model in use alongside every eval run.
|
||||
- **Ollama caveat:** as of April 2026, Qwen 3 / 3.5 / Gemma 4 tool-calling is broken in Ollama — confirmed via live issue tracker (ollama/ollama#14493 Qwen 3.5 27B non-functional; #14601 Qwen3 malformed tool defs via `/api/chat`; #14745 qwen3.5:9b print-not-execute; #15315 Gemma 4 tool parser). Root cause: Ollama's renderer/parser maps Qwen 3.5 through a Hermes-style JSON pipeline, but the model was trained on Qwen3-Coder XML format; unclosed `<think>` tags corrupt multi-turn. **Known-good local tool-calling baseline: `qwen2.5-coder:32b-instruct-q8_0`** (the variant pinned by `ModelID.qwen25coder32b`; vanilla `qwen2.5-coder:32b` is not used). Qwen3 stays disabled as opt-in until upstream closes these issues. Llama 4 with the `llama4_pythonic` parser is worth testing but not yet trusted. Document the local model in use alongside every eval run.
|
||||
- **Ollama transport gotcha:** `/api/chat` (native NDJSON) emits `tool_calls` on the chunk **preceding** the `done: true` terminator, not with it. Decoder must read `tool_calls` whenever seen and never gate on `done`. Separate decoders for `/api/chat` vs `/v1/chat/completions` (OpenAI-compat SSE with atomic `tool_calls`).
|
||||
- **Tool-choice discipline:** `LLMProvider.stream(..., toolChoice:)` is mandatory. Cap-recovery "one more call" sets `.none` (Anthropic: `{type: "none"}`; Ollama: drop `tools` array entirely). Eval asserts zero `.toolUseRequested` on recovery call.
|
||||
|
||||
@@ -87,7 +87,7 @@ The top-level shape is fixed and agreed. Push back with specific reasoning only
|
||||
**Memory stack:**
|
||||
- **Store:** SQLite with **FTS5** (keyword search) + **sqlite-vec** extension (vector search), single file under `~/Library/Application Support/Jarvis/`.
|
||||
- **Embeddings:** `nomic-embed-text` via Ollama (local, free, 768-dim).
|
||||
- **Extraction:** mem0's `ADD / UPDATE / NOOP` pattern — after each turn, a local extractor (Qwen 2.5-Coder 32B via Ollama) decides what facts to commit/update/ignore. **Fully local; no user data ever leaves the machine for memory.**
|
||||
- **Extraction:** mem0's `ADD / UPDATE / NOOP` pattern — after each turn, a local extractor (`qwen3.6:latest` via Ollama, per commit `8d0a959`) decides what facts to commit/update/ignore. **Fully local; no user data ever leaves the machine for memory.**
|
||||
- **Temporal validity:** record `valid_from` / `valid_to` on facts (Zep/Graphiti style) so "Sarah works at Acme" can be superseded without deleting history.
|
||||
- **Scope tiers:** `UserDefaults` for trivial prefs, JSON for tool configs/feature flags, SQLite for everything conversational.
|
||||
- Secrets (API keys) → macOS **Keychain**, never plaintext, never checked in.
|
||||
@@ -227,7 +227,7 @@ Codesign + entitlement verification: `bash scripts/verify-entitlements.sh --pre-
|
||||
|
||||
```bash
|
||||
JARVIS_REAL_MODELS=1 swift test --package-path packages/Memory --filter MemoryRegressionCorpusTests
|
||||
# requires: local Ollama with nomic-embed-text + qwen2.5-coder:32b + loadable vec0.dylib
|
||||
# requires: local Ollama with nomic-embed-text + qwen3.6:latest (extractor) + qwen2.5-coder:32b-instruct-q8_0 (agent fallback) + loadable vec0.dylib
|
||||
|
||||
JARVIS_REAL_CAMERA=1 swift test --package-path packages/Vision --filter CameraCaptureRealHardwareTests
|
||||
# requires: real AVCaptureDevice + .authorized TCC
|
||||
@@ -236,7 +236,7 @@ JARVIS_REAL_MODELS=1 swift test --package-path packages/Voice --filter OrpheusTT
|
||||
# requires: ~6 GB Orpheus weights downloaded; produces empirical TTFA measurement
|
||||
```
|
||||
|
||||
Eval harness (Anthropic + local-model eval matrix; corpus in `.planning/evals/`) — pinned to `qwen2.5-coder:32b` for the local-model lane.
|
||||
Eval harness (Anthropic + local-model eval matrix; corpus in `.planning/evals/`) — pinned to `qwen2.5-coder:32b-instruct-q8_0` for the local-model lane (matches `ModelID.qwen25coder32b`).
|
||||
|
||||
### Self-hosted CI runner (Grogu)
|
||||
|
||||
@@ -250,6 +250,7 @@ Eval harness (Anthropic + local-model eval matrix; corpus in `.planning/evals/`)
|
||||
- **Cache:** the `actions/cache@v4` step is gated to `runner.environment == 'github-hosted'`. Self-hosted relies on the persistent `_work/` dir on HD-2 (and SwiftPM's natural `.build/` reuse) instead. Github-hosted caches bake absolute `/Users/runner/...` paths into `.build/artifacts/onnxruntime.xcframework` and other XCFrameworks, which Grogu cannot resolve — restricting the cache action keeps these caches strictly per-environment.
|
||||
- **Runner keychain** (`~/Library/Keychains/jarvis-runner.keychain-db`): created once on Grogu with password `grogu-jarvis-ci` (also stored as repo secret `JARVIS_RUNNER_KEYCHAIN_PASSWORD`). The LaunchAgent has no UI session, so `SecItemAdd` against a locked login keychain returns `errSecInteractionNotAllowed (-25308)`. CI unlocks `jarvis-runner.keychain-db`, prepends it to the user search list, AND sets it as the user's default keychain before `swift test`. The default-keychain step is necessary because on the legacy file-based macOS keychain, `SecItemAdd` writes to the user's *default* keychain (not the first writable in the search list). **Side effects on interactive use:** (1) `jarvis-runner.keychain-db` sits first in your user search list, and (2) it is also your default keychain — so `SecItemAdd` calls *without* an explicit keychain target write there instead of `login.keychain-db`. Apps that target login keychain explicitly (most do) are unaffected. To undo: `security default-keychain -d user -s ~/Library/Keychains/login.keychain-db && security list-keychains -d user -s ~/Library/Keychains/login.keychain-db`.
|
||||
- **When Grogu reboots or sleeps**, the LaunchAgent restarts on login but jobs queued during downtime will sit waiting. For long offline windows, push to a branch other than develop or temporarily revert `runs-on:` to `macos-15`.
|
||||
- **Process hygiene:** the workflow's first step on self-hosted is a `Reap orphan xctest/swift-test processes older than 1h` cleanup. Grogu is a shared workstation — interactive Xcode test runs, aborted `swift test`, or debugger sessions can leak `xctest` / `swift-test` / `swift-driver` processes that hold Mach services + sockets and cause subsequent CI jobs to deadlock (observed: AgentCore hung 34 min on a develop run when 6 such zombies aged 22–24h were present). The age filter (≥1h) keeps concurrent CI work on sibling runners safe (jobs complete in <10 min).
|
||||
|
||||
# CLAUDE.md
|
||||
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
# Handoff — 2026-05-14: multi-runner shipped, v0.2 ready
|
||||
|
||||
**Session window:** 2026-05-13 afternoon (Grogu single runner setup) → 2026-05-14 (multi-runner expansion + first auto-Release)
|
||||
**Branch:** `develop`, clean
|
||||
**HEAD at write time:** `774e60a0` — `ci(actions): create GitHub Release directly in publish-tag job (#187)`
|
||||
**Open PR count:** 2 (#185, #191 — both with shepherds running)
|
||||
**Repo visibility:** PUBLIC
|
||||
|
||||
---
|
||||
|
||||
## TL;DR
|
||||
|
||||
- **3-runner self-hosted CI on Grogu is live** — `Grogu-jarvis`, `-2`, `-3`. Matrix parallelism observed (≥3 SPM jobs concurrent). Develop CI green in ~5 min vs. ~13 min single-runner.
|
||||
- **Auto-Release flow proven** — `v0.1.0-dev.64` has a real GitHub Release object, created by the new `gh release create` step in `publish-tag`. No more manual `gh release create` needed.
|
||||
- **Brainstorm → spec → plan → execute discipline established** via the Superpowers skills. Multi-runner work has a committed spec + plan under `docs/superpowers/{specs,plans}/`. Pattern to follow for v0.2 onward.
|
||||
- **Memory test flake** surfaced: `MemoryWiringEndToEndTests.testMemoryEnqueueReceivesNonNilUserAndAssistantText` fails intermittently on PR branches but passes on develop. Hypothesis: race in `TurnTranscriptStore` actor under concurrent-runner load. Pre-existing; not from any 2026-05-13/14 code change.
|
||||
|
||||
---
|
||||
|
||||
## What shipped today
|
||||
|
||||
### v0.1 mop-up + first dev tag
|
||||
- **PR #184** (squash `5fc2d3fa`) — Ollama model baseline reconciled to `qwen3.6:latest` (extractor) + `qwen2.5-coder:32b-instruct-q8_0` (agent fallback). Closes #58 (last v0.1 issue).
|
||||
- **v0.1.0-dev.50** — first release tag of the project. Release object created manually (workflow couldn't trigger downstream).
|
||||
- **v0.1.0-dev.64** — second release tag. Release object created **automatically** via the new inline `gh release create` step in `publish-tag`.
|
||||
|
||||
### Self-hosted runner saga
|
||||
- **PR #181** (`eef17252`) — Moved `swift-packages` matrix to `Grogu-jarvis`. DEVELOPER_DIR Xcode-select. Fork-PR gate.
|
||||
- **PR #183** (`81ce64c3`) — Orphan-process reaper at job start (closes #182).
|
||||
- **PR #189** (`4cdb3560`) — Reaper gets a `>=1h` age filter so sibling runners on Grogu don't kill each other.
|
||||
- **PR #187** (`774e60a0`) — `publish-tag` step creates the Release object directly instead of relying on `release.yml` triggering on tag push (GITHUB_TOKEN can't trigger downstream workflows).
|
||||
- **PR #190** (`a26f49fd`) — Multi-runner spec + plan committed to `docs/superpowers/`.
|
||||
- **PR #185** — open, AgentCore #23+#24 fix. Currently has a Memory-flake CI failure under shepherd watch.
|
||||
- **PR #191** — open, CLAUDE.md inventory update. Same Memory flake situation.
|
||||
|
||||
### Grogu state
|
||||
- 3 runners installed at `~/actions-runner-jarvis{,-2,-3}/`, each with `_work` symlinked to `/Volumes/HD-2/jarvis-runner-work{,-2,-3}/`.
|
||||
- LaunchAgents managed by `~/Library/LaunchAgents/actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis{,-2,-3}.plist`.
|
||||
- Keychain workflow: `~/Library/Keychains/jarvis-runner.keychain-db`, password in repo secret `JARVIS_RUNNER_KEYCHAIN_PASSWORD`. CI unlocks + makes default + prepends to search list.
|
||||
- Side effect: `jarvis-runner.keychain-db` is your user's default keychain after CI runs. Undo command in CLAUDE.md.
|
||||
|
||||
---
|
||||
|
||||
## Active background processes at handoff
|
||||
|
||||
| Task ID | What | Expected exit |
|
||||
|---|---|---|
|
||||
| `bny76q2ad` | Final shepherd watching PR #185 + #191 | When both PRs are handled (merged or marked `ci-failed`), or 60-min timeout |
|
||||
|
||||
**Log location:** `/tmp/final-shepherd.log`
|
||||
|
||||
Reading the log will tell you what the shepherd did. If it marked both PRs `ci-failed` because of the Memory flake, see "Memory flake disposition" below.
|
||||
|
||||
---
|
||||
|
||||
## Memory flake disposition (what to do when you pick up)
|
||||
|
||||
`MemoryWiringEndToEndTests.testMemoryEnqueueReceivesNonNilUserAndAssistantText` fails with:
|
||||
```
|
||||
XCTAssertEqual failed: ("0") is not equal to ("1") - enqueue was not called — BLOCKER-1 regression
|
||||
XCTAssertEqual failed: ("nil") is not equal to ("Optional("hi")") - userText empty/nil — Plan 4's submit-time append did not propagate via TurnTranscriptStore
|
||||
XCTAssertEqual failed: ("nil") is not equal to ("Optional("hello there")") - assistantText empty/nil — Plan 1's transcript subscriber did not propagate via TurnTranscriptStore
|
||||
```
|
||||
|
||||
**Evidence it's a flake, not a regression:**
|
||||
- PR #191 doesn't touch AgentCore at all (only `CLAUDE.md`) yet hits the same exact failure.
|
||||
- Develop's latest CI (`25832341662`, on `774e60a0`) passed Memory in 45s.
|
||||
- Both #185 and #191 are branched off slightly different develop SHAs — the test passes on the latest develop but fails on these branch HEADs after forward-merge.
|
||||
|
||||
**Recommended next steps:**
|
||||
1. Open a new issue: `bug,severity:medium,area:memory` — "MemoryWiringEndToEndTests intermittent failure under concurrent runner load". Describe the `TurnTranscriptStore` race hypothesis.
|
||||
2. Either rerun the failed jobs (CI may pass on next attempt) OR merge both PRs as-is since branch protection has 0 required checks and the test passes reliably on develop.
|
||||
3. Long-term: file the test as known-flaky in v0.2 milestone.
|
||||
|
||||
---
|
||||
|
||||
## v0.2 Wave 1 (next session start)
|
||||
|
||||
Tiny / doc-only / deterministic fixes — safe to land in a single session each:
|
||||
|
||||
| Issue | What | Effort | Why first |
|
||||
|---|---|---|---|
|
||||
| **#19** | MemoryStore.swift stale Phase 7 hedge doc-comments | ~5 min | Pure docs, zero risk |
|
||||
| **#45** | Bus 'JarvisBusWorld' docstring drift (20+ docstrings) | ~15 min | Pure docs, mechanical |
|
||||
| **#46** | Stale webview bundle artifacts in `App/Resources/webview/assets/` (~50MB) | ~10 min | File deletes |
|
||||
| **#37** | STTBackendSelector snake_case vs camelCase mismatch | ~15 min | Single rename |
|
||||
| **#178** | Add OpenSSF Scorecard workflow | ~15 min | One new `.github/workflows/` file |
|
||||
| **#47** | WebviewBridge.sendRaw NaN→null JSON guard | ~20 min | One-line bug fix |
|
||||
|
||||
Each gets brainstorm → spec → plan via the Superpowers skills (specs/plans templates established in `docs/superpowers/`). Brief specs (a few paragraphs) — these are "issue body IS the design" cases.
|
||||
|
||||
## v0.2 Wave 2 / Wave 3 — wait
|
||||
|
||||
Larger items that need real diagnostic work or design decisions:
|
||||
- #18 MemoryReplaySink error propagation — needs care
|
||||
- #23 + #24 already in PR #185 (held)
|
||||
- #44 HUD audioLevel consumer rewire — MED, design choice
|
||||
- #43 sessionHistory dead-letter — overlaps M-7, defer
|
||||
- #176 vitest 4.x major bump — needs eval
|
||||
- #26 confirmation panel multi-monitor — needs real hardware
|
||||
- Any M-0..M-7 issue — strategic, gets its own spec → plan
|
||||
|
||||
---
|
||||
|
||||
## Key references
|
||||
|
||||
- **Spec template:** `docs/superpowers/specs/2026-05-13-multi-runner-grogu-design.md`
|
||||
- **Plan template:** `docs/superpowers/plans/2026-05-13-multi-runner-grogu.md`
|
||||
- **Previous handoff:** `docs/HANDOFF-2026-05-13-v0.1-shipped.md` (now superseded for the multi-runner story but still describes v0.1 mop-up)
|
||||
- **Roadmap:** `.planning/ROADMAP-2026-05-12.md`
|
||||
- **Live state:** GitHub Issues + Milestones at https://github.com/KofTwentyTwo/Jarvis
|
||||
- **CI workflow:** `.github/workflows/ci.yml` — has the age-filtered cleanup + inline release create
|
||||
- **CLAUDE.md `Self-hosted CI runner (Grogu)`** section — operational reference for the 3-runner setup
|
||||
|
||||
## What this handoff deliberately doesn't relitigate
|
||||
|
||||
- The Memory flake's root cause — open a v0.2 bug and move on; develop's CI is reliably green so it's not blocking.
|
||||
- Whether to merge PR #185 / PR #191 with a known-flaky failure — branch protection allows it; user judgment.
|
||||
- Wave 2/3 v0.2 issues — they need design conversations; pick them up after Wave 1.
|
||||
- M-0..M-7 — strategic, gets its own session.
|
||||
|
||||
---
|
||||
|
||||
*Written 2026-05-14 at the natural session-end after multi-runner expansion shipped. Next session should start with `/clear`, read this doc, then read the latest `/tmp/final-shepherd.log` to see what the background shepherd did, then pick up Wave 1.*
|
||||
@@ -0,0 +1,374 @@
|
||||
# Multi-Runner Expansion on Grogu — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Scale Grogu from 1 to 3 self-hosted GitHub Actions runners so the 13-package SwiftPM matrix parallelizes, cutting develop CI wall-clock from ~13 min serial to ~5 min.
|
||||
|
||||
**Architecture:** Two new LaunchAgent runners (`Grogu-jarvis-2`, `Grogu-jarvis-3`), each in its own `~/actions-runner-jarvis-{2,3}/` directory with `_work/` symlinked to `/Volumes/HD-2/jarvis-runner-work-{2,3}/`. All three runners share labels `[self-hosted, macOS, ARM64, jarvis]` so matrix jobs dispatch to whichever is free.
|
||||
|
||||
**Tech Stack:** GitHub Actions self-hosted runner v2.334.0 (osx-arm64), `gh` CLI for token + verification, macOS LaunchAgents for per-user service management.
|
||||
|
||||
**Spec:** `docs/superpowers/specs/2026-05-13-multi-runner-grogu-design.md`
|
||||
|
||||
**Related issues:** #180 (single-runner setup), #182 (zombie cleanup), #186 (auto-release), #188 (age-filter — precondition).
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
This plan is primarily operational; the only repo file modified is `CLAUDE.md` (runner inventory documentation).
|
||||
|
||||
| Change | Path | Responsibility |
|
||||
|---|---|---|
|
||||
| **System (new)** | `~/actions-runner-jarvis-2/` | Runner-2 agent binary tree (already pre-staged) |
|
||||
| **System (new)** | `~/actions-runner-jarvis-2/_work` → `/Volumes/HD-2/jarvis-runner-work-2/` | Symlink for build artifacts (already pre-staged) |
|
||||
| **System (new)** | `~/Library/LaunchAgents/actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis-2.plist` | LaunchAgent service plist (created by `svc.sh install`) |
|
||||
| **System (new)** | `~/actions-runner-jarvis-3/` | Runner-3 agent binary tree (already pre-staged) |
|
||||
| **System (new)** | `~/actions-runner-jarvis-3/_work` → `/Volumes/HD-2/jarvis-runner-work-3/` | Symlink (already pre-staged) |
|
||||
| **System (new)** | `~/Library/LaunchAgents/actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis-3.plist` | LaunchAgent plist |
|
||||
| **Repo (modify)** | `CLAUDE.md` | Runner inventory section updated to list all 3 runners |
|
||||
|
||||
The pre-staging step (download + extract agent + create `_work` symlinks for runner 2 and 3) was performed earlier in the session. Task 2 step 1 verifies that pre-stage rather than redoing it.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Verify precondition — PR #189 merged to develop
|
||||
|
||||
**Files:**
|
||||
- Read-only check: `origin/develop` git log
|
||||
|
||||
PR #189 (age-filter zombie reaper) MUST be on `develop` before either new runner starts accepting jobs. The current cleanup step kills any `xctest`/`swift-test`/`swift-driver` regardless of age; concurrent runners' cleanup steps would mutually kill each other's in-flight tests.
|
||||
|
||||
- [ ] **Step 1: Fetch latest develop**
|
||||
|
||||
```bash
|
||||
git fetch origin develop
|
||||
```
|
||||
|
||||
Expected: no output, or "Updating <hash>..<hash>".
|
||||
|
||||
- [ ] **Step 2: Verify the age-filter commit is on develop**
|
||||
|
||||
```bash
|
||||
git log origin/develop --grep="age-filter zombie reaper" --oneline | head -3
|
||||
```
|
||||
|
||||
Expected: at least one line containing `676c54d ci(actions): age-filter zombie reaper (>= 1h) for multi-runner safety` (the exact SHA prefix may differ post-squash-merge).
|
||||
|
||||
If empty: PR #189 is not yet merged. Stop the plan here and re-run from Task 1 once it is. Do NOT proceed to Task 2 — starting concurrent runners against an un-filtered cleanup step will trigger mutual xctest kill.
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Register Grogu-jarvis-2
|
||||
|
||||
**Files:**
|
||||
- System: `~/actions-runner-jarvis-2/` (pre-staged binaries — verify only)
|
||||
- System: `~/actions-runner-jarvis-2/_work` symlink (pre-staged — verify only)
|
||||
- System (new): `~/actions-runner-jarvis-2/.runner` (created by `config.sh`)
|
||||
- System (new): `~/Library/LaunchAgents/actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis-2.plist` (created by `svc.sh install`)
|
||||
|
||||
- [ ] **Step 1: Verify pre-stage**
|
||||
|
||||
```bash
|
||||
test -x ~/actions-runner-jarvis-2/config.sh && \
|
||||
readlink ~/actions-runner-jarvis-2/_work | grep -q '/Volumes/HD-2/jarvis-runner-work-2$' && \
|
||||
echo "OK: pre-stage intact" || echo "FAIL"
|
||||
```
|
||||
|
||||
Expected: `OK: pre-stage intact`.
|
||||
|
||||
If FAIL: re-run the pre-stage block from earlier in the session — `mkdir -p` the dirs on HD-2, download `actions-runner-osx-arm64-2.334.0.tar.gz`, extract into `~/actions-runner-jarvis-2/`, `ln -s /Volumes/HD-2/jarvis-runner-work-2 ~/actions-runner-jarvis-2/_work`.
|
||||
|
||||
- [ ] **Step 2: Get a fresh registration token**
|
||||
|
||||
```bash
|
||||
TOKEN=$(gh api -X POST /repos/KofTwentyTwo/Jarvis/actions/runners/registration-token -q '.token')
|
||||
test -n "$TOKEN" && echo "Token acquired (len=${#TOKEN})" || echo "FAIL"
|
||||
```
|
||||
|
||||
Expected: `Token acquired (len=33)` or similar non-zero length. Token expires in 1 hour; do not pause between this step and step 4.
|
||||
|
||||
If FAIL: check `gh auth status` — the active account must have `workflow` scope on the KofTwentyTwo/Jarvis repo.
|
||||
|
||||
- [ ] **Step 3: Configure runner-2 (unattended)**
|
||||
|
||||
```bash
|
||||
cd ~/actions-runner-jarvis-2
|
||||
./config.sh \
|
||||
--url https://github.com/KofTwentyTwo/Jarvis \
|
||||
--token "$TOKEN" \
|
||||
--name "Grogu-jarvis-2" \
|
||||
--labels "jarvis" \
|
||||
--work "_work" \
|
||||
--unattended \
|
||||
--replace 2>&1 | grep -v -i 'token' | tail -10
|
||||
```
|
||||
|
||||
Expected last lines include `√ Connected to GitHub`, `√ Runner successfully added`, `√ Settings Saved.`. The `grep -v token` strips any line containing the literal token string from the logged output as a defense-in-depth measure (the token itself is short-lived but should not be left in scrollback).
|
||||
|
||||
- [ ] **Step 4: Install + start as LaunchAgent**
|
||||
|
||||
```bash
|
||||
cd ~/actions-runner-jarvis-2
|
||||
./svc.sh install
|
||||
./svc.sh start
|
||||
./svc.sh status
|
||||
```
|
||||
|
||||
Expected last block:
|
||||
```
|
||||
status actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis-2:
|
||||
/Users/james.maes/Library/LaunchAgents/actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis-2.plist
|
||||
|
||||
Started:
|
||||
<PID> 0 actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis-2
|
||||
```
|
||||
|
||||
(PID will vary.)
|
||||
|
||||
- [ ] **Step 5: Verify runner-2 is online via GitHub API**
|
||||
|
||||
```bash
|
||||
gh api /repos/KofTwentyTwo/Jarvis/actions/runners \
|
||||
-q '.runners[] | select(.name=="Grogu-jarvis-2") | {name, status, busy, labels: [.labels[].name]}'
|
||||
```
|
||||
|
||||
Expected:
|
||||
```json
|
||||
{"busy":false,"labels":["self-hosted","macOS","ARM64","jarvis"],"name":"Grogu-jarvis-2","status":"online"}
|
||||
```
|
||||
|
||||
If `status` is anything other than `online`, check `~/Library/Logs/actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis-2/stdout.log` for the runner-side error and stop the plan.
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Register Grogu-jarvis-3
|
||||
|
||||
Identical sequence to Task 2 with `-2` replaced by `-3` everywhere. Per the spec's sequential-validation principle, do NOT start runner-3 until runner-2 is confirmed `status=online, busy=false`.
|
||||
|
||||
**Files:**
|
||||
- System: `~/actions-runner-jarvis-3/` (pre-staged — verify only)
|
||||
- System: `~/actions-runner-jarvis-3/_work` symlink (pre-staged — verify only)
|
||||
- System (new): `~/actions-runner-jarvis-3/.runner` (created by `config.sh`)
|
||||
- System (new): `~/Library/LaunchAgents/actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis-3.plist`
|
||||
|
||||
- [ ] **Step 1: Verify pre-stage**
|
||||
|
||||
```bash
|
||||
test -x ~/actions-runner-jarvis-3/config.sh && \
|
||||
readlink ~/actions-runner-jarvis-3/_work | grep -q '/Volumes/HD-2/jarvis-runner-work-3$' && \
|
||||
echo "OK: pre-stage intact" || echo "FAIL"
|
||||
```
|
||||
|
||||
Expected: `OK: pre-stage intact`.
|
||||
|
||||
- [ ] **Step 2: Get a fresh registration token**
|
||||
|
||||
```bash
|
||||
TOKEN=$(gh api -X POST /repos/KofTwentyTwo/Jarvis/actions/runners/registration-token -q '.token')
|
||||
test -n "$TOKEN" && echo "Token acquired (len=${#TOKEN})" || echo "FAIL"
|
||||
```
|
||||
|
||||
Expected: `Token acquired (len=33)` or similar.
|
||||
|
||||
- [ ] **Step 3: Configure runner-3 (unattended)**
|
||||
|
||||
```bash
|
||||
cd ~/actions-runner-jarvis-3
|
||||
./config.sh \
|
||||
--url https://github.com/KofTwentyTwo/Jarvis \
|
||||
--token "$TOKEN" \
|
||||
--name "Grogu-jarvis-3" \
|
||||
--labels "jarvis" \
|
||||
--work "_work" \
|
||||
--unattended \
|
||||
--replace 2>&1 | grep -v -i 'token' | tail -10
|
||||
```
|
||||
|
||||
Expected last lines: `√ Connected to GitHub`, `√ Runner successfully added`, `√ Settings Saved.`
|
||||
|
||||
- [ ] **Step 4: Install + start as LaunchAgent**
|
||||
|
||||
```bash
|
||||
cd ~/actions-runner-jarvis-3
|
||||
./svc.sh install
|
||||
./svc.sh start
|
||||
./svc.sh status
|
||||
```
|
||||
|
||||
Expected: `Started: <PID> 0 actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis-3`.
|
||||
|
||||
- [ ] **Step 5: Verify runner-3 is online**
|
||||
|
||||
```bash
|
||||
gh api /repos/KofTwentyTwo/Jarvis/actions/runners \
|
||||
-q '.runners[] | select(.name=="Grogu-jarvis-3") | {name, status, busy, labels: [.labels[].name]}'
|
||||
```
|
||||
|
||||
Expected:
|
||||
```json
|
||||
{"busy":false,"labels":["self-hosted","macOS","ARM64","jarvis"],"name":"Grogu-jarvis-3","status":"online"}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Confirm all 3 runners visible**
|
||||
|
||||
```bash
|
||||
gh api /repos/KofTwentyTwo/Jarvis/actions/runners \
|
||||
-q '.runners | length, [.runners[].name] | sort'
|
||||
```
|
||||
|
||||
Expected (exactly):
|
||||
```
|
||||
3
|
||||
["Grogu-jarvis","Grogu-jarvis-2","Grogu-jarvis-3"]
|
||||
```
|
||||
|
||||
If the count is anything other than 3, stop and investigate which runner failed to register.
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Acceptance — observe matrix parallelism
|
||||
|
||||
**Files:** none.
|
||||
|
||||
**Premise:** PR #185 (`fix/agent-cache-text-blocks-23-24`) was cancelled earlier in this session with one previously-failed SPM job. Rerun it to get a CI run with the new runner topology in effect. Watch for ≥3 SPM jobs in `in_progress` simultaneously.
|
||||
|
||||
- [ ] **Step 1: Identify the PR #185 run id**
|
||||
|
||||
```bash
|
||||
RUN_ID=$(gh run list --branch fix/agent-cache-text-blocks-23-24 --workflow=CI --limit 1 --json databaseId -q '.[0].databaseId')
|
||||
echo "PR #185 latest run: $RUN_ID"
|
||||
```
|
||||
|
||||
Expected: a numeric run id, e.g. `25827802988`.
|
||||
|
||||
- [ ] **Step 2: Trigger a full rerun**
|
||||
|
||||
```bash
|
||||
gh run rerun "$RUN_ID" --repo KofTwentyTwo/Jarvis
|
||||
sleep 5
|
||||
gh run view "$RUN_ID" --json status,conclusion -q '"\(.status) / \(.conclusion // "(none)")"'
|
||||
```
|
||||
|
||||
Expected: `queued / (none)` or `in_progress / (none)`. (Using `gh run rerun` without `--failed` reruns the whole workflow so we observe a fresh matrix dispatch across all runners.)
|
||||
|
||||
- [ ] **Step 3: Wait for matrix dispatch, then sample concurrent jobs**
|
||||
|
||||
Wait ~60-90 seconds for the webview/boundary jobs to complete and the SPM matrix to start dispatching.
|
||||
|
||||
```bash
|
||||
sleep 90
|
||||
gh run view "$RUN_ID" --json jobs -q '[.jobs[] | select(.name | startswith("SPM ")) | select(.status == "in_progress") | .name] | "in_progress count: \(length)\nnames: \(.)"'
|
||||
```
|
||||
|
||||
Expected: `in_progress count: 3` (or higher if the matrix has dispatched more quickly than expected) followed by three SPM job names.
|
||||
|
||||
If `in_progress count: 1`: GitHub may have only just started dispatching. Wait another 60 seconds and re-sample. If still 1 after 5 minutes, investigate — possibly one or both new runners are offline. Cross-check with `gh api /repos/.../actions/runners -q '.runners[].busy'` (should show ≥2 `true`).
|
||||
|
||||
- [ ] **Step 4: Cross-verify on Grogu**
|
||||
|
||||
```bash
|
||||
gh api /repos/KofTwentyTwo/Jarvis/actions/runners \
|
||||
-q '[.runners[] | select(.busy == true) | .name] | "busy runners: \(length)\nnames: \(.)"'
|
||||
```
|
||||
|
||||
Expected: `busy runners: 3` and the three runner names. (If 2, the matrix is partially dispatched; if 1, only the original runner is being used — investigate.)
|
||||
|
||||
- [ ] **Step 5: Wait for the run to terminate**
|
||||
|
||||
```bash
|
||||
gh run watch "$RUN_ID" --repo KofTwentyTwo/Jarvis --exit-status
|
||||
echo "Exit: $?"
|
||||
```
|
||||
|
||||
Expected: process exits with 0 and a `✓ run completed with 'success'` message. Wall-clock from trigger to completion should be approximately 5–7 minutes (vs the prior ~13 min single-runner baseline). If the run conclusion is `failure`, inspect failed jobs via `gh run view "$RUN_ID" --log-failed` to determine whether the failure is multi-runner-related (e.g., test-suite race) or pre-existing flakiness (e.g., the AgentCore deadlock seen earlier).
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Update CLAUDE.md with the new runner inventory
|
||||
|
||||
**Files:**
|
||||
- Modify: `CLAUDE.md` (the "Self-hosted CI runner (Grogu)" section)
|
||||
|
||||
- [ ] **Step 1: Create a branch**
|
||||
|
||||
```bash
|
||||
git fetch origin develop
|
||||
git checkout -b docs/multi-runner-inventory origin/develop
|
||||
```
|
||||
|
||||
Expected: `Switched to a new branch 'docs/multi-runner-inventory'`.
|
||||
|
||||
- [ ] **Step 2: Locate the runner section in CLAUDE.md**
|
||||
|
||||
```bash
|
||||
grep -n "Where it lives\|Service:\|jarvis-runner.keychain" CLAUDE.md | head -3
|
||||
```
|
||||
|
||||
Expected: three line numbers pointing into the `### Self-hosted CI runner (Grogu)` section.
|
||||
|
||||
- [ ] **Step 3: Replace the "Where it lives" + "Service" bullets with multi-runner equivalents**
|
||||
|
||||
Edit the two bullets via the `Edit` tool. The OLD text:
|
||||
|
||||
```
|
||||
- **Where it lives:** `~/actions-runner-jarvis/` on Grogu. `_work/` symlinked to `/Volumes/HD-2/jarvis-runner-work/` (1.3 TB) to keep boot-disk pressure off.
|
||||
- **Service:** launchd LaunchAgent `actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis` (`~/Library/LaunchAgents/`). Manage with `cd ~/actions-runner-jarvis && ./svc.sh {status,start,stop}`. Only runs while user is logged in.
|
||||
```
|
||||
|
||||
The NEW text:
|
||||
|
||||
```
|
||||
- **Where they live:** three runners on Grogu — `~/actions-runner-jarvis/` (Grogu-jarvis), `~/actions-runner-jarvis-2/` (Grogu-jarvis-2), `~/actions-runner-jarvis-3/` (Grogu-jarvis-3). Each `_work/` symlinked to `/Volumes/HD-2/jarvis-runner-work{,-2,-3}/` (1.3 TB available, ~10 GB peak per runner) to keep boot-disk pressure off.
|
||||
- **Service:** one launchd LaunchAgent per runner — `actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis{,-2,-3}.plist` in `~/Library/LaunchAgents/`. Manage individually with `cd ~/actions-runner-jarvis-N && ./svc.sh {status,start,stop}` (omit `-N` for runner 1). All three only run while user is logged in.
|
||||
- **Why three:** the 13-package SPM matrix parallelizes across runners. Single-runner serial = ~13 min CI; three-runner parallel = ~5 min. Grogu has 24 cores / 128 GB RAM; 3 concurrent SwiftPM cold builds peak at ~24 cores + ~24 GB, well within budget. See `docs/superpowers/specs/2026-05-13-multi-runner-grogu-design.md`.
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Stage + commit**
|
||||
|
||||
```bash
|
||||
git add CLAUDE.md
|
||||
git diff --cached --stat
|
||||
git commit -m "docs(claude-md): runner inventory updated for 3-runner Grogu setup
|
||||
|
||||
Replaces single-runner 'Where it lives' / 'Service' bullets with the
|
||||
3-runner inventory (Grogu-jarvis, -2, -3). Adds a brief 'Why three'
|
||||
bullet documenting the parallelism win and links to the spec.
|
||||
|
||||
Refs #180"
|
||||
```
|
||||
|
||||
Expected output: `[docs/multi-runner-inventory <sha>] docs(claude-md): runner inventory updated...` and `1 file changed`.
|
||||
|
||||
- [ ] **Step 5: Push + open PR**
|
||||
|
||||
```bash
|
||||
git push -u origin docs/multi-runner-inventory
|
||||
gh pr create --base develop --head docs/multi-runner-inventory \
|
||||
--title "docs(claude-md): runner inventory updated for 3-runner Grogu setup" \
|
||||
--body "Updates CLAUDE.md's \`Self-hosted CI runner\` section to reflect the new 3-runner Grogu topology (Grogu-jarvis, -2, -3) post-multi-runner-expansion. No behavior change — documentation only.
|
||||
|
||||
Spec: \`docs/superpowers/specs/2026-05-13-multi-runner-grogu-design.md\`
|
||||
Plan: \`docs/superpowers/plans/2026-05-13-multi-runner-grogu.md\`
|
||||
|
||||
Refs #180"
|
||||
```
|
||||
|
||||
Expected: the new PR URL is printed. Note the PR number; the runner-side CI for this PR is itself an acceptance test (it will exercise the new 3-runner topology).
|
||||
|
||||
---
|
||||
|
||||
## Definition of done
|
||||
|
||||
- All steps above are checked off.
|
||||
- `gh api /repos/KofTwentyTwo/Jarvis/actions/runners` returns 3 runners, all `status=online`.
|
||||
- The retriggered PR #185 CI run terminates with `conclusion=success` and showed ≥3 SPM jobs `in_progress` simultaneously during execution.
|
||||
- `docs/multi-runner-inventory` PR is open (or merged) updating CLAUDE.md.
|
||||
|
||||
## Self-review (run by author before handoff)
|
||||
|
||||
- ✅ **Spec coverage:** every component in the spec is covered by a task — runner registration (Tasks 2+3), acceptance validation (Task 4), doc update (Task 5), precondition (Task 1).
|
||||
- ✅ **No placeholders:** every step has concrete commands and expected output.
|
||||
- ✅ **Type consistency:** runner names (`Grogu-jarvis-N`), labels (`jarvis`), plist paths, and `_work` symlink targets are spelled identically across tasks.
|
||||
- ✅ **Spec out-of-scope items not present:** no warm-cache work, no monitoring/alerting, no per-runner labels.
|
||||
- ✅ **Rollback path documented in spec, referenced as needed.**
|
||||
@@ -0,0 +1,133 @@
|
||||
# Multi-runner expansion on Grogu — design
|
||||
|
||||
**Date:** 2026-05-13
|
||||
**Status:** approved (brainstorm phase, awaiting implementation plan)
|
||||
**Workstream:** A — local CI capacity scaling
|
||||
**Related issues:** #180 (single-runner setup), #182 (zombie cleanup), #186 (auto-release), #188 (age-filter)
|
||||
|
||||
## Problem
|
||||
|
||||
Develop CI wall-clock is bottlenecked on a single self-hosted runner (`Grogu-jarvis`) serializing the 13-package SwiftPM matrix. Each cold matrix job takes 1–2 minutes, so a full CI run is 13–15 minutes of serial work plus app-build on github-hosted in parallel. Grogu's hardware (24 physical cores, 128 GB RAM, 1.3 TB free on HD-2) is heavily under-utilized — single-runner load is ~5% of capacity.
|
||||
|
||||
Adding sibling runners would parallelize the matrix and roughly cut CI wall-clock by the runner count. Three runners is the recommended target: comfortably within hardware capacity (~24 cores used at peak when all three are mid-SwiftPM-build) without contending with interactive developer work.
|
||||
|
||||
## Goal
|
||||
|
||||
Three self-hosted runners on Grogu (`Grogu-jarvis`, `Grogu-jarvis-2`, `Grogu-jarvis-3`), each with its own `_work/` directory on HD-2, each registered as a LaunchAgent under james.maes's user session, all sharing labels `[self-hosted, macOS, ARM64, jarvis]` so GitHub Actions dispatches matrix jobs to whichever runner is free.
|
||||
|
||||
**Acceptance bar (minimum):** a green develop CI run after registration where `gh run view --json jobs` shows ≥3 SPM matrix jobs in `in_progress` state simultaneously. Warm-cache persistence and runner-health monitoring are explicit non-goals for this phase — they fall to follow-up tickets.
|
||||
|
||||
## Components
|
||||
|
||||
| Runner | Agent dir | `_work` symlink target | LaunchAgent plist | GitHub name |
|
||||
|---|---|---|---|---|
|
||||
| 1 (existing) | `~/actions-runner-jarvis/` | `/Volumes/HD-2/jarvis-runner-work/` | `actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis.plist` | `Grogu-jarvis` |
|
||||
| 2 (new) | `~/actions-runner-jarvis-2/` | `/Volumes/HD-2/jarvis-runner-work-2/` | `actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis-2.plist` | `Grogu-jarvis-2` |
|
||||
| 3 (new) | `~/actions-runner-jarvis-3/` | `/Volumes/HD-2/jarvis-runner-work-3/` | `actions.runner.KofTwentyTwo-Jarvis.Grogu-jarvis-3.plist` | `Grogu-jarvis-3` |
|
||||
|
||||
All three runners share the same label set; jobs targeting `runs-on: [self-hosted, macOS, jarvis]` will dispatch to any free runner.
|
||||
|
||||
Agent binaries are v2.334.0 (osx-arm64) — already extracted into the new runner dirs as a pre-staging step; only `config.sh` + `svc.sh install/start` remain.
|
||||
|
||||
## Precondition
|
||||
|
||||
PR #189 (age-filter for the orphan-process cleanup step) **must be on `develop`** before either new runner starts accepting jobs. The current cleanup step kills ANY `xctest`/`swift-test`/`swift-driver` process at job start; with concurrent runners, sibling cleanup steps would mutually destroy each other's in-flight tests. PR #189 narrows the kill set to processes ≥1 hour old — concurrent CI jobs (typically <10 min) are safe.
|
||||
|
||||
The gating sequence is enforced operationally, not via code: don't run `./svc.sh start` on runner 2 or 3 until `git log origin/develop` shows the age-filter merge.
|
||||
|
||||
## Sequence — register one runner
|
||||
|
||||
For each `N` in `{2, 3}`:
|
||||
|
||||
1. Obtain a fresh registration token (1-hour expiry):
|
||||
```
|
||||
gh api -X POST /repos/KofTwentyTwo/Jarvis/actions/runners/registration-token
|
||||
```
|
||||
2. Configure:
|
||||
```
|
||||
cd ~/actions-runner-jarvis-N
|
||||
./config.sh \
|
||||
--url https://github.com/KofTwentyTwo/Jarvis \
|
||||
--token "$TOKEN" \
|
||||
--name "Grogu-jarvis-N" \
|
||||
--labels "jarvis" \
|
||||
--work "_work" \
|
||||
--unattended \
|
||||
--replace
|
||||
```
|
||||
3. Install as LaunchAgent + start:
|
||||
```
|
||||
./svc.sh install
|
||||
./svc.sh start
|
||||
```
|
||||
4. Verify online via GitHub API:
|
||||
```
|
||||
gh api /repos/KofTwentyTwo/Jarvis/actions/runners -q '.runners[] | select(.name=="Grogu-jarvis-N") | {status, busy, labels}'
|
||||
```
|
||||
Expected: `status=online, busy=false, labels=[self-hosted, macOS, ARM64, jarvis]`.
|
||||
|
||||
Sequential ordering (runner 2 fully online before configuring runner 3) makes any misconfiguration easy to bisect.
|
||||
|
||||
## Acceptance validation
|
||||
|
||||
After both runners are online:
|
||||
|
||||
1. Retrigger a stuck or cancelled CI run via `gh run rerun <id>` (PR #185 already has a failed AgentCore job that can serve as the test bed).
|
||||
2. While the run is in progress, run:
|
||||
```
|
||||
gh run view <id> --json jobs -q '.jobs[] | select(.status == "in_progress") | .name'
|
||||
```
|
||||
Expected: at least 3 SPM matrix job names listed simultaneously, e.g., `SPM Bus`, `SPM Memory`, `SPM Voice`.
|
||||
3. Cross-verify on Grogu:
|
||||
```
|
||||
gh api /repos/KofTwentyTwo/Jarvis/actions/runners -q '.runners[] | {name, busy}'
|
||||
```
|
||||
Expected: at least 3 runners with `busy=true` during peak matrix execution.
|
||||
4. Wait for the run to terminate. Confirm `conclusion = success` (no cross-runner interference caused failures).
|
||||
|
||||
## Failure modes & mitigations
|
||||
|
||||
| Mode | Probability | Mitigation |
|
||||
|---|---|---|
|
||||
| Mutual xctest kill from un-filtered cleanup | High (without #189) | PR #189 precondition — age-filter at ≥1h. Don't start new runners until merged. |
|
||||
| HD-2 disk pressure | Negligible | 1.3 TB free; 3 `_work` dirs × ~10 GB peak = 30 GB. |
|
||||
| CPU/RAM saturation | Low | 3 concurrent SwiftPM builds peak at ~24 cores + ~24 GB. Within 24/128 budget. Headroom for interactive work. |
|
||||
| Keychain races (`unlock-keychain`) | Negligible | `security unlock-keychain` is idempotent; default-keychain set persistently for the user. |
|
||||
| Shared SwiftPM global cache races | Low | `~/Library/Caches/org.swift.swiftpm` is mostly read-after-populate. If contention surfaces, the per-package `actions/checkout clean: true` wipes per-job state cleanly. |
|
||||
| Test port conflicts | Low | Jarvis tests use process-local Mach IPC (WKWebView) and per-process temp DBs (SQLite). No fixed-port servers found in audit. |
|
||||
| One runner goes offline silently | Medium | Surfaces only at the next CI run (other runners pick up jobs). No alerting — explicit follow-up scope. |
|
||||
| Runner agent auto-update breaks during long-running job | Low | Unlikely on a single-job timescale; runner updates happen between jobs. |
|
||||
|
||||
## Rollback
|
||||
|
||||
Each runner is independently revertible.
|
||||
|
||||
- **Pause a runner** (keeps registration): `cd ~/actions-runner-jarvis-N && ./svc.sh stop`
|
||||
- **Deregister a runner** (free the slot):
|
||||
- On Grogu: `./svc.sh uninstall`
|
||||
- Via GitHub: `gh api -X DELETE /repos/KofTwentyTwo/Jarvis/actions/runners/<id>` (or `./config.sh remove --token $(gh api -X POST /repos/.../actions/runners/remove-token)`)
|
||||
- **Revert to single runner**: stop + uninstall + deregister runners 2 and 3. Workflow unchanged (the `runs-on` selector targets the shared label set, not specific runners).
|
||||
|
||||
No code or workflow changes are required to roll back, which is the strongest possible safety property for an infrastructure change.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Warm-cache persistence (`actions/checkout clean: false` or custom DerivedData paths). Saves ~3 min per CI run but adds workflow complexity. Follow-up issue.
|
||||
- Runner-health monitoring / alerting. Currently relies on `gh api .../actions/runners` snapshots and manual log inspection on Grogu. Follow-up issue.
|
||||
- Per-runner-distinct labels (e.g., `runner-2`, `runner-3`). Default identical labels maximize matrix parallelism; per-runner labels would only matter for targeting a specific machine, which we don't need.
|
||||
- Scaling beyond 3 runners. Hardware can support more, but matrix size (13 jobs) caps useful parallelism at ~4 runners. Cross that bridge only if matrix grows.
|
||||
|
||||
## Definition of done
|
||||
|
||||
- `gh api /repos/KofTwentyTwo/Jarvis/actions/runners` shows three runners, all `status=online`.
|
||||
- A retriggered CI run shows ≥3 SPM jobs `in_progress` simultaneously at peak.
|
||||
- The retriggered run terminates `success` (no failures attributable to multi-runner setup).
|
||||
- `CLAUDE.md` updated with the new runner inventory and operational notes.
|
||||
- Spec document committed and merged.
|
||||
|
||||
## References
|
||||
|
||||
- #180 — original single-runner setup
|
||||
- #182 — zombie cleanup
|
||||
- #188 — age-filter for multi-runner safety (precondition)
|
||||
- `~/.claude/plugins/cache/claude-plugins-official/superpowers/5.1.0/skills/brainstorming` — process this design followed
|
||||
@@ -0,0 +1,114 @@
|
||||
# v0.2 Wave 1 sequencing and execution — design
|
||||
|
||||
**Status:** approved 2026-05-14 (autonomous execution authorized)
|
||||
**Authors:** brainstorm with James + Claude (Opus 4.7)
|
||||
**Scope:** Six tiny v0.2 issues (#19, #37, #45, #46, #47, #178) — sequenced into three PRs.
|
||||
|
||||
## Goal
|
||||
|
||||
Land all six Wave 1 issues against `develop` in three coherent PRs, in an order that minimises CI exposure to the known `MemoryWiringEndToEndTests` flake and keeps each PR's narrative reviewable as a single story.
|
||||
|
||||
## Why three PRs, not six and not two
|
||||
|
||||
Six individual PRs (the handoff's implicit default) maximises history granularity but burns 6× the SPM CI matrix and exposes us to the Memory flake on every branch. Two giant PRs maximise throughput but mix surfaces (build script + new workflow + bundle rebuild + Swift code) into a single review and a single revert unit.
|
||||
|
||||
Three PRs grouped by surface area give us:
|
||||
|
||||
- 3× CI runs instead of 6×.
|
||||
- Each PR has one coherent narrative (docs ⋄ infra ⋄ code).
|
||||
- Each PR can be reverted independently without untangling unrelated work.
|
||||
|
||||
## Composition
|
||||
|
||||
### PR 1 — "Doc drift cleanup" (`docs/v0.2-wave-1-doc-drift`)
|
||||
|
||||
**Issues:** #19 (MemoryStore stale Phase-7 doc-comments) + #45 (Bus JarvisBusWorld drift across 20+ docstrings post `fb41c5f`).
|
||||
|
||||
**Why grouped:** both are descriptive/textual changes to documentation that no longer matches the code. #45 already requires a `@jarvis/bus` bundle rebuild and a new grep-gate, so it warrants doc-class company rather than mixing with real code fixes. Both touch package-internal documentation only; no behavioural change.
|
||||
|
||||
**Deliverables:**
|
||||
|
||||
- `packages/Memory/Sources/Memory/MemoryStore.swift` — scrub "Plan 07-02 wires", "Plan 07-03 will replace", "future ops plan" hedges. Keep the 2026-05-11 comment at line 464–468 intact (it is correct).
|
||||
- Bus comments renamed across `WebviewBridge.swift`, `Injection.js`, `webview/packages/bus/src/bridge.ts`, `webview/packages/hud/src/main.tsx`, `App/AppDelegate.swift` — replace "JarvisBusWorld" / "WKContentWorld('JarvisBusWorld')" / cross-world-attach language with "page world" / "shared world" / "defensive double-install guard".
|
||||
- `bridge.ts:60–99` rationale rewritten — the duck-type `isJarvisBusShape` guard is now defensive against double-install (both `Injection.js`'s stub and the bundle's `installJarvisBus` running in the same page world), not cross-world attach. Comment must say this without misleading; do **not** delete the guard itself.
|
||||
- Bundle rebuilt: `webview/packages/bus/dist/bridge.{js,d.ts}` regenerated via `pnpm --filter @jarvis/bus build` (or equivalent).
|
||||
- `@jarvis/bus` `package.json` patch-version bumped to reflect comment changes in dist (matches existing project convention of patch-bumping when the published comments shift).
|
||||
- New grep-gate: `scripts/check-no-jarvis-bus-world-references.sh` — fails if any tracked file outside `.planning/`, `docs/`, or git-history references contains the literal `JarvisBusWorld`. Wired into the boundary-gates list in CI and in `CLAUDE.md`.
|
||||
|
||||
**Closes:** #19, #45.
|
||||
|
||||
### PR 2 — "Build hygiene" (`ci/v0.2-wave-1-build-hygiene`)
|
||||
|
||||
**Issues:** #46 (stale webview bundle artifacts) + #178 (OpenSSF Scorecard workflow).
|
||||
|
||||
**Why grouped:** both touch repo-level infrastructure (`scripts/` and `.github/workflows/`) and neither touches Swift packages, so the heavy SPM matrix runs but with low chance of new test failures. Quick CI; safe to land first.
|
||||
|
||||
**Deliverables:**
|
||||
|
||||
- `scripts/build-webview.sh` — before the existing `rsync -a ... dist/ -> App/Resources/webview/` at line 88, add an explicit cleanup of `App/Resources/webview/assets/*.js`, `*.js.map`, `*.css` (preserving `bus-harness.html`, the directory's `.gitignore`, and any other already-excluded files). Easiest mechanism: pass `--delete-excluded` plus an `--exclude` for each preserve target, or `rm -f` the asset glob before rsync. Either works; preference is `rm -f` of the explicit asset glob because it's grep-able.
|
||||
- `App/Resources/webview/assets/` — delete the 10 stale `index-*.js` files and their `.map` siblings in the same commit so the directory is clean post-merge.
|
||||
- `.github/workflows/scorecard.yml` — new file. Trigger `schedule: cron: "0 6 * * 1"` (weekly Mondays) + `push: branches: [develop]` + `workflow_dispatch`. Permissions: `id-token: write`, `contents: read`, `security-events: write`. Uses `ossf/scorecard-action@v2.4.0` (or latest at execution time). Upload SARIF to GitHub code-scanning.
|
||||
- `README.md` — add OpenSSF Scorecard badge near the top (under the title or alongside existing badges). Badge URL pattern is the standard `https://api.scorecard.dev/projects/github.com/KofTwentyTwo/Jarvis/badge`.
|
||||
|
||||
**Closes:** #46, #178.
|
||||
|
||||
### PR 3 — "Small code fixes" (`fix/v0.2-wave-1-code-fixes`)
|
||||
|
||||
**Issues:** #37 (STTBackendSelector snake_case vs call-site camelCase) + #47 (WebviewBridge NaN→null JSON guard for `audioLevel`).
|
||||
|
||||
**Why grouped:** both are genuine one-shot code fixes with a unit test each, in different packages (Voice/App + Bus). Landing them together gives one Swift-package-test cycle for both fixes.
|
||||
|
||||
**Deliverables for #37:**
|
||||
|
||||
- Decision: standardise on **camelCase** at call sites (matches existing identifier style elsewhere in the codebase). Update the constants in `STTBackendSelector` to camelCase string values, OR delete the unused constants in favour of an enum if call sites can adopt cheaply. Master agent picks the smaller diff after a quick read.
|
||||
- Single rename or constant-shape change; ensure `App/AppDelegate.swift:2028` (and any other call sites a grep reveals) is consistent.
|
||||
- Existing tests must still pass; if none cover the constant identity, add a small one in `packages/Voice/Tests/` asserting the chosen shape.
|
||||
|
||||
**Deliverables for #47:**
|
||||
|
||||
- First verify #47 is still latent or has become active post-#42: grep for the audioLevel consumer in `webview/packages/hud/` and check whether real decode errors are firing in any logged scenario. Either way, ship the clamp.
|
||||
- `packages/Bus/Sources/Bus/WebviewBridge.swift` — in `sendRaw` (or the `BusOutbound.audioLevel.encode` path, whichever is the cleanest place), replace non-finite `rms` floats with `0.0` before encoding. Inline fix only; do not refactor `AudioLevelEmitter` simultaneously (separate concern).
|
||||
- New unit test: `BusOutboundEncodingTests` (or extend the nearest existing suite) asserts `BusOutbound.audioLevel(rms: .nan)` decodes JS-side with `rms == 0.0`. Use the existing harness convention (`Mock*` only if recording, `Stub*` for canned data, `Fake*` for stateful) per CLAUDE.md.
|
||||
|
||||
**Closes:** #37, #47.
|
||||
|
||||
## Landing order
|
||||
|
||||
1. **PR 2 first** — lowest blast radius, lightest CI footprint, validates the multi-runner pipeline on something we can revert in seconds if it breaks.
|
||||
2. **PR 1 second** — purely descriptive, no behavioural change. CI surface is large (Bus + Memory packages, full webview build), but new tests should not fail because no logic moves.
|
||||
3. **PR 3 last** — real code touches. By the time we get here, the Memory flake is the only plausible failure mode, so if a job goes red we know it is the flake.
|
||||
|
||||
## Memory-flake disposition
|
||||
|
||||
Per `docs/HANDOFF-2026-05-14-multi-runner-shipped.md`: `MemoryWiringEndToEndTests` flakes intermittently on PR branches under concurrent-runner load. Branch protection has zero required checks, so flake-induced red is not blocking.
|
||||
|
||||
Master agent policy:
|
||||
|
||||
1. If a PR's CI fails on Memory only, rerun the failed jobs once.
|
||||
2. If they pass on rerun, merge.
|
||||
3. If they fail a second time, leave the PR open with the Memory failure visible and open `bug,severity:medium,area:memory` issue documenting the test as known-flaky in v0.2. Do not merge without user sign-off.
|
||||
4. If any *non-Memory* check fails, stop and report back — do not rerun, do not merge.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Do not touch PRs #185, #191, #192. They are tracked in a separate disposition.
|
||||
- Do not open or address Wave 2/3 issues (#18, #23/#24, #43, #44, #176, #26, M-series epics).
|
||||
- Do not refactor `AudioLevelEmitter` while fixing #47.
|
||||
- Do not change branch protection settings.
|
||||
- Do not bypass pre-commit hooks (`--no-verify`).
|
||||
- Do not force-push.
|
||||
- Do not rebase #185/#191/#192 onto the new tip of develop.
|
||||
|
||||
## Success criteria
|
||||
|
||||
- Issues #19, #37, #45, #46, #47, #178 all closed via merge.
|
||||
- `develop` HEAD's CI is green after all three merges.
|
||||
- `App/Resources/webview/assets/` contains only the active bundle (one `index-*.js` + `.map` + `.css`) after PR 2 merges.
|
||||
- OpenSSF Scorecard workflow appears in Actions → Workflows and has at least one successful run (or a clearly-named permission failure to address in a follow-up).
|
||||
- Grep for `JarvisBusWorld` outside `.planning/`, `docs/`, and committed history returns nothing.
|
||||
- `swift test --package-path packages/Bus` passes including the new NaN-clamp test.
|
||||
- No new test doubles violate the `Mock*`/`Stub*`/`Fake*` convention.
|
||||
|
||||
## Effort budget
|
||||
|
||||
~90 min of coding across the three PRs, plus CI wait time. Master agent should target completion in 90 min wall clock; if it falls behind, it must report partial progress rather than rush.
|
||||
@@ -43,6 +43,20 @@ public struct CacheHints: Sendable, Equatable {
|
||||
/// Once the system prompt grows past the threshold (e.g. memory
|
||||
/// hydration adds context, presence enrichment lengthens the prompt)
|
||||
/// the hint resumes automatically.
|
||||
///
|
||||
/// **Current production state (as of v0.1):** `selfAwarePreamble` is
|
||||
/// ~700 chars and the orchestrator is constructed with an `.empty`
|
||||
/// initial context, so the threshold is never crossed on a healthy
|
||||
/// boot — `eligibleForSystemPrompt` returns `nil` on every turn and
|
||||
/// the prompt cache is effectively dormant. This is intentional
|
||||
/// cost-acceptance: padding the preamble to ≥4096 chars would
|
||||
/// require real prompt-engineering work and risks bleeding
|
||||
/// irrelevant context into the model. Cache hits resume naturally
|
||||
/// once Memory M-4 (v0.8 milestone) lands `searchFacts`-driven
|
||||
/// hydration that grows the prompt past 1024 tokens organically;
|
||||
/// see #23 for the audit context. Until then, every turn pays full
|
||||
/// input-token cost — a ~30–80% cost reduction is on the table once
|
||||
/// M-4 ships.
|
||||
public static func eligibleForSystemPrompt(
|
||||
_ systemPromptText: String,
|
||||
ttl: CacheTTL = .extended1h
|
||||
|
||||
@@ -508,6 +508,15 @@ public actor AgentOrchestrator {
|
||||
// for the current pass. Reset on each pass restart (initial entry +
|
||||
// T1→T2 escalation re-enter via `continue outer`).
|
||||
var assistantTextSoFar: String = ""
|
||||
// #24: text accumulated since the most recent toolUse append (or
|
||||
// since the start of the round if no toolUse yet). Distinct from
|
||||
// `assistantTextSoFar` which stays cumulative for D-02 vision
|
||||
// routing on endTurn. This buffer feeds the assistant message
|
||||
// history alongside each toolUse block so the model sees its own
|
||||
// narration ("Let me check..." → tool_use(get_time)) on subsequent
|
||||
// round-trips; previously the text was dropped entirely from
|
||||
// history and the model would deny ever saying it.
|
||||
var assistantTextSinceLastFlush: String = ""
|
||||
// D-02: escalation gets ONE shot. The second low-confidence outcome
|
||||
// stays on T1 (the user got the response we have). This mirrors the
|
||||
// AGENT-09 retry budget but is independent of it.
|
||||
@@ -575,6 +584,10 @@ public actor AgentOrchestrator {
|
||||
// can hand it to VisionRouter.evaluatePostResponse.
|
||||
// Reset on `continue outer` from the escalation arm.
|
||||
assistantTextSoFar += s
|
||||
// #24: also accumulate into the per-tool-use flush
|
||||
// buffer so the next toolUseRequested attaches the
|
||||
// preceding narration to its assistant message.
|
||||
assistantTextSinceLastFlush += s
|
||||
await replayLog.record(.textDelta(s), for: currentTurnId)
|
||||
await events.send(.tokenDelta(turnId: currentTurnId, text: s))
|
||||
|
||||
@@ -601,9 +614,29 @@ public actor AgentOrchestrator {
|
||||
// history so providers that require the original
|
||||
// tool_use round-trip (Anthropic) can resolve the
|
||||
// tool_use_id when the next tool_result lands.
|
||||
messages.append(LLMMessage(role: .assistant, content: [
|
||||
.toolUse(id: req.id, name: req.name, argsJSON: req.argsJSON),
|
||||
]))
|
||||
//
|
||||
// #24: bundle any text the model emitted since the
|
||||
// last flush — Opus 4.7 commonly narrates ("Let me
|
||||
// check the time...") before calling a tool. Without
|
||||
// this the text was dropped from history and the
|
||||
// model would deny ever saying it on a follow-up.
|
||||
// Trim whitespace-only text to avoid emitting empty
|
||||
// text blocks (which Anthropic rejects with a 400).
|
||||
var assistantContent: [LLMMessage.ContentBlock] = []
|
||||
let pendingText = assistantTextSinceLastFlush
|
||||
.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if !pendingText.isEmpty {
|
||||
assistantContent.append(.text(pendingText))
|
||||
}
|
||||
assistantContent.append(
|
||||
.toolUse(id: req.id, name: req.name, argsJSON: req.argsJSON)
|
||||
)
|
||||
messages.append(LLMMessage(role: .assistant, content: assistantContent))
|
||||
// Consumed — the next toolUseRequested in the same
|
||||
// round (rare but possible) gets only the text that
|
||||
// came BETWEEN the two tool_use blocks. D-02's
|
||||
// cumulative `assistantTextSoFar` is untouched.
|
||||
assistantTextSinceLastFlush = ""
|
||||
|
||||
do {
|
||||
let resultData = try await toolDispatcher.dispatch(toolUse: req)
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
import XCTest
|
||||
import AgentCore
|
||||
import Config
|
||||
import Replay
|
||||
@testable import AgentOrchestrator
|
||||
|
||||
/// **#24 — assistant text blocks alongside tool_use must not be dropped.**
|
||||
///
|
||||
/// Opus 4.7 frequently narrates before invoking a tool ("Let me check the
|
||||
/// time for you..." → `tool_use(get_time)`). Before this fix, the
|
||||
/// orchestrator appended only the `.toolUse` block to `messages`, so on
|
||||
/// the next round-trip the model had no record of saying anything — a
|
||||
/// user follow-up like "what did you say before checking the time?"
|
||||
/// would get a denial.
|
||||
///
|
||||
/// These tests assert the assistant message at the round-trip boundary
|
||||
/// contains BOTH the preceding text and the tool_use block. D-02's
|
||||
/// cumulative `assistantTextSoFar` is preserved separately; only the
|
||||
/// per-flush buffer is consumed.
|
||||
@MainActor
|
||||
final class AssistantTextWithToolUseTests: XCTestCase {
|
||||
|
||||
var tempHome: TempReplayHome!
|
||||
|
||||
override func setUp() async throws {
|
||||
try await super.setUp()
|
||||
tempHome = TempReplayHome.make()
|
||||
}
|
||||
|
||||
override func tearDown() async throws {
|
||||
tempHome?.cleanup()
|
||||
try await super.tearDown()
|
||||
}
|
||||
|
||||
private func buildOrchestrator(
|
||||
provider: MockLLMProvider,
|
||||
dispatcher: any ToolDispatcher
|
||||
) async throws -> AgentOrchestrator {
|
||||
let replay = try ReplayLog(databaseURL: tempHome.dbURL)
|
||||
let session = try await replay.beginSession(appVersion: "test", buildSHA: "deadbeef")
|
||||
return AgentOrchestrator(
|
||||
configStore: makeConfigStore(),
|
||||
providerFactory: { _ in provider },
|
||||
toolDispatcher: dispatcher,
|
||||
replayLog: replay,
|
||||
sessionId: session,
|
||||
systemPrompt: "you are jarvis",
|
||||
availableTools: []
|
||||
)
|
||||
}
|
||||
|
||||
private func collectUntilIdle(orch: AgentOrchestrator, timeout: TimeInterval = 5.0) async {
|
||||
let deadline = Date().addingTimeInterval(timeout)
|
||||
for await event in orch.events {
|
||||
if case .stateChange(.idle) = event { return }
|
||||
if Date() > deadline { return }
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - ATU-1
|
||||
|
||||
/// Single text-then-tool sequence: model emits "Let me check..." then
|
||||
/// `tool_use(get_time)`. The assistant message in the second provider
|
||||
/// call must contain a `.text` block AND a `.toolUse` block.
|
||||
func test_ATU1_textBeforeToolUseBundledIntoSameAssistantMessage() async throws {
|
||||
let toolReq = ToolUseRequest(id: "tu1", name: "get_time", argsJSON: Data("{}".utf8))
|
||||
let mock = MockLLMProvider(scripts: [
|
||||
.init(events: [
|
||||
.messageStart(LLMMessageStart(messageId: "m1", model: "x", usagePrefix: nil)),
|
||||
.textDelta("Let me check the time for you... "),
|
||||
.toolUseRequested(toolReq),
|
||||
.stopReason(.toolUse),
|
||||
.messageStop,
|
||||
]),
|
||||
.init(events: [
|
||||
.messageStart(LLMMessageStart(messageId: "m2", model: "x", usagePrefix: nil)),
|
||||
.textDelta("It's 12:34."),
|
||||
.stopReason(.endTurn),
|
||||
.messageStop,
|
||||
]),
|
||||
])
|
||||
let dispatcher = StubToolDispatcher(dispatch: { _ in Data("12:34".utf8) })
|
||||
|
||||
let orch = try await buildOrchestrator(provider: mock, dispatcher: dispatcher)
|
||||
_ = await orch.submit(.text("what time"))
|
||||
await collectUntilIdle(orch: orch)
|
||||
|
||||
let calls = await mock.getRecordedCalls()
|
||||
XCTAssertEqual(calls.count, 2, "expected 2 provider calls (tool-use loop)")
|
||||
|
||||
// Find the assistant message containing the tool_use block in the
|
||||
// second call's history.
|
||||
let secondCall = calls[1].messages
|
||||
guard let assistantWithTool = secondCall.first(where: { msg in
|
||||
msg.role == .assistant && msg.content.contains(where: { block in
|
||||
if case .toolUse(id: "tu1", _, _) = block { return true }
|
||||
return false
|
||||
})
|
||||
}) else {
|
||||
return XCTFail("no assistant message with the tool_use block found in second call")
|
||||
}
|
||||
|
||||
// Block ordering: text first, then tool_use.
|
||||
let blockKinds = assistantWithTool.content.map { block -> String in
|
||||
switch block {
|
||||
case .text: return "text"
|
||||
case .toolUse: return "toolUse"
|
||||
case .toolResult: return "toolResult"
|
||||
}
|
||||
}
|
||||
XCTAssertEqual(blockKinds, ["text", "toolUse"],
|
||||
"assistant block sequence must be [text, toolUse] — got \(blockKinds)")
|
||||
|
||||
// The text content matches what the model streamed (trimmed).
|
||||
if case .text(let txt) = assistantWithTool.content[0] {
|
||||
XCTAssertEqual(txt, "Let me check the time for you...",
|
||||
"text block must contain the streamed narration (trimmed)")
|
||||
} else {
|
||||
XCTFail("expected first block to be .text")
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - ATU-2
|
||||
|
||||
/// If the model emits a tool_use with NO preceding text (Opus
|
||||
/// occasionally does this — straight to tool), the assistant message
|
||||
/// must contain ONLY the toolUse block (no empty `.text("")` block,
|
||||
/// which Anthropic rejects with a 400).
|
||||
func test_ATU2_noPrecedingTextOnlyToolUseBlock() async throws {
|
||||
let toolReq = ToolUseRequest(id: "tu2", name: "get_time", argsJSON: Data("{}".utf8))
|
||||
let mock = MockLLMProvider(scripts: [
|
||||
.init(events: [
|
||||
.messageStart(LLMMessageStart(messageId: "m1", model: "x", usagePrefix: nil)),
|
||||
.toolUseRequested(toolReq),
|
||||
.stopReason(.toolUse),
|
||||
.messageStop,
|
||||
]),
|
||||
.init(events: [
|
||||
.messageStart(LLMMessageStart(messageId: "m2", model: "x", usagePrefix: nil)),
|
||||
.textDelta("done"),
|
||||
.stopReason(.endTurn),
|
||||
.messageStop,
|
||||
]),
|
||||
])
|
||||
let dispatcher = StubToolDispatcher(dispatch: { _ in Data("12:34".utf8) })
|
||||
|
||||
let orch = try await buildOrchestrator(provider: mock, dispatcher: dispatcher)
|
||||
_ = await orch.submit(.text("time"))
|
||||
await collectUntilIdle(orch: orch)
|
||||
|
||||
let calls = await mock.getRecordedCalls()
|
||||
guard let assistantWithTool = calls[1].messages.first(where: { msg in
|
||||
msg.role == .assistant && msg.content.contains(where: { block in
|
||||
if case .toolUse(id: "tu2", _, _) = block { return true }
|
||||
return false
|
||||
})
|
||||
}) else {
|
||||
return XCTFail("expected assistant message with tool_use block")
|
||||
}
|
||||
|
||||
XCTAssertEqual(assistantWithTool.content.count, 1,
|
||||
"no preceding text — assistant message must contain only the toolUse block, got \(assistantWithTool.content.count) blocks")
|
||||
if case .toolUse = assistantWithTool.content[0] {
|
||||
// ok
|
||||
} else {
|
||||
XCTFail("expected sole content block to be .toolUse")
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - ATU-3
|
||||
|
||||
/// Whitespace-only preceding text must NOT be emitted as a `.text`
|
||||
/// block (Anthropic rejects empty/whitespace-only text blocks). The
|
||||
/// implementation trims whitespace before testing for emptiness.
|
||||
func test_ATU3_whitespaceOnlyTextNotEmitted() async throws {
|
||||
let toolReq = ToolUseRequest(id: "tu3", name: "get_time", argsJSON: Data("{}".utf8))
|
||||
let mock = MockLLMProvider(scripts: [
|
||||
.init(events: [
|
||||
.messageStart(LLMMessageStart(messageId: "m1", model: "x", usagePrefix: nil)),
|
||||
.textDelta(" \n\t "), // whitespace only
|
||||
.toolUseRequested(toolReq),
|
||||
.stopReason(.toolUse),
|
||||
.messageStop,
|
||||
]),
|
||||
.init(events: [
|
||||
.messageStart(LLMMessageStart(messageId: "m2", model: "x", usagePrefix: nil)),
|
||||
.stopReason(.endTurn),
|
||||
.messageStop,
|
||||
]),
|
||||
])
|
||||
let dispatcher = StubToolDispatcher(dispatch: { _ in Data("ok".utf8) })
|
||||
|
||||
let orch = try await buildOrchestrator(provider: mock, dispatcher: dispatcher)
|
||||
_ = await orch.submit(.text("hi"))
|
||||
await collectUntilIdle(orch: orch)
|
||||
|
||||
let calls = await mock.getRecordedCalls()
|
||||
guard let assistantWithTool = calls[1].messages.first(where: { msg in
|
||||
msg.role == .assistant && msg.content.contains(where: { block in
|
||||
if case .toolUse(id: "tu3", _, _) = block { return true }
|
||||
return false
|
||||
})
|
||||
}) else {
|
||||
return XCTFail("expected assistant message with tool_use block")
|
||||
}
|
||||
|
||||
XCTAssertEqual(assistantWithTool.content.count, 1,
|
||||
"whitespace-only preceding text must be dropped, leaving only the toolUse block")
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,7 @@ import OllamaProvider
|
||||
//
|
||||
// Gated by JARVIS_REAL_MODELS=1 — skipped in CI. Runs interactively against
|
||||
// a real local Ollama with both `nomic-embed-text` (embedding) and
|
||||
// `qwen2.5-coder:32b` (extractor) pulled.
|
||||
// `qwen3.6:latest` (extractor, per commit 8d0a959) pulled.
|
||||
//
|
||||
// Run interactively on the developer's Apple Silicon Mac:
|
||||
// JARVIS_REAL_MODELS=1 swift test --package-path packages/Memory --filter MemoryRegressionCorpusTests
|
||||
@@ -37,7 +37,7 @@ final class MemoryRegressionCorpusTests: XCTestCase {
|
||||
private func skipIfNotRealModels() throws {
|
||||
try XCTSkipUnless(
|
||||
ProcessInfo.processInfo.environment["JARVIS_REAL_MODELS"] == "1",
|
||||
"Set JARVIS_REAL_MODELS=1 to run memory regression corpus (requires local Ollama with nomic-embed-text + qwen2.5-coder:32b, plus a loadable vec0.dylib)"
|
||||
"Set JARVIS_REAL_MODELS=1 to run memory regression corpus (requires local Ollama with nomic-embed-text + qwen3.6:latest, plus a loadable vec0.dylib)"
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user