Files
VisionPOC/Sources/App/MainWindowController.swift
T
james.maesandClaude Opus 4.7 79ecd0184a Tier 1 + Tier 2 polish: 3 parallel agents — overlays, snapshots, menus
Coordinated three agents working in parallel on non-overlapping files,
with a shared `ProcessedFrame.swift` contract locking the API surface
between detector and renderer. All build clean on first integrate.

## Detector (ObjectDetector.swift)
- `VNRecognizeTextRequest` runs alongside YOLO at ~5 Hz; recognized
  strings + boxes published via `currentTextDetections()`
- `VNDetectHumanBodyPoseRequest` produces a 14-segment skeleton (nose-
  neck-shoulders-arms, neck-root-hips-legs); confidence-gated at 0.3
- `VNDetectHumanHandPoseRequest` produces 5 finger chains + palm ring
  per hand, up to 2 hands; published as kind=.hand poses
- Body + hand published as `[PoseDetection]` via `currentPoses()`
- New `lastDetectMode: DetectMode { .yolo | .track }` so the footer can
  distinguish the two latency regimes
- YOLO warmup runs one dummy inference at startup on the detector
  queue — first user-visible detection no longer pays the ~200ms
  model-compile latency

## Renderer (Renderer.swift + Render/ColorHash.swift)
- Box labels include 10%-bucketed confidence: "CUP 80%", "MOBILE PHONE
  60%". Bucket keeps the label texture cache from exploding.
- Per-track stable color via UUID-byte → golden-ratio hash → HSV(h,
  0.8, 1.0). Face labels stay cyan (calmer when JARVIS pane has
  multiple labeled people in frame).
- Detection labels clamp to pane edges so boxes near the right margin
  don't overflow into the next pane.
- Footer now reads "DET 28.4ms (yolo)" or "DET 2.1ms (track)" — the
  flicker between the two values is informative, not a bug.
- LRU bound on `labelTextureCache` (128 entries, parallel access-order
  array, drop-head-on-insert when at capacity).
- OCR text rendered as magenta boxes with truncated text labels (32
  chars).
- Body pose drawn in mint, hand pose in amber; axis-aligned segment
  rects with a 0.003 minimum thickness so horizontal/vertical lines
  don't vanish at small viewport sizes. Joint dots at each unique
  point.
- New `snapshotPNG() -> Data?` public method. Render loop piggybacks
  one extra blit pass + waitUntilCompleted + CGImage encoding. <50ms
  in practice, 1-second semaphore timeout if rendering stalls.

## App / Window / Capture
- Camera menu with a dynamic Devices submenu (rebuilt via
  NSMenuDelegate.menuNeedsUpdate). Active device gets a checkmark,
  Refresh List forces re-enumeration.
- CameraCapture.switchToDevice(_:) stops the session, swaps the input,
  restarts. activeDeviceUniqueID exposed so the menu can match.
- Save Snapshot (⌘S) writes the 4-pane composite to
  ~/Desktop/VisionPOC-YYYY-MM-DD-HHMMSS.png
- Window-frame persistence for all three windows via UserDefaults
  keys VPOC.MainWindowFrame / VPOC.SettingsWindowFrame /
  VPOC.LogWindowFrame
- AVCaptureDevice.wasDisconnectedNotification → session stop + flag
  set; permissionDenied returns true so the existing
  "WAITING FOR CAMERA…" status path triggers without renderer changes

## Shared contract (new file)
- Sources/Process/ProcessedFrame.swift — TextDetection, PoseDetection,
  PoseDetection.Segment, PoseKind (.body | .hand), DetectMode
  (.yolo | .track). Sendable, value types, normalized [0..1] Vision
  coords throughout.

## Theme
- Palette gains textStroke (magenta), bodyPose (mint), handPose (amber)
  for the new overlays.

Note: Orbitron font and app icon (also Tier 2 items) deferred — no
clean source available for either without manual work.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-16 16:39:43 -05:00

142 lines
4.6 KiB
Swift

import AppKit
import AVFoundation
import MetalKit
final class MainWindowController: NSWindowController, NSWindowDelegate {
private let mtkView: MTKView
private let capture: CameraCapture
let detector: ObjectDetector
private let edgePass: EdgePass
private let jarvisPass: JarvisStylePass
private let asciiPass: AsciiPass
let renderer: Renderer
private static let frameDefaultsKey = "VPOC.MainWindowFrame"
init() {
guard let device = MTLCreateSystemDefaultDevice() else {
fatalError("Metal is required: no default system device available.")
}
let defaultSize = Theme.Layout.defaultWindowSize
let minSize = Theme.Layout.minWindowSize
let styleMask: NSWindow.StyleMask = [.titled, .closable, .miniaturizable, .resizable]
let window = NSWindow(
contentRect: NSRect(origin: .zero, size: defaultSize),
styleMask: styleMask,
backing: .buffered,
defer: false
)
window.title = "VisionPOC"
window.minSize = minSize
window.appearance = NSAppearance(named: .darkAqua)
window.isReleasedWhenClosed = false
window.center()
let view = MTKView(frame: NSRect(origin: .zero, size: defaultSize), device: device)
view.colorPixelFormat = .bgra8Unorm
view.preferredFramesPerSecond = Int(Theme.Tick.renderFPS)
view.framebufferOnly = true
let bg = Theme.Palette.background
view.clearColor = MTLClearColor(red: Double(bg.x), green: Double(bg.y), blue: Double(bg.z), alpha: Double(bg.w))
view.autoResizeDrawable = true
view.autoresizingMask = [.width, .height]
window.contentView = view
self.mtkView = view
let capture = CameraCapture(device: device)
let detector = ObjectDetector()
let edgePass = EdgePass(device: device)
let jarvisPass = JarvisStylePass()
guard let asciiPass = AsciiPass(device: device) else {
fatalError("Failed to build the ASCII atlas.")
}
guard let renderer = Renderer(
view: view,
device: device,
capture: capture,
detector: detector,
edgePass: edgePass,
jarvisPass: jarvisPass,
asciiPass: asciiPass
) else {
fatalError("Failed to initialize Renderer.")
}
self.capture = capture
self.detector = detector
self.edgePass = edgePass
self.jarvisPass = jarvisPass
self.asciiPass = asciiPass
self.renderer = renderer
view.delegate = renderer
super.init(window: window)
window.delegate = self
restoreFrame()
}
required init?(coder: NSCoder) {
fatalError("init(coder:) is not supported")
}
override func showWindow(_ sender: Any?) {
super.showWindow(sender)
window?.makeKeyAndOrderFront(sender)
Task { await capture.start() }
}
// MARK: - Camera
var activeCameraUniqueID: String? {
capture.activeDeviceUniqueID
}
func switchCamera(to device: AVCaptureDevice) {
capture.switchToDevice(device)
}
// MARK: - Snapshot
@MainActor
func saveSnapshotToDesktop() {
guard let data = renderer.snapshotPNG() else {
LogStream.shared.log("snapshot failed (renderer returned nil)", level: .warn, source: .app)
return
}
let df = DateFormatter()
df.dateFormat = "yyyy-MM-dd-HHmmss"
let name = "VisionPOC-\(df.string(from: Date())).png"
let url = FileManager.default.urls(for: .desktopDirectory, in: .userDomainMask)[0].appendingPathComponent(name)
do {
try data.write(to: url)
LogStream.shared.log("saved \(url.lastPathComponent)", level: .info, source: .app)
} catch {
LogStream.shared.log("snapshot write failed: \(error)", level: .error, source: .app)
}
}
// MARK: - Window persistence
private func restoreFrame() {
guard let window,
let saved = UserDefaults.standard.string(forKey: MainWindowController.frameDefaultsKey) else {
return
}
let rect = NSRectFromString(saved)
if rect.size.width > 0 && rect.size.height > 0 {
window.setFrame(rect, display: false)
}
}
func windowWillClose(_ notification: Notification) {
guard let window else { return }
UserDefaults.standard.set(NSStringFromRect(window.frame),
forKey: MainWindowController.frameDefaultsKey)
}
}