Tier 1 + Tier 2 polish: 3 parallel agents — overlays, snapshots, menus
Coordinated three agents working in parallel on non-overlapping files,
with a shared `ProcessedFrame.swift` contract locking the API surface
between detector and renderer. All build clean on first integrate.
## Detector (ObjectDetector.swift)
- `VNRecognizeTextRequest` runs alongside YOLO at ~5 Hz; recognized
strings + boxes published via `currentTextDetections()`
- `VNDetectHumanBodyPoseRequest` produces a 14-segment skeleton (nose-
neck-shoulders-arms, neck-root-hips-legs); confidence-gated at 0.3
- `VNDetectHumanHandPoseRequest` produces 5 finger chains + palm ring
per hand, up to 2 hands; published as kind=.hand poses
- Body + hand published as `[PoseDetection]` via `currentPoses()`
- New `lastDetectMode: DetectMode { .yolo | .track }` so the footer can
distinguish the two latency regimes
- YOLO warmup runs one dummy inference at startup on the detector
queue — first user-visible detection no longer pays the ~200ms
model-compile latency
## Renderer (Renderer.swift + Render/ColorHash.swift)
- Box labels include 10%-bucketed confidence: "CUP 80%", "MOBILE PHONE
60%". Bucket keeps the label texture cache from exploding.
- Per-track stable color via UUID-byte → golden-ratio hash → HSV(h,
0.8, 1.0). Face labels stay cyan (calmer when JARVIS pane has
multiple labeled people in frame).
- Detection labels clamp to pane edges so boxes near the right margin
don't overflow into the next pane.
- Footer now reads "DET 28.4ms (yolo)" or "DET 2.1ms (track)" — the
flicker between the two values is informative, not a bug.
- LRU bound on `labelTextureCache` (128 entries, parallel access-order
array, drop-head-on-insert when at capacity).
- OCR text rendered as magenta boxes with truncated text labels (32
chars).
- Body pose drawn in mint, hand pose in amber; axis-aligned segment
rects with a 0.003 minimum thickness so horizontal/vertical lines
don't vanish at small viewport sizes. Joint dots at each unique
point.
- New `snapshotPNG() -> Data?` public method. Render loop piggybacks
one extra blit pass + waitUntilCompleted + CGImage encoding. <50ms
in practice, 1-second semaphore timeout if rendering stalls.
## App / Window / Capture
- Camera menu with a dynamic Devices submenu (rebuilt via
NSMenuDelegate.menuNeedsUpdate). Active device gets a checkmark,
Refresh List forces re-enumeration.
- CameraCapture.switchToDevice(_:) stops the session, swaps the input,
restarts. activeDeviceUniqueID exposed so the menu can match.
- Save Snapshot (⌘S) writes the 4-pane composite to
~/Desktop/VisionPOC-YYYY-MM-DD-HHMMSS.png
- Window-frame persistence for all three windows via UserDefaults
keys VPOC.MainWindowFrame / VPOC.SettingsWindowFrame /
VPOC.LogWindowFrame
- AVCaptureDevice.wasDisconnectedNotification → session stop + flag
set; permissionDenied returns true so the existing
"WAITING FOR CAMERA…" status path triggers without renderer changes
## Shared contract (new file)
- Sources/Process/ProcessedFrame.swift — TextDetection, PoseDetection,
PoseDetection.Segment, PoseKind (.body | .hand), DetectMode
(.yolo | .track). Sendable, value types, normalized [0..1] Vision
coords throughout.
## Theme
- Palette gains textStroke (magenta), bodyPose (mint), handPose (amber)
for the new overlays.
Note: Orbitron font and app icon (also Tier 2 items) deferred — no
clean source available for either without manual work.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
1 parent
0d23d7ede9
commit
79ecd0184a
10 files changed
+1026
-21
No files matched your search
@@ -1,12 +1,14 @@
|
||||
import AppKit
|
||||
import AVFoundation
|
||||
import CoreText
|
||||
import Vision
|
||||
|
||||
@main
|
||||
final class AppDelegate: NSObject, NSApplicationDelegate, @unchecked Sendable {
|
||||
final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate, @unchecked Sendable {
|
||||
private var windowController: MainWindowController?
|
||||
@MainActor private var settingsController: SettingsWindowController?
|
||||
@MainActor private var logController: LogStreamWindowController?
|
||||
@MainActor private var cameraDevicesSubmenu: NSMenu?
|
||||
|
||||
static func main() {
|
||||
let app = NSApplication.shared
|
||||
@@ -63,6 +65,17 @@ final class AppDelegate: NSObject, NSApplicationDelegate, @unchecked Sendable {
|
||||
|
||||
appMenu.addItem(.separator())
|
||||
|
||||
let snapshot = NSMenuItem(
|
||||
title: "Save Snapshot",
|
||||
action: #selector(saveSnapshot(_:)),
|
||||
keyEquivalent: "s"
|
||||
)
|
||||
snapshot.keyEquivalentModifierMask = [.command]
|
||||
snapshot.target = self
|
||||
appMenu.addItem(snapshot)
|
||||
|
||||
appMenu.addItem(.separator())
|
||||
|
||||
let quit = NSMenuItem(
|
||||
title: "Quit \(appName)",
|
||||
action: #selector(NSApplication.terminate(_:)),
|
||||
@@ -97,9 +110,93 @@ final class AppDelegate: NSObject, NSApplicationDelegate, @unchecked Sendable {
|
||||
|
||||
facesItem.submenu = facesMenu
|
||||
|
||||
// Camera menu — device picker (dynamic) + manual refresh.
|
||||
let cameraItem = NSMenuItem()
|
||||
mainMenu.addItem(cameraItem)
|
||||
let cameraMenu = NSMenu(title: "Camera")
|
||||
|
||||
let devicesItem = NSMenuItem(title: "Devices", action: nil, keyEquivalent: "")
|
||||
let devicesSubmenu = NSMenu(title: "Devices")
|
||||
devicesSubmenu.delegate = self
|
||||
devicesItem.submenu = devicesSubmenu
|
||||
cameraDevicesSubmenu = devicesSubmenu
|
||||
cameraMenu.addItem(devicesItem)
|
||||
|
||||
let refreshItem = NSMenuItem(
|
||||
title: "Refresh List",
|
||||
action: #selector(refreshCameraList(_:)),
|
||||
keyEquivalent: ""
|
||||
)
|
||||
refreshItem.target = self
|
||||
cameraMenu.addItem(refreshItem)
|
||||
|
||||
cameraItem.submenu = cameraMenu
|
||||
|
||||
NSApplication.shared.mainMenu = mainMenu
|
||||
}
|
||||
|
||||
// MARK: - Camera menu (dynamic)
|
||||
|
||||
@MainActor
|
||||
@objc func menuNeedsUpdate(_ menu: NSMenu) {
|
||||
guard menu === cameraDevicesSubmenu else { return }
|
||||
rebuildCameraDevicesMenu(menu)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private func rebuildCameraDevicesMenu(_ menu: NSMenu) {
|
||||
menu.removeAllItems()
|
||||
|
||||
let discovery = AVCaptureDevice.DiscoverySession(
|
||||
deviceTypes: [.builtInWideAngleCamera, .external, .continuityCamera],
|
||||
mediaType: .video,
|
||||
position: .unspecified
|
||||
)
|
||||
let devices = discovery.devices
|
||||
|
||||
if devices.isEmpty {
|
||||
let empty = NSMenuItem(title: "No cameras found", action: nil, keyEquivalent: "")
|
||||
empty.isEnabled = false
|
||||
menu.addItem(empty)
|
||||
return
|
||||
}
|
||||
|
||||
let activeID = windowController?.activeCameraUniqueID
|
||||
for device in devices {
|
||||
let item = NSMenuItem(
|
||||
title: device.localizedName,
|
||||
action: #selector(selectCameraDevice(_:)),
|
||||
keyEquivalent: ""
|
||||
)
|
||||
item.target = self
|
||||
item.representedObject = device
|
||||
if device.uniqueID == activeID {
|
||||
item.state = .on
|
||||
}
|
||||
menu.addItem(item)
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
@objc private func refreshCameraList(_ sender: Any?) {
|
||||
guard let menu = cameraDevicesSubmenu else { return }
|
||||
rebuildCameraDevicesMenu(menu)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
@objc private func selectCameraDevice(_ sender: Any?) {
|
||||
guard let item = sender as? NSMenuItem,
|
||||
let device = item.representedObject as? AVCaptureDevice else { return }
|
||||
windowController?.switchCamera(to: device)
|
||||
}
|
||||
|
||||
// MARK: - Snapshot
|
||||
|
||||
@MainActor
|
||||
@objc private func saveSnapshot(_ sender: Any?) {
|
||||
windowController?.saveSnapshotToDesktop()
|
||||
}
|
||||
|
||||
// MARK: - Settings
|
||||
|
||||
@MainActor
|
||||
|
||||
@@ -2,7 +2,9 @@ import AppKit
|
||||
import SwiftUI
|
||||
|
||||
@MainActor
|
||||
final class LogStreamWindowController: NSWindowController {
|
||||
final class LogStreamWindowController: NSWindowController, NSWindowDelegate {
|
||||
private static let frameDefaultsKey = "VPOC.LogWindowFrame"
|
||||
|
||||
init() {
|
||||
let hosting = NSHostingController(rootView: LogStreamView())
|
||||
let window = NSWindow(contentViewController: hosting)
|
||||
@@ -12,9 +14,29 @@ final class LogStreamWindowController: NSWindowController {
|
||||
window.isReleasedWhenClosed = false
|
||||
window.center()
|
||||
super.init(window: window)
|
||||
|
||||
window.delegate = self
|
||||
restoreFrame()
|
||||
}
|
||||
|
||||
required init?(coder: NSCoder) {
|
||||
fatalError("init(coder:) is not supported")
|
||||
}
|
||||
|
||||
private func restoreFrame() {
|
||||
guard let window,
|
||||
let saved = UserDefaults.standard.string(forKey: LogStreamWindowController.frameDefaultsKey) else {
|
||||
return
|
||||
}
|
||||
let rect = NSRectFromString(saved)
|
||||
if rect.size.width > 0 && rect.size.height > 0 {
|
||||
window.setFrame(rect, display: false)
|
||||
}
|
||||
}
|
||||
|
||||
func windowWillClose(_ notification: Notification) {
|
||||
guard let window else { return }
|
||||
UserDefaults.standard.set(NSStringFromRect(window.frame),
|
||||
forKey: LogStreamWindowController.frameDefaultsKey)
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,17 @@
|
||||
import AppKit
|
||||
import AVFoundation
|
||||
import MetalKit
|
||||
|
||||
final class MainWindowController: NSWindowController {
|
||||
final class MainWindowController: NSWindowController, NSWindowDelegate {
|
||||
private let mtkView: MTKView
|
||||
private let capture: CameraCapture
|
||||
let detector: ObjectDetector
|
||||
private let edgePass: EdgePass
|
||||
private let jarvisPass: JarvisStylePass
|
||||
private let asciiPass: AsciiPass
|
||||
private let renderer: Renderer
|
||||
let renderer: Renderer
|
||||
|
||||
private static let frameDefaultsKey = "VPOC.MainWindowFrame"
|
||||
|
||||
init() {
|
||||
guard let device = MTLCreateSystemDefaultDevice() else {
|
||||
@@ -72,6 +75,9 @@ final class MainWindowController: NSWindowController {
|
||||
view.delegate = renderer
|
||||
|
||||
super.init(window: window)
|
||||
|
||||
window.delegate = self
|
||||
restoreFrame()
|
||||
}
|
||||
|
||||
required init?(coder: NSCoder) {
|
||||
@@ -83,4 +89,53 @@ final class MainWindowController: NSWindowController {
|
||||
window?.makeKeyAndOrderFront(sender)
|
||||
Task { await capture.start() }
|
||||
}
|
||||
|
||||
// MARK: - Camera
|
||||
|
||||
var activeCameraUniqueID: String? {
|
||||
capture.activeDeviceUniqueID
|
||||
}
|
||||
|
||||
func switchCamera(to device: AVCaptureDevice) {
|
||||
capture.switchToDevice(device)
|
||||
}
|
||||
|
||||
// MARK: - Snapshot
|
||||
|
||||
@MainActor
|
||||
func saveSnapshotToDesktop() {
|
||||
guard let data = renderer.snapshotPNG() else {
|
||||
LogStream.shared.log("snapshot failed (renderer returned nil)", level: .warn, source: .app)
|
||||
return
|
||||
}
|
||||
let df = DateFormatter()
|
||||
df.dateFormat = "yyyy-MM-dd-HHmmss"
|
||||
let name = "VisionPOC-\(df.string(from: Date())).png"
|
||||
let url = FileManager.default.urls(for: .desktopDirectory, in: .userDomainMask)[0].appendingPathComponent(name)
|
||||
do {
|
||||
try data.write(to: url)
|
||||
LogStream.shared.log("saved \(url.lastPathComponent)", level: .info, source: .app)
|
||||
} catch {
|
||||
LogStream.shared.log("snapshot write failed: \(error)", level: .error, source: .app)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Window persistence
|
||||
|
||||
private func restoreFrame() {
|
||||
guard let window,
|
||||
let saved = UserDefaults.standard.string(forKey: MainWindowController.frameDefaultsKey) else {
|
||||
return
|
||||
}
|
||||
let rect = NSRectFromString(saved)
|
||||
if rect.size.width > 0 && rect.size.height > 0 {
|
||||
window.setFrame(rect, display: false)
|
||||
}
|
||||
}
|
||||
|
||||
func windowWillClose(_ notification: Notification) {
|
||||
guard let window else { return }
|
||||
UserDefaults.standard.set(NSStringFromRect(window.frame),
|
||||
forKey: MainWindowController.frameDefaultsKey)
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,9 @@ import AppKit
|
||||
import SwiftUI
|
||||
|
||||
@MainActor
|
||||
final class SettingsWindowController: NSWindowController {
|
||||
final class SettingsWindowController: NSWindowController, NSWindowDelegate {
|
||||
private static let frameDefaultsKey = "VPOC.SettingsWindowFrame"
|
||||
|
||||
init() {
|
||||
let hosting = NSHostingController(rootView: SettingsView())
|
||||
let window = NSWindow(contentViewController: hosting)
|
||||
@@ -12,9 +14,29 @@ final class SettingsWindowController: NSWindowController {
|
||||
window.isReleasedWhenClosed = false
|
||||
window.center()
|
||||
super.init(window: window)
|
||||
|
||||
window.delegate = self
|
||||
restoreFrame()
|
||||
}
|
||||
|
||||
required init?(coder: NSCoder) {
|
||||
fatalError("init(coder:) is not supported")
|
||||
}
|
||||
|
||||
private func restoreFrame() {
|
||||
guard let window,
|
||||
let saved = UserDefaults.standard.string(forKey: SettingsWindowController.frameDefaultsKey) else {
|
||||
return
|
||||
}
|
||||
let rect = NSRectFromString(saved)
|
||||
if rect.size.width > 0 && rect.size.height > 0 {
|
||||
window.setFrame(rect, display: false)
|
||||
}
|
||||
}
|
||||
|
||||
func windowWillClose(_ notification: Notification) {
|
||||
guard let window else { return }
|
||||
UserDefaults.standard.set(NSStringFromRect(window.frame),
|
||||
forKey: SettingsWindowController.frameDefaultsKey)
|
||||
}
|
||||
}
|
||||
@@ -22,12 +22,16 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
||||
private let sessionQueue = DispatchQueue(label: "com.dmdbrands.VisionPOC.capture.session")
|
||||
private let sampleQueue = DispatchQueue(label: "com.dmdbrands.VisionPOC.capture.samples")
|
||||
private var textureCache: CVMetalTextureCache?
|
||||
private var currentInput: AVCaptureDeviceInput?
|
||||
private var output: AVCaptureVideoDataOutput?
|
||||
|
||||
private var lock = os_unfair_lock_s()
|
||||
private var latest: Frame?
|
||||
private var _permissionDenied = false
|
||||
private var _deviceDisconnected = false
|
||||
private var _width: Int = 1920
|
||||
private var _height: Int = 1080
|
||||
private var _activeDeviceUniqueID: String?
|
||||
|
||||
init(device: MTLDevice) {
|
||||
self.device = device
|
||||
@@ -38,12 +42,31 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
||||
NSLog("CVMetalTextureCacheCreate failed: \(status)")
|
||||
}
|
||||
self.textureCache = cache
|
||||
|
||||
NotificationCenter.default.addObserver(
|
||||
self,
|
||||
selector: #selector(handleDeviceDisconnected(_:)),
|
||||
name: .AVCaptureDeviceWasDisconnected,
|
||||
object: nil
|
||||
)
|
||||
}
|
||||
|
||||
deinit {
|
||||
NotificationCenter.default.removeObserver(self)
|
||||
}
|
||||
|
||||
var permissionDenied: Bool {
|
||||
os_unfair_lock_lock(&lock)
|
||||
defer { os_unfair_lock_unlock(&lock) }
|
||||
return _permissionDenied
|
||||
// Treat disconnect as "permission denied" so renderer falls back to its
|
||||
// existing "WAITING FOR CAMERA…" status path without modification.
|
||||
return _permissionDenied || _deviceDisconnected
|
||||
}
|
||||
|
||||
var deviceDisconnected: Bool {
|
||||
os_unfair_lock_lock(&lock)
|
||||
defer { os_unfair_lock_unlock(&lock) }
|
||||
return _deviceDisconnected
|
||||
}
|
||||
|
||||
var width: Int {
|
||||
@@ -58,6 +81,12 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
||||
return _height
|
||||
}
|
||||
|
||||
var activeDeviceUniqueID: String? {
|
||||
os_unfair_lock_lock(&lock)
|
||||
defer { os_unfair_lock_unlock(&lock) }
|
||||
return _activeDeviceUniqueID
|
||||
}
|
||||
|
||||
func latestTexture() -> MTLTexture? {
|
||||
os_unfair_lock_lock(&lock)
|
||||
defer { os_unfair_lock_unlock(&lock) }
|
||||
@@ -108,7 +137,7 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
||||
}
|
||||
|
||||
let discovery = AVCaptureDevice.DiscoverySession(
|
||||
deviceTypes: [.builtInWideAngleCamera, .external],
|
||||
deviceTypes: [.builtInWideAngleCamera, .external, .continuityCamera],
|
||||
mediaType: .video,
|
||||
position: .unspecified
|
||||
)
|
||||
@@ -129,6 +158,7 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
||||
let input = try AVCaptureDeviceInput(device: camera)
|
||||
if self.session.canAddInput(input) {
|
||||
self.session.addInput(input)
|
||||
self.currentInput = input
|
||||
}
|
||||
} catch {
|
||||
self.session.commitConfiguration()
|
||||
@@ -146,6 +176,11 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
||||
if self.session.canAddOutput(output) {
|
||||
self.session.addOutput(output)
|
||||
}
|
||||
self.output = output
|
||||
|
||||
os_unfair_lock_lock(&self.lock)
|
||||
self._activeDeviceUniqueID = camera.uniqueID
|
||||
os_unfair_lock_unlock(&self.lock)
|
||||
|
||||
self.session.commitConfiguration()
|
||||
self.session.startRunning()
|
||||
@@ -154,6 +189,93 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
|
||||
}
|
||||
}
|
||||
|
||||
/// Switch the running session over to a different camera device. Safe to call
|
||||
/// from the main thread — actual session reconfiguration happens on sessionQueue.
|
||||
func switchToDevice(_ device: AVCaptureDevice) {
|
||||
sessionQueue.async { [weak self] in
|
||||
guard let self else { return }
|
||||
|
||||
if self.session.isRunning {
|
||||
self.session.stopRunning()
|
||||
}
|
||||
|
||||
self.session.beginConfiguration()
|
||||
|
||||
if let existing = self.currentInput {
|
||||
self.session.removeInput(existing)
|
||||
self.currentInput = nil
|
||||
}
|
||||
|
||||
do {
|
||||
let input = try AVCaptureDeviceInput(device: device)
|
||||
if self.session.canAddInput(input) {
|
||||
self.session.addInput(input)
|
||||
self.currentInput = input
|
||||
} else {
|
||||
self.session.commitConfiguration()
|
||||
LogStream.shared.log("cannot add input for \(device.localizedName)", level: .error, source: .camera)
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
self.session.commitConfiguration()
|
||||
LogStream.shared.log("failed to create input for \(device.localizedName): \(error)",
|
||||
level: .error, source: .camera)
|
||||
return
|
||||
}
|
||||
|
||||
// If the output got dropped (e.g., never created yet), re-add it.
|
||||
if self.output == nil {
|
||||
let output = AVCaptureVideoDataOutput()
|
||||
output.videoSettings = [
|
||||
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
||||
kCVPixelBufferMetalCompatibilityKey as String: true
|
||||
]
|
||||
output.alwaysDiscardsLateVideoFrames = true
|
||||
output.setSampleBufferDelegate(self, queue: self.sampleQueue)
|
||||
if self.session.canAddOutput(output) {
|
||||
self.session.addOutput(output)
|
||||
}
|
||||
self.output = output
|
||||
}
|
||||
|
||||
os_unfair_lock_lock(&self.lock)
|
||||
self._activeDeviceUniqueID = device.uniqueID
|
||||
self._deviceDisconnected = false
|
||||
// Best-effort default; real dimensions arrive with the first sample buffer.
|
||||
self._width = 1920
|
||||
self._height = 1080
|
||||
os_unfair_lock_unlock(&self.lock)
|
||||
|
||||
self.session.commitConfiguration()
|
||||
self.session.startRunning()
|
||||
|
||||
LogStream.shared.log("switched to \(device.localizedName)", level: .info, source: .camera)
|
||||
}
|
||||
}
|
||||
|
||||
@objc private func handleDeviceDisconnected(_ note: Notification) {
|
||||
guard let disconnected = note.object as? AVCaptureDevice else { return }
|
||||
|
||||
os_unfair_lock_lock(&lock)
|
||||
let activeID = _activeDeviceUniqueID
|
||||
os_unfair_lock_unlock(&lock)
|
||||
|
||||
guard disconnected.uniqueID == activeID else { return }
|
||||
|
||||
let name = disconnected.localizedName
|
||||
sessionQueue.async { [weak self] in
|
||||
guard let self else { return }
|
||||
if self.session.isRunning {
|
||||
self.session.stopRunning()
|
||||
}
|
||||
os_unfair_lock_lock(&self.lock)
|
||||
self._deviceDisconnected = true
|
||||
self.latest = nil
|
||||
os_unfair_lock_unlock(&self.lock)
|
||||
LogStream.shared.log("camera \(name) disconnected", level: .warn, source: .camera)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - AVCaptureVideoDataOutputSampleBufferDelegate
|
||||
|
||||
func captureOutput(
|
||||
|
||||
@@ -43,6 +43,9 @@ final class ObjectDetector: @unchecked Sendable {
|
||||
|
||||
private var lock = os_unfair_lock_s()
|
||||
private var _currentDetections: [Detection] = []
|
||||
private var _currentTextDetections: [TextDetection] = []
|
||||
private var _currentPoses: [PoseDetection] = []
|
||||
private var _lastDetectMode: DetectMode = .yolo
|
||||
private var _lastInferenceMs: Double = 0
|
||||
private var _inFlight = false
|
||||
|
||||
@@ -63,6 +66,53 @@ final class ObjectDetector: @unchecked Sendable {
|
||||
} else {
|
||||
LogStream.shared.log("loaded \(Theme.Performance.detectorModelName), tracker enabled (~5 Hz YOLO + ~60 Hz tracker)", level: .info, source: .detect)
|
||||
}
|
||||
|
||||
// Warm up the YOLO model on the detector queue so the first user-visible
|
||||
// detection doesn't pay ~200ms of CoreML/ANE compile latency. Dispatched
|
||||
// async so init() returns immediately and app launch isn't blocked.
|
||||
queue.async { [weak self] in
|
||||
self?.warmUpYolo()
|
||||
}
|
||||
}
|
||||
|
||||
private func warmUpYolo() {
|
||||
guard let yolo = yoloRequest else { return }
|
||||
let start = CFAbsoluteTimeGetCurrent()
|
||||
|
||||
// 416x416 mid-gray bitmap — YOLO's typical input size. The model will
|
||||
// rescale internally if needed; what matters is the CoreML graph gets
|
||||
// compiled and the ANE picks up the weights.
|
||||
let width = 416
|
||||
let height = 416
|
||||
let bytesPerRow = width * 4
|
||||
let colorSpace = CGColorSpaceCreateDeviceRGB()
|
||||
guard let ctx = CGContext(
|
||||
data: nil,
|
||||
width: width,
|
||||
height: height,
|
||||
bitsPerComponent: 8,
|
||||
bytesPerRow: bytesPerRow,
|
||||
space: colorSpace,
|
||||
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
|
||||
) else {
|
||||
LogStream.shared.log("yolo warmup skipped: bitmap context alloc failed", level: .warn, source: .detect)
|
||||
return
|
||||
}
|
||||
ctx.setFillColor(CGColor(red: 0.5, green: 0.5, blue: 0.5, alpha: 1.0))
|
||||
ctx.fill(CGRect(x: 0, y: 0, width: width, height: height))
|
||||
guard let cgImage = ctx.makeImage() else {
|
||||
LogStream.shared.log("yolo warmup skipped: cgImage creation failed", level: .warn, source: .detect)
|
||||
return
|
||||
}
|
||||
|
||||
let handler = VNImageRequestHandler(cgImage: cgImage, orientation: .up, options: [:])
|
||||
do {
|
||||
try handler.perform([yolo])
|
||||
let ms = (CFAbsoluteTimeGetCurrent() - start) * 1000.0
|
||||
LogStream.shared.log("warmed up YOLO in \(String(format: "%.0f", ms))ms", level: .debug, source: .detect)
|
||||
} catch {
|
||||
LogStream.shared.log("yolo warmup failed: \(error)", level: .warn, source: .detect)
|
||||
}
|
||||
}
|
||||
|
||||
private static func makeYoloRequest(modelName: String) -> (request: VNCoreMLRequest?, failureReason: String?) {
|
||||
@@ -96,6 +146,24 @@ final class ObjectDetector: @unchecked Sendable {
|
||||
return _currentDetections
|
||||
}
|
||||
|
||||
func currentTextDetections() -> [TextDetection] {
|
||||
os_unfair_lock_lock(&lock)
|
||||
defer { os_unfair_lock_unlock(&lock) }
|
||||
return _currentTextDetections
|
||||
}
|
||||
|
||||
func currentPoses() -> [PoseDetection] {
|
||||
os_unfair_lock_lock(&lock)
|
||||
defer { os_unfair_lock_unlock(&lock) }
|
||||
return _currentPoses
|
||||
}
|
||||
|
||||
var lastDetectMode: DetectMode {
|
||||
os_unfair_lock_lock(&lock)
|
||||
defer { os_unfair_lock_unlock(&lock) }
|
||||
return _lastDetectMode
|
||||
}
|
||||
|
||||
func submit(pixelBuffer: CVPixelBuffer) {
|
||||
os_unfair_lock_lock(&lock)
|
||||
if _inFlight {
|
||||
@@ -118,11 +186,19 @@ final class ObjectDetector: @unchecked Sendable {
|
||||
let yoloInterval = 1.0 / max(0.5, Theme.Performance.yoloDetectionHz)
|
||||
let shouldDetect = (start - lastYoloAt) >= yoloInterval
|
||||
|
||||
var newTextDetections: [TextDetection] = []
|
||||
var newPoses: [PoseDetection] = []
|
||||
let detectMode: DetectMode
|
||||
|
||||
if shouldDetect {
|
||||
detectAndBootstrap(pixelBuffer: pixelBuffer, now: start)
|
||||
let extras = detectAndBootstrap(pixelBuffer: pixelBuffer, now: start)
|
||||
newTextDetections = extras.text
|
||||
newPoses = extras.poses
|
||||
lastYoloAt = start
|
||||
detectMode = .yolo
|
||||
} else {
|
||||
trackOnly(pixelBuffer: pixelBuffer, now: start)
|
||||
detectMode = .track
|
||||
}
|
||||
|
||||
// Drop tracks that have aged out (no YOLO refresh recently).
|
||||
@@ -157,6 +233,14 @@ final class ObjectDetector: @unchecked Sendable {
|
||||
|
||||
os_unfair_lock_lock(&lock)
|
||||
_currentDetections = snapshot
|
||||
if shouldDetect {
|
||||
// Only refresh text & poses on YOLO-cadence frames; tracker frames
|
||||
// keep the most recent set so the renderer doesn't flicker between
|
||||
// populated and empty in the ~50ms between YOLO refreshes.
|
||||
_currentTextDetections = newTextDetections
|
||||
_currentPoses = newPoses
|
||||
}
|
||||
_lastDetectMode = detectMode
|
||||
_lastInferenceMs = elapsedMs
|
||||
_inFlight = false
|
||||
os_unfair_lock_unlock(&lock)
|
||||
@@ -164,10 +248,19 @@ final class ObjectDetector: @unchecked Sendable {
|
||||
|
||||
// MARK: - YOLO + supplementary detectors
|
||||
|
||||
private func detectAndBootstrap(pixelBuffer: CVPixelBuffer, now: CFAbsoluteTime) {
|
||||
private func detectAndBootstrap(pixelBuffer: CVPixelBuffer, now: CFAbsoluteTime) -> (text: [TextDetection], poses: [PoseDetection]) {
|
||||
let faces = VNDetectFaceRectanglesRequest()
|
||||
let animals = VNRecognizeAnimalsRequest()
|
||||
var requests: [VNRequest] = [faces, animals]
|
||||
|
||||
let text = VNRecognizeTextRequest()
|
||||
text.recognitionLevel = .accurate
|
||||
text.usesLanguageCorrection = true
|
||||
|
||||
let bodyPose = VNDetectHumanBodyPoseRequest()
|
||||
let handPose = VNDetectHumanHandPoseRequest()
|
||||
handPose.maximumHandCount = 2
|
||||
|
||||
var requests: [VNRequest] = [faces, animals, text, bodyPose, handPose]
|
||||
if let yolo = yoloRequest {
|
||||
requests.append(yolo)
|
||||
}
|
||||
@@ -248,6 +341,156 @@ final class ObjectDetector: @unchecked Sendable {
|
||||
level: .info, source: .track)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Text recognition ----
|
||||
let textDetections: [TextDetection] = (text.results ?? []).compactMap { obs in
|
||||
guard let top = obs.topCandidates(1).first else { return nil }
|
||||
return TextDetection(rect: obs.boundingBox, text: top.string, confidence: top.confidence)
|
||||
}
|
||||
|
||||
// ---- Body pose ----
|
||||
var bodyPoses: [PoseDetection] = []
|
||||
if let bodyResults = bodyPose.results {
|
||||
for obs in bodyResults {
|
||||
if let pose = makeBodyPose(from: obs) {
|
||||
bodyPoses.append(pose)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Hand pose ----
|
||||
var handPoses: [PoseDetection] = []
|
||||
if let handResults = handPose.results {
|
||||
for obs in handResults {
|
||||
if let pose = makeHandPose(from: obs) {
|
||||
handPoses.append(pose)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return (text: textDetections, poses: bodyPoses + handPoses)
|
||||
}
|
||||
|
||||
// MARK: - Pose builders
|
||||
|
||||
private static let poseConfidenceThreshold: Float = 0.3
|
||||
|
||||
private func makeBodyPose(from obs: VNHumanBodyPoseObservation) -> PoseDetection? {
|
||||
// Apple's SDK joint names. We map each to its CGPoint via
|
||||
// recognizedPoint(_:); points with confidence > threshold are kept.
|
||||
typealias J = VNHumanBodyPoseObservation.JointName
|
||||
let connections: [(J, J)] = [
|
||||
(.nose, .neck),
|
||||
(.neck, .leftShoulder),
|
||||
(.leftShoulder, .leftElbow),
|
||||
(.leftElbow, .leftWrist),
|
||||
(.neck, .rightShoulder),
|
||||
(.rightShoulder, .rightElbow),
|
||||
(.rightElbow, .rightWrist),
|
||||
(.neck, .root),
|
||||
(.root, .leftHip),
|
||||
(.leftHip, .leftKnee),
|
||||
(.leftKnee, .leftAnkle),
|
||||
(.root, .rightHip),
|
||||
(.rightHip, .rightKnee),
|
||||
(.rightKnee, .rightAnkle)
|
||||
]
|
||||
|
||||
var segments: [PoseDetection.Segment] = []
|
||||
// Use a small key->point map keyed by joint rawValue so points stay
|
||||
// unique even if a joint appears in multiple connections.
|
||||
var uniquePoints: [String: CGPoint] = [:]
|
||||
var maxConf: Float = 0
|
||||
|
||||
for (a, b) in connections {
|
||||
guard
|
||||
let p1 = try? obs.recognizedPoint(a),
|
||||
let p2 = try? obs.recognizedPoint(b)
|
||||
else { continue }
|
||||
if p1.confidence > ObjectDetector.poseConfidenceThreshold {
|
||||
uniquePoints[a.rawValue.rawValue] = p1.location
|
||||
maxConf = max(maxConf, p1.confidence)
|
||||
}
|
||||
if p2.confidence > ObjectDetector.poseConfidenceThreshold {
|
||||
uniquePoints[b.rawValue.rawValue] = p2.location
|
||||
maxConf = max(maxConf, p2.confidence)
|
||||
}
|
||||
if p1.confidence > ObjectDetector.poseConfidenceThreshold,
|
||||
p2.confidence > ObjectDetector.poseConfidenceThreshold {
|
||||
segments.append(PoseDetection.Segment(start: p1.location, end: p2.location))
|
||||
}
|
||||
}
|
||||
|
||||
if segments.isEmpty && uniquePoints.isEmpty { return nil }
|
||||
return PoseDetection(
|
||||
segments: segments,
|
||||
points: Array(uniquePoints.values),
|
||||
kind: .body,
|
||||
confidence: maxConf
|
||||
)
|
||||
}
|
||||
|
||||
private func makeHandPose(from obs: VNHumanHandPoseObservation) -> PoseDetection? {
|
||||
typealias J = VNHumanHandPoseObservation.JointName
|
||||
|
||||
// Each finger chain: tip -> middle phalange -> proximal -> metacarpal/base.
|
||||
// Apple's hand-pose joint names (verified against the macOS SDK):
|
||||
// thumb: thumbTip, thumbIP, thumbMP, thumbCMC
|
||||
// index: indexTip, indexDIP, indexPIP, indexMCP
|
||||
// middle: middleTip, middleDIP,middlePIP,middleMCP
|
||||
// ring: ringTip, ringDIP, ringPIP, ringMCP
|
||||
// little: littleTip, littleDIP,littlePIP,littleMCP
|
||||
let fingerChains: [[J]] = [
|
||||
[.thumbTip, .thumbIP, .thumbMP, .thumbCMC],
|
||||
[.indexTip, .indexDIP, .indexPIP, .indexMCP],
|
||||
[.middleTip, .middleDIP, .middlePIP, .middleMCP],
|
||||
[.ringTip, .ringDIP, .ringPIP, .ringMCP],
|
||||
[.littleTip, .littleDIP, .littlePIP, .littleMCP]
|
||||
]
|
||||
// Palm ring: connect the 5 base/knuckle joints around the palm.
|
||||
// Start at thumb base (CMC) and walk index→middle→ring→little MCPs,
|
||||
// then close the ring back to the thumb base.
|
||||
let palmRing: [J] = [.thumbCMC, .indexMCP, .middleMCP, .ringMCP, .littleMCP, .thumbCMC]
|
||||
|
||||
var segments: [PoseDetection.Segment] = []
|
||||
var uniquePoints: [String: CGPoint] = [:]
|
||||
var maxConf: Float = 0
|
||||
|
||||
func addPoint(_ name: J) -> (location: CGPoint, ok: Bool) {
|
||||
guard let p = try? obs.recognizedPoint(name) else { return (.zero, false) }
|
||||
if p.confidence > ObjectDetector.poseConfidenceThreshold {
|
||||
uniquePoints[name.rawValue.rawValue] = p.location
|
||||
maxConf = max(maxConf, p.confidence)
|
||||
return (p.location, true)
|
||||
}
|
||||
return (.zero, false)
|
||||
}
|
||||
|
||||
for chain in fingerChains {
|
||||
for i in 0..<(chain.count - 1) {
|
||||
let a = addPoint(chain[i])
|
||||
let b = addPoint(chain[i + 1])
|
||||
if a.ok && b.ok {
|
||||
segments.append(PoseDetection.Segment(start: a.location, end: b.location))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i in 0..<(palmRing.count - 1) {
|
||||
let a = addPoint(palmRing[i])
|
||||
let b = addPoint(palmRing[i + 1])
|
||||
if a.ok && b.ok {
|
||||
segments.append(PoseDetection.Segment(start: a.location, end: b.location))
|
||||
}
|
||||
}
|
||||
|
||||
if segments.isEmpty && uniquePoints.isEmpty { return nil }
|
||||
return PoseDetection(
|
||||
segments: segments,
|
||||
points: Array(uniquePoints.values),
|
||||
kind: .hand,
|
||||
confidence: maxConf
|
||||
)
|
||||
}
|
||||
|
||||
// MARK: - Tracker-only update
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import Foundation
|
||||
import CoreGraphics
|
||||
|
||||
/// Shared frame-annotation types produced by `ObjectDetector` and consumed by
|
||||
/// `Renderer`. All coordinates use Vision's normalized [0..1] image space
|
||||
/// (origin bottom-left), matching the existing `Detection.rect`. The renderer
|
||||
/// converts to viewport NDC the same way it does for detections, so adding a
|
||||
/// new overlay type is just an array of rects-or-points and a draw call.
|
||||
|
||||
/// Recognized text region — produced by `VNRecognizeTextRequest`.
|
||||
struct TextDetection: Sendable {
|
||||
var rect: CGRect // normalized, origin bottom-left
|
||||
var text: String // the recognized string
|
||||
var confidence: Float
|
||||
}
|
||||
|
||||
enum PoseKind: Sendable {
|
||||
case body
|
||||
case hand
|
||||
}
|
||||
|
||||
/// A pose detection — a set of normalized line segments (joint pairs) plus
|
||||
/// the keypoint positions themselves. The renderer draws lines between the
|
||||
/// segment endpoints and small dots at each unique point.
|
||||
struct PoseDetection: Sendable {
|
||||
/// Normalized line segments (each segment goes from start to end in
|
||||
/// Vision's [0..1] image coords, origin bottom-left).
|
||||
var segments: [Segment]
|
||||
/// All unique joint points present in this pose, for dot rendering.
|
||||
var points: [CGPoint]
|
||||
/// Whether this pose is a body skeleton or a hand skeleton.
|
||||
var kind: PoseKind
|
||||
/// Overall confidence of the pose (max of constituent joint confidences).
|
||||
var confidence: Float
|
||||
|
||||
struct Segment: Sendable {
|
||||
var start: CGPoint
|
||||
var end: CGPoint
|
||||
}
|
||||
}
|
||||
|
||||
/// Tells the renderer which detector mode produced the most recent
|
||||
/// inference, so the footer can label `DET 28.4ms (yolo)` vs `DET 2.1ms (track)`.
|
||||
enum DetectMode: String, Sendable {
|
||||
case yolo = "yolo"
|
||||
case track = "track"
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import Foundation
|
||||
import simd
|
||||
|
||||
/// Deterministic per-trackId color generator. Hashes a UUID into a hue value
|
||||
/// in [0, 1) and converts HSV(hue, 0.8, 1.0) to RGB. Used to give each tracked
|
||||
/// object a stable, visually-distinct box + label color so it's clear which
|
||||
/// label belongs to which box when several boxes overlap.
|
||||
///
|
||||
/// Uniqueness caveat: with hue alone covering ~32 buckets of human-discernible
|
||||
/// difference, ~6 simultaneous tracks of the same hue family will start to look
|
||||
/// similar. For the POC's typical 3-8 tracks this is fine; if track count
|
||||
/// regularly exceeds ~12 a second axis (saturation or value) would help.
|
||||
enum ColorHash {
|
||||
/// Returns a stable RGBA color for the given track id.
|
||||
/// Saturation 0.8, value 1.0, alpha 1.0.
|
||||
static func colorFor(trackId: UUID) -> SIMD4<Float> {
|
||||
// Spread the UUID's bytes across the hash so adjacent UUIDs (which the
|
||||
// standard library doesn't actually generate adjacently, but be safe)
|
||||
// map to far-apart hues. Mix several bytes from the uuid tuple.
|
||||
let bytes = trackId.uuid
|
||||
// Combine bytes 0, 3, 7, 11 with a multiplicative hash. The constant
|
||||
// 2654435769 is Knuth's golden-ratio hash multiplier — distributes
|
||||
// small input differences across the full 32-bit output range.
|
||||
var h: UInt32 = UInt32(bytes.0)
|
||||
h = h &* 2654435769 &+ UInt32(bytes.3)
|
||||
h = h &* 2654435769 &+ UInt32(bytes.7)
|
||||
h = h &* 2654435769 &+ UInt32(bytes.11)
|
||||
let hue = Float(h % 360) / 360.0
|
||||
return hsvToRGB(h: hue, s: 0.8, v: 1.0)
|
||||
}
|
||||
|
||||
/// HSV → RGB, standard formula. Hue is in [0,1), s and v in [0,1].
|
||||
private static func hsvToRGB(h: Float, s: Float, v: Float) -> SIMD4<Float> {
|
||||
let hh = (h - floor(h)) * 6.0
|
||||
let i = Int(floor(hh))
|
||||
let f = hh - Float(i)
|
||||
let p = v * (1 - s)
|
||||
let q = v * (1 - s * f)
|
||||
let t = v * (1 - s * (1 - f))
|
||||
let r: Float
|
||||
let g: Float
|
||||
let b: Float
|
||||
switch i % 6 {
|
||||
case 0: r = v; g = t; b = p
|
||||
case 1: r = q; g = v; b = p
|
||||
case 2: r = p; g = v; b = t
|
||||
case 3: r = p; g = q; b = v
|
||||
case 4: r = t; g = p; b = v
|
||||
default: r = v; g = p; b = q
|
||||
}
|
||||
return SIMD4<Float>(r, g, b, 1.0)
|
||||
}
|
||||
}
|
||||
+348
-11
@@ -1,6 +1,9 @@
|
||||
import MetalKit
|
||||
import AppKit
|
||||
import simd
|
||||
import os
|
||||
import ImageIO
|
||||
import UniformTypeIdentifiers
|
||||
|
||||
/// Camera textures coming from AVFoundation are stored origin-top-left. The MSL
|
||||
/// `quad_vertex` shader emits UVs with `v = 1.0 - v` so image samples appear right-side-up
|
||||
@@ -36,9 +39,23 @@ final class Renderer: NSObject, MTKViewDelegate {
|
||||
|
||||
private let textRasterizer: TextRasterizer
|
||||
private var labelTextureCache: [String: MTLTexture] = [:]
|
||||
/// Parallel array tracking access order for `labelTextureCache`. Head is
|
||||
/// LRU, tail is MRU. On cache hit, move-to-end. On cache miss with the
|
||||
/// cache at capacity, drop head before inserting.
|
||||
private var labelTextureCacheKeys: [String] = []
|
||||
private let labelTextureCacheCapacity = 128
|
||||
private let nearestSampler: MTLSamplerState
|
||||
private let linearSampler: MTLSamplerState
|
||||
|
||||
/// Set by `snapshotPNG()` on a non-render thread. The next `draw(in:)`
|
||||
/// pass renders normally, then copies the drawable's texture into a
|
||||
/// CPU-readable MTLTexture, encodes it as PNG, fires this completion,
|
||||
/// and clears the flag.
|
||||
private var pendingSnapshotCompletion: ((Data?) -> Void)?
|
||||
/// Guards `pendingSnapshotCompletion` so the snapshot caller (background)
|
||||
/// and the render loop (MainActor) don't race on read/write.
|
||||
private var snapshotLock = os_unfair_lock_s()
|
||||
|
||||
private let startTime: CFAbsoluteTime = CFAbsoluteTimeGetCurrent()
|
||||
private var lastFrameTime: CFAbsoluteTime
|
||||
private var frameCounter: Int = 0
|
||||
@@ -179,6 +196,10 @@ final class Renderer: NSObject, MTKViewDelegate {
|
||||
drawJarvis(encoder: encoder, texture: source, time: elapsed, paneSize: inner.size)
|
||||
drawDetections(encoder: encoder, detections: detections)
|
||||
drawDetectionLabels(encoder: encoder, detections: detections, paneRect: inner)
|
||||
let texts = detector.currentTextDetections()
|
||||
drawTextDetections(encoder: encoder, texts: texts, paneRect: inner)
|
||||
let poses = detector.currentPoses()
|
||||
drawPoses(encoder: encoder, poses: poses, paneRect: inner)
|
||||
case 2:
|
||||
drawImage(encoder: encoder, pipeline: pipelines.edges, texture: edgesTexture, sampler: nearestSampler)
|
||||
case 3:
|
||||
@@ -193,8 +214,148 @@ final class Renderer: NSObject, MTKViewDelegate {
|
||||
drawFooter(encoder: encoder, drawableSize: drawableSize)
|
||||
|
||||
encoder.endEncoding()
|
||||
commandBuffer.present(drawable)
|
||||
commandBuffer.commit()
|
||||
|
||||
// Snapshot path — piggyback on this frame's command buffer. We blit
|
||||
// the drawable's texture into a CPU-readable shared-storage texture,
|
||||
// then synchronously wait for the GPU to finish before reading back
|
||||
// and encoding as PNG. ~10-30ms typical (dominated by waitUntilCompleted).
|
||||
let snapshotCompletion = takePendingSnapshotCompletion()
|
||||
if let completion = snapshotCompletion {
|
||||
let readback = makeReadbackTexture(matching: drawable.texture)
|
||||
if let readback,
|
||||
let blit = commandBuffer.makeBlitCommandEncoder() {
|
||||
blit.copy(from: drawable.texture, to: readback)
|
||||
blit.endEncoding()
|
||||
commandBuffer.present(drawable)
|
||||
commandBuffer.commit()
|
||||
commandBuffer.waitUntilCompleted()
|
||||
let data = encodePNG(from: readback)
|
||||
completion(data)
|
||||
} else {
|
||||
// Couldn't allocate readback; fail gracefully.
|
||||
commandBuffer.present(drawable)
|
||||
commandBuffer.commit()
|
||||
completion(nil)
|
||||
}
|
||||
} else {
|
||||
commandBuffer.present(drawable)
|
||||
commandBuffer.commit()
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Snapshot
|
||||
|
||||
/// Public synchronous API. Sets the pending-completion flag, waits via a
|
||||
/// semaphore for the next render loop to fulfill it, returns the PNG
|
||||
/// data (or nil on failure). Safe to call off the MainActor; the wait is
|
||||
/// at most ~30ms per shot.
|
||||
func snapshotPNG() -> Data? {
|
||||
let semaphore = DispatchSemaphore(value: 0)
|
||||
// Capture into a class-level Sendable box so the completion can write
|
||||
// it from whatever thread fires the completion.
|
||||
let box = SnapshotBox()
|
||||
let completion: (Data?) -> Void = { data in
|
||||
box.data = data
|
||||
semaphore.signal()
|
||||
}
|
||||
os_unfair_lock_lock(&snapshotLock)
|
||||
// If a previous snapshot is still in flight, fail this one rather
|
||||
// than stomp on it. The caller can retry.
|
||||
if pendingSnapshotCompletion != nil {
|
||||
os_unfair_lock_unlock(&snapshotLock)
|
||||
return nil
|
||||
}
|
||||
pendingSnapshotCompletion = completion
|
||||
os_unfair_lock_unlock(&snapshotLock)
|
||||
|
||||
// 1 second is generous; if the render loop is stalled for longer than
|
||||
// that, returning nil and letting the user retry is better than
|
||||
// hanging the menu thread forever.
|
||||
let result = semaphore.wait(timeout: .now() + .seconds(1))
|
||||
if result == .timedOut {
|
||||
os_unfair_lock_lock(&snapshotLock)
|
||||
pendingSnapshotCompletion = nil
|
||||
os_unfair_lock_unlock(&snapshotLock)
|
||||
return nil
|
||||
}
|
||||
return box.data
|
||||
}
|
||||
|
||||
private final class SnapshotBox: @unchecked Sendable {
|
||||
var data: Data?
|
||||
}
|
||||
|
||||
private func takePendingSnapshotCompletion() -> ((Data?) -> Void)? {
|
||||
os_unfair_lock_lock(&snapshotLock)
|
||||
let c = pendingSnapshotCompletion
|
||||
pendingSnapshotCompletion = nil
|
||||
os_unfair_lock_unlock(&snapshotLock)
|
||||
return c
|
||||
}
|
||||
|
||||
/// Allocates a shared-storage texture matching the given drawable texture's
|
||||
/// dimensions and pixel format, so its bytes are CPU-addressable after the
|
||||
/// blit completes. The drawable's own texture has private storage and
|
||||
/// cannot be read back directly.
|
||||
private func makeReadbackTexture(matching source: MTLTexture) -> MTLTexture? {
|
||||
let descriptor = MTLTextureDescriptor.texture2DDescriptor(
|
||||
pixelFormat: source.pixelFormat,
|
||||
width: source.width,
|
||||
height: source.height,
|
||||
mipmapped: false
|
||||
)
|
||||
descriptor.usage = [.shaderRead]
|
||||
descriptor.storageMode = .shared
|
||||
return device.makeTexture(descriptor: descriptor)
|
||||
}
|
||||
|
||||
private func encodePNG(from texture: MTLTexture) -> Data? {
|
||||
let width = texture.width
|
||||
let height = texture.height
|
||||
let bytesPerRow = width * 4
|
||||
var bytes = [UInt8](repeating: 0, count: bytesPerRow * height)
|
||||
let region = MTLRegionMake2D(0, 0, width, height)
|
||||
bytes.withUnsafeMutableBytes { ptr in
|
||||
if let base = ptr.baseAddress {
|
||||
texture.getBytes(base, bytesPerRow: bytesPerRow, from: region, mipmapLevel: 0)
|
||||
}
|
||||
}
|
||||
|
||||
// Drawable is bgra8Unorm. CGBitmapContext wants the byte order spelled
|
||||
// out via CGBitmapInfo: byteOrder32Little + premultipliedFirst means
|
||||
// memory layout BGRA, which matches the texture exactly — no swizzle.
|
||||
let colorSpace = CGColorSpace(name: CGColorSpace.sRGB) ?? CGColorSpaceCreateDeviceRGB()
|
||||
let bitmapInfo: CGBitmapInfo = [
|
||||
CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedFirst.rawValue),
|
||||
.byteOrder32Little
|
||||
]
|
||||
guard let context = bytes.withUnsafeMutableBytes({ ptr -> CGContext? in
|
||||
CGContext(
|
||||
data: ptr.baseAddress,
|
||||
width: width,
|
||||
height: height,
|
||||
bitsPerComponent: 8,
|
||||
bytesPerRow: bytesPerRow,
|
||||
space: colorSpace,
|
||||
bitmapInfo: bitmapInfo.rawValue
|
||||
)
|
||||
}) else {
|
||||
return nil
|
||||
}
|
||||
guard let cgImage = context.makeImage() else { return nil }
|
||||
|
||||
let nsData = NSMutableData()
|
||||
guard let destination = CGImageDestinationCreateWithData(
|
||||
nsData as CFMutableData,
|
||||
UTType.png.identifier as CFString,
|
||||
1,
|
||||
nil
|
||||
) else {
|
||||
return nil
|
||||
}
|
||||
CGImageDestinationAddImage(destination, cgImage, nil)
|
||||
guard CGImageDestinationFinalize(destination) else { return nil }
|
||||
return nsData as Data
|
||||
}
|
||||
|
||||
// MARK: - No-source path
|
||||
@@ -316,7 +477,7 @@ final class Renderer: NSObject, MTKViewDelegate {
|
||||
let sizeNDC = SIMD2<Float>(Float(r.width) * 2, Float(r.height) * 2)
|
||||
var uniforms = BoxUniforms(
|
||||
rect: SIMD4<Float>(originNDC.x, originNDC.y, sizeNDC.x, sizeNDC.y),
|
||||
color: Theme.Palette.boxStroke,
|
||||
color: boxColor(for: detection),
|
||||
params: SIMD4<Float>(0, 0, 0, 0) // mode 0 = hollow stroke
|
||||
)
|
||||
encoder.setVertexBytes(&uniforms, length: MemoryLayout<BoxUniforms>.stride, index: 0)
|
||||
@@ -325,6 +486,28 @@ final class Renderer: NSObject, MTKViewDelegate {
|
||||
}
|
||||
}
|
||||
|
||||
/// Cyan for face/identity labels, per-track hue for everything else.
|
||||
/// Face labels are recognizable enough on their own that a stable cyan
|
||||
/// makes the UI feel less chaotic than randomized hues.
|
||||
private func boxColor(for detection: Detection) -> SIMD4<Float> {
|
||||
if isFaceLabel(detection.label) {
|
||||
return Theme.Palette.boxStroke
|
||||
}
|
||||
return ColorHash.colorFor(trackId: detection.trackId)
|
||||
}
|
||||
|
||||
private func isFaceLabel(_ label: String) -> Bool {
|
||||
if label == "FACE" { return true }
|
||||
// FaceRegistry names are uppercased before being assigned as a label
|
||||
// (see ObjectDetector.detectAndBootstrap). Anything in the registry
|
||||
// should keep the cyan stroke. Falling back to "all caps single word"
|
||||
// would accidentally include YOLO labels like CUP; check the registry
|
||||
// instead. The enrolledNames() call is cheap (dictionary keys snapshot
|
||||
// behind a small lock); we run it at most once per detection per frame.
|
||||
let names = detector.faceRegistry.enrolledNames()
|
||||
return names.contains(where: { $0.uppercased() == label })
|
||||
}
|
||||
|
||||
private func drawCornerBrackets(encoder: MTLRenderCommandEncoder, drawableSize: CGSize, innerRects: [CGRect]) {
|
||||
let bracketLen = Theme.HUD.cornerBracketLength * backingScale
|
||||
let thickness = Theme.HUD.cornerBracketThickness * backingScale
|
||||
@@ -405,10 +588,12 @@ final class Renderer: NSObject, MTKViewDelegate {
|
||||
case .balanced: profile = "BALANCED"
|
||||
case .quality: profile = "QUALITY"
|
||||
}
|
||||
let mode = detector.lastDetectMode.rawValue
|
||||
let text = String(
|
||||
format: "VISIONPOC · FPS %d · DET %.1fms · %@",
|
||||
format: "VISIONPOC · FPS %d · DET %.1fms (%@) · %@",
|
||||
Int(smoothedFPS.rounded()),
|
||||
detector.lastInferenceMs,
|
||||
mode,
|
||||
profile
|
||||
)
|
||||
footerTexture = makeLabelTexture(text, font: Theme.Font.body(Theme.Font.footerSize), wide: true)
|
||||
@@ -434,10 +619,28 @@ final class Renderer: NSObject, MTKViewDelegate {
|
||||
|
||||
// MARK: - Detection labels & ASCII art
|
||||
|
||||
/// LRU-bounded label texture cache. Capacity capped at
|
||||
/// `labelTextureCacheCapacity`. On hit, the key is moved to the tail
|
||||
/// (most-recently-used). On miss with the cache full, the head (LRU) is
|
||||
/// evicted before insertion.
|
||||
private func labelTexture(for text: String) -> MTLTexture? {
|
||||
if let cached = labelTextureCache[text] { return cached }
|
||||
let tex = makeLabelTexture(text, font: Theme.Font.body(Theme.Font.boxLabelSize))
|
||||
if let tex { labelTextureCache[text] = tex }
|
||||
if let cached = labelTextureCache[text] {
|
||||
// Move-to-end on hit. removeAll(where:) is O(n) for a 128-element
|
||||
// array; trivially fast and avoids pulling in a real LRU library.
|
||||
labelTextureCacheKeys.removeAll(where: { $0 == text })
|
||||
labelTextureCacheKeys.append(text)
|
||||
return cached
|
||||
}
|
||||
guard let tex = makeLabelTexture(text, font: Theme.Font.body(Theme.Font.boxLabelSize)) else {
|
||||
return nil
|
||||
}
|
||||
if labelTextureCacheKeys.count >= labelTextureCacheCapacity,
|
||||
let oldest = labelTextureCacheKeys.first {
|
||||
labelTextureCacheKeys.removeFirst()
|
||||
labelTextureCache.removeValue(forKey: oldest)
|
||||
}
|
||||
labelTextureCache[text] = tex
|
||||
labelTextureCacheKeys.append(text)
|
||||
return tex
|
||||
}
|
||||
|
||||
@@ -449,7 +652,12 @@ final class Renderer: NSObject, MTKViewDelegate {
|
||||
let labelHeight: CGFloat = 18 * backingScale
|
||||
let inset: CGFloat = 4 * backingScale
|
||||
for det in detections {
|
||||
guard let texture = labelTexture(for: det.label) else { continue }
|
||||
// Confidence quantized to 10% buckets to keep label cache small.
|
||||
// A CUP at 80% and a CUP at 90% get separate cached textures,
|
||||
// but two CUPs at 85% share one (both round to 90%).
|
||||
let bucket = Int((det.confidence * 10).rounded()) * 10
|
||||
let displayed = "\(det.label) \(bucket)%"
|
||||
guard let texture = labelTexture(for: displayed) else { continue }
|
||||
let r = det.rect
|
||||
let boxLeftPx = paneRect.origin.x + CGFloat(r.origin.x) * paneRect.width
|
||||
// Box top edge in screen pixels (y-down). Vision y is origin
|
||||
@@ -458,14 +666,143 @@ final class Renderer: NSObject, MTKViewDelegate {
|
||||
let boxTopPx = paneRect.origin.y
|
||||
+ (1.0 - (CGFloat(r.origin.y) + CGFloat(r.height))) * paneRect.height
|
||||
let labelWidth = CGFloat(texture.width)
|
||||
// Clamp to pane edges so the label doesn't extend past the
|
||||
// drawable when a box is hugging the right or top edge.
|
||||
let rawX = boxLeftPx + inset
|
||||
let rawY = boxTopPx + inset
|
||||
let clampedX = min(rawX, paneRect.maxX - labelWidth - inset)
|
||||
let clampedY = max(rawY, paneRect.minY + inset)
|
||||
let rect = CGRect(
|
||||
x: boxLeftPx + inset,
|
||||
y: boxTopPx + inset,
|
||||
x: clampedX,
|
||||
y: clampedY,
|
||||
width: labelWidth,
|
||||
height: labelHeight
|
||||
)
|
||||
setViewport(encoder: encoder, rect: rect)
|
||||
drawTextQuad(encoder: encoder, texture: texture, tint: Theme.Palette.boxStroke)
|
||||
drawTextQuad(encoder: encoder, texture: texture, tint: boxColor(for: det))
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - OCR overlay
|
||||
|
||||
private func drawTextDetections(
|
||||
encoder: MTLRenderCommandEncoder,
|
||||
texts: [TextDetection],
|
||||
paneRect: CGRect
|
||||
) {
|
||||
guard !texts.isEmpty else { return }
|
||||
|
||||
// Hollow magenta box per recognized text region.
|
||||
encoder.setRenderPipelineState(pipelines.boxes)
|
||||
for t in texts {
|
||||
let r = t.rect
|
||||
let originNDC = SIMD2<Float>(Float(r.origin.x) * 2 - 1, Float(r.origin.y) * 2 - 1)
|
||||
let sizeNDC = SIMD2<Float>(Float(r.width) * 2, Float(r.height) * 2)
|
||||
var uniforms = BoxUniforms(
|
||||
rect: SIMD4<Float>(originNDC.x, originNDC.y, sizeNDC.x, sizeNDC.y),
|
||||
color: Theme.Palette.textStroke,
|
||||
params: SIMD4<Float>(0, 0, 0, 0)
|
||||
)
|
||||
encoder.setVertexBytes(&uniforms, length: MemoryLayout<BoxUniforms>.stride, index: 0)
|
||||
encoder.setFragmentBytes(&uniforms, length: MemoryLayout<BoxUniforms>.stride, index: 0)
|
||||
encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 6)
|
||||
}
|
||||
|
||||
// Magenta label INSIDE the top-left of each text rect, clamped to pane.
|
||||
let labelHeight: CGFloat = 18 * backingScale
|
||||
let inset: CGFloat = 4 * backingScale
|
||||
for t in texts {
|
||||
// OCR strings can be arbitrarily long; truncate to a sane prefix
|
||||
// before caching so cache key cardinality stays bounded. The LRU
|
||||
// cap also kicks in here.
|
||||
let key = t.text.count > 32
|
||||
? String(t.text.prefix(32))
|
||||
: t.text
|
||||
guard let texture = labelTexture(for: key) else { continue }
|
||||
let r = t.rect
|
||||
let boxLeftPx = paneRect.origin.x + CGFloat(r.origin.x) * paneRect.width
|
||||
let boxTopPx = paneRect.origin.y
|
||||
+ (1.0 - (CGFloat(r.origin.y) + CGFloat(r.height))) * paneRect.height
|
||||
let labelWidth = CGFloat(texture.width)
|
||||
let rawX = boxLeftPx + inset
|
||||
let rawY = boxTopPx + inset
|
||||
let clampedX = min(rawX, paneRect.maxX - labelWidth - inset)
|
||||
let clampedY = max(rawY, paneRect.minY + inset)
|
||||
let rect = CGRect(
|
||||
x: clampedX,
|
||||
y: clampedY,
|
||||
width: labelWidth,
|
||||
height: labelHeight
|
||||
)
|
||||
setViewport(encoder: encoder, rect: rect)
|
||||
drawTextQuad(encoder: encoder, texture: texture, tint: Theme.Palette.textStroke)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Pose overlay
|
||||
|
||||
private func drawPoses(
|
||||
encoder: MTLRenderCommandEncoder,
|
||||
poses: [PoseDetection],
|
||||
paneRect: CGRect
|
||||
) {
|
||||
guard !poses.isEmpty else { return }
|
||||
encoder.setRenderPipelineState(pipelines.boxes)
|
||||
|
||||
for pose in poses {
|
||||
let color: SIMD4<Float>
|
||||
switch pose.kind {
|
||||
case .body: color = Theme.Palette.bodyPose
|
||||
case .hand: color = Theme.Palette.handPose
|
||||
}
|
||||
|
||||
// Segments — Option A: draw each as the axis-aligned bounding rect
|
||||
// of its endpoints. Blocky for diagonals but a single draw call per
|
||||
// segment and matches the existing shader contract exactly. A v2
|
||||
// could subdivide into mini-rects for smoother diagonals.
|
||||
for seg in pose.segments {
|
||||
let minX = Float(min(seg.start.x, seg.end.x))
|
||||
let maxX = Float(max(seg.start.x, seg.end.x))
|
||||
let minY = Float(min(seg.start.y, seg.end.y))
|
||||
let maxY = Float(max(seg.start.y, seg.end.y))
|
||||
// Give zero-extent (perfectly horizontal/vertical) segments a
|
||||
// visible thickness — without this they'd render as a 0-pixel
|
||||
// strip and vanish.
|
||||
let thicknessNorm: Float = 0.003
|
||||
let w = max(maxX - minX, thicknessNorm)
|
||||
let h = max(maxY - minY, thicknessNorm)
|
||||
let originNDC = SIMD2<Float>(minX * 2 - 1, minY * 2 - 1)
|
||||
let sizeNDC = SIMD2<Float>(w * 2, h * 2)
|
||||
var uniforms = BoxUniforms(
|
||||
rect: SIMD4<Float>(originNDC.x, originNDC.y, sizeNDC.x, sizeNDC.y),
|
||||
color: color,
|
||||
params: SIMD4<Float>(1, 0, 0, 0) // solid fill
|
||||
)
|
||||
encoder.setVertexBytes(&uniforms, length: MemoryLayout<BoxUniforms>.stride, index: 0)
|
||||
encoder.setFragmentBytes(&uniforms, length: MemoryLayout<BoxUniforms>.stride, index: 0)
|
||||
encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 6)
|
||||
}
|
||||
|
||||
// Joint dots — 3 px squares. Convert 3 px to normalized image
|
||||
// coords using the pane size so the dots stay roughly the same
|
||||
// visual size regardless of pane.
|
||||
let dotPx: CGFloat = 3 * backingScale
|
||||
let dotNormX = Float(dotPx / max(paneRect.width, 1))
|
||||
let dotNormY = Float(dotPx / max(paneRect.height, 1))
|
||||
for p in pose.points {
|
||||
let px = Float(p.x) - dotNormX * 0.5
|
||||
let py = Float(p.y) - dotNormY * 0.5
|
||||
let originNDC = SIMD2<Float>(px * 2 - 1, py * 2 - 1)
|
||||
let sizeNDC = SIMD2<Float>(dotNormX * 2, dotNormY * 2)
|
||||
var uniforms = BoxUniforms(
|
||||
rect: SIMD4<Float>(originNDC.x, originNDC.y, sizeNDC.x, sizeNDC.y),
|
||||
color: color,
|
||||
params: SIMD4<Float>(1, 0, 0, 0)
|
||||
)
|
||||
encoder.setVertexBytes(&uniforms, length: MemoryLayout<BoxUniforms>.stride, index: 0)
|
||||
encoder.setFragmentBytes(&uniforms, length: MemoryLayout<BoxUniforms>.stride, index: 0)
|
||||
encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 6)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,13 @@ enum Theme {
|
||||
static let boxStroke = SIMD4<Float>(0.20, 0.95, 1.00, 1.0)
|
||||
/// Bounding-box label background.
|
||||
static let boxLabelBg = SIMD4<Float>(0.0, 0.15, 0.18, 0.85)
|
||||
|
||||
/// Text-recognition (OCR) overlay color — distinct from object boxes.
|
||||
static let textStroke = SIMD4<Float>(1.00, 0.45, 0.85, 1.0) // magenta
|
||||
/// Body-pose skeleton color.
|
||||
static let bodyPose = SIMD4<Float>(0.30, 1.00, 0.80, 1.0) // mint
|
||||
/// Hand-pose skeleton color.
|
||||
static let handPose = SIMD4<Float>(1.00, 0.75, 0.20, 1.0) // amber
|
||||
}
|
||||
|
||||
// MARK: - Layout
|
||||
|
||||
Reference in new issue
Block a user