diff --git a/Sources/App/AppDelegate.swift b/Sources/App/AppDelegate.swift index 6c658a6..50c10f0 100644 --- a/Sources/App/AppDelegate.swift +++ b/Sources/App/AppDelegate.swift @@ -1,12 +1,14 @@ import AppKit +import AVFoundation import CoreText import Vision @main -final class AppDelegate: NSObject, NSApplicationDelegate, @unchecked Sendable { +final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate, @unchecked Sendable { private var windowController: MainWindowController? @MainActor private var settingsController: SettingsWindowController? @MainActor private var logController: LogStreamWindowController? + @MainActor private var cameraDevicesSubmenu: NSMenu? static func main() { let app = NSApplication.shared @@ -63,6 +65,17 @@ final class AppDelegate: NSObject, NSApplicationDelegate, @unchecked Sendable { appMenu.addItem(.separator()) + let snapshot = NSMenuItem( + title: "Save Snapshot", + action: #selector(saveSnapshot(_:)), + keyEquivalent: "s" + ) + snapshot.keyEquivalentModifierMask = [.command] + snapshot.target = self + appMenu.addItem(snapshot) + + appMenu.addItem(.separator()) + let quit = NSMenuItem( title: "Quit \(appName)", action: #selector(NSApplication.terminate(_:)), @@ -97,9 +110,93 @@ final class AppDelegate: NSObject, NSApplicationDelegate, @unchecked Sendable { facesItem.submenu = facesMenu + // Camera menu — device picker (dynamic) + manual refresh. + let cameraItem = NSMenuItem() + mainMenu.addItem(cameraItem) + let cameraMenu = NSMenu(title: "Camera") + + let devicesItem = NSMenuItem(title: "Devices", action: nil, keyEquivalent: "") + let devicesSubmenu = NSMenu(title: "Devices") + devicesSubmenu.delegate = self + devicesItem.submenu = devicesSubmenu + cameraDevicesSubmenu = devicesSubmenu + cameraMenu.addItem(devicesItem) + + let refreshItem = NSMenuItem( + title: "Refresh List", + action: #selector(refreshCameraList(_:)), + keyEquivalent: "" + ) + refreshItem.target = self + cameraMenu.addItem(refreshItem) + + cameraItem.submenu = cameraMenu + NSApplication.shared.mainMenu = mainMenu } + // MARK: - Camera menu (dynamic) + + @MainActor + @objc func menuNeedsUpdate(_ menu: NSMenu) { + guard menu === cameraDevicesSubmenu else { return } + rebuildCameraDevicesMenu(menu) + } + + @MainActor + private func rebuildCameraDevicesMenu(_ menu: NSMenu) { + menu.removeAllItems() + + let discovery = AVCaptureDevice.DiscoverySession( + deviceTypes: [.builtInWideAngleCamera, .external, .continuityCamera], + mediaType: .video, + position: .unspecified + ) + let devices = discovery.devices + + if devices.isEmpty { + let empty = NSMenuItem(title: "No cameras found", action: nil, keyEquivalent: "") + empty.isEnabled = false + menu.addItem(empty) + return + } + + let activeID = windowController?.activeCameraUniqueID + for device in devices { + let item = NSMenuItem( + title: device.localizedName, + action: #selector(selectCameraDevice(_:)), + keyEquivalent: "" + ) + item.target = self + item.representedObject = device + if device.uniqueID == activeID { + item.state = .on + } + menu.addItem(item) + } + } + + @MainActor + @objc private func refreshCameraList(_ sender: Any?) { + guard let menu = cameraDevicesSubmenu else { return } + rebuildCameraDevicesMenu(menu) + } + + @MainActor + @objc private func selectCameraDevice(_ sender: Any?) { + guard let item = sender as? NSMenuItem, + let device = item.representedObject as? AVCaptureDevice else { return } + windowController?.switchCamera(to: device) + } + + // MARK: - Snapshot + + @MainActor + @objc private func saveSnapshot(_ sender: Any?) { + windowController?.saveSnapshotToDesktop() + } + // MARK: - Settings @MainActor diff --git a/Sources/App/LogStreamWindowController.swift b/Sources/App/LogStreamWindowController.swift index b1890c6..ae68073 100644 --- a/Sources/App/LogStreamWindowController.swift +++ b/Sources/App/LogStreamWindowController.swift @@ -2,7 +2,9 @@ import AppKit import SwiftUI @MainActor -final class LogStreamWindowController: NSWindowController { +final class LogStreamWindowController: NSWindowController, NSWindowDelegate { + private static let frameDefaultsKey = "VPOC.LogWindowFrame" + init() { let hosting = NSHostingController(rootView: LogStreamView()) let window = NSWindow(contentViewController: hosting) @@ -12,9 +14,29 @@ final class LogStreamWindowController: NSWindowController { window.isReleasedWhenClosed = false window.center() super.init(window: window) + + window.delegate = self + restoreFrame() } required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") } + + private func restoreFrame() { + guard let window, + let saved = UserDefaults.standard.string(forKey: LogStreamWindowController.frameDefaultsKey) else { + return + } + let rect = NSRectFromString(saved) + if rect.size.width > 0 && rect.size.height > 0 { + window.setFrame(rect, display: false) + } + } + + func windowWillClose(_ notification: Notification) { + guard let window else { return } + UserDefaults.standard.set(NSStringFromRect(window.frame), + forKey: LogStreamWindowController.frameDefaultsKey) + } } diff --git a/Sources/App/MainWindowController.swift b/Sources/App/MainWindowController.swift index 5ef6759..3b0f108 100644 --- a/Sources/App/MainWindowController.swift +++ b/Sources/App/MainWindowController.swift @@ -1,14 +1,17 @@ import AppKit +import AVFoundation import MetalKit -final class MainWindowController: NSWindowController { +final class MainWindowController: NSWindowController, NSWindowDelegate { private let mtkView: MTKView private let capture: CameraCapture let detector: ObjectDetector private let edgePass: EdgePass private let jarvisPass: JarvisStylePass private let asciiPass: AsciiPass - private let renderer: Renderer + let renderer: Renderer + + private static let frameDefaultsKey = "VPOC.MainWindowFrame" init() { guard let device = MTLCreateSystemDefaultDevice() else { @@ -72,6 +75,9 @@ final class MainWindowController: NSWindowController { view.delegate = renderer super.init(window: window) + + window.delegate = self + restoreFrame() } required init?(coder: NSCoder) { @@ -83,4 +89,53 @@ final class MainWindowController: NSWindowController { window?.makeKeyAndOrderFront(sender) Task { await capture.start() } } + + // MARK: - Camera + + var activeCameraUniqueID: String? { + capture.activeDeviceUniqueID + } + + func switchCamera(to device: AVCaptureDevice) { + capture.switchToDevice(device) + } + + // MARK: - Snapshot + + @MainActor + func saveSnapshotToDesktop() { + guard let data = renderer.snapshotPNG() else { + LogStream.shared.log("snapshot failed (renderer returned nil)", level: .warn, source: .app) + return + } + let df = DateFormatter() + df.dateFormat = "yyyy-MM-dd-HHmmss" + let name = "VisionPOC-\(df.string(from: Date())).png" + let url = FileManager.default.urls(for: .desktopDirectory, in: .userDomainMask)[0].appendingPathComponent(name) + do { + try data.write(to: url) + LogStream.shared.log("saved \(url.lastPathComponent)", level: .info, source: .app) + } catch { + LogStream.shared.log("snapshot write failed: \(error)", level: .error, source: .app) + } + } + + // MARK: - Window persistence + + private func restoreFrame() { + guard let window, + let saved = UserDefaults.standard.string(forKey: MainWindowController.frameDefaultsKey) else { + return + } + let rect = NSRectFromString(saved) + if rect.size.width > 0 && rect.size.height > 0 { + window.setFrame(rect, display: false) + } + } + + func windowWillClose(_ notification: Notification) { + guard let window else { return } + UserDefaults.standard.set(NSStringFromRect(window.frame), + forKey: MainWindowController.frameDefaultsKey) + } } diff --git a/Sources/App/SettingsWindowController.swift b/Sources/App/SettingsWindowController.swift index 0f7ada5..b2b2364 100644 --- a/Sources/App/SettingsWindowController.swift +++ b/Sources/App/SettingsWindowController.swift @@ -2,7 +2,9 @@ import AppKit import SwiftUI @MainActor -final class SettingsWindowController: NSWindowController { +final class SettingsWindowController: NSWindowController, NSWindowDelegate { + private static let frameDefaultsKey = "VPOC.SettingsWindowFrame" + init() { let hosting = NSHostingController(rootView: SettingsView()) let window = NSWindow(contentViewController: hosting) @@ -12,9 +14,29 @@ final class SettingsWindowController: NSWindowController { window.isReleasedWhenClosed = false window.center() super.init(window: window) + + window.delegate = self + restoreFrame() } required init?(coder: NSCoder) { fatalError("init(coder:) is not supported") } + + private func restoreFrame() { + guard let window, + let saved = UserDefaults.standard.string(forKey: SettingsWindowController.frameDefaultsKey) else { + return + } + let rect = NSRectFromString(saved) + if rect.size.width > 0 && rect.size.height > 0 { + window.setFrame(rect, display: false) + } + } + + func windowWillClose(_ notification: Notification) { + guard let window else { return } + UserDefaults.standard.set(NSStringFromRect(window.frame), + forKey: SettingsWindowController.frameDefaultsKey) + } } diff --git a/Sources/Capture/CameraCapture.swift b/Sources/Capture/CameraCapture.swift index 342825d..7d0a21b 100644 --- a/Sources/Capture/CameraCapture.swift +++ b/Sources/Capture/CameraCapture.swift @@ -22,12 +22,16 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat private let sessionQueue = DispatchQueue(label: "com.dmdbrands.VisionPOC.capture.session") private let sampleQueue = DispatchQueue(label: "com.dmdbrands.VisionPOC.capture.samples") private var textureCache: CVMetalTextureCache? + private var currentInput: AVCaptureDeviceInput? + private var output: AVCaptureVideoDataOutput? private var lock = os_unfair_lock_s() private var latest: Frame? private var _permissionDenied = false + private var _deviceDisconnected = false private var _width: Int = 1920 private var _height: Int = 1080 + private var _activeDeviceUniqueID: String? init(device: MTLDevice) { self.device = device @@ -38,12 +42,31 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat NSLog("CVMetalTextureCacheCreate failed: \(status)") } self.textureCache = cache + + NotificationCenter.default.addObserver( + self, + selector: #selector(handleDeviceDisconnected(_:)), + name: .AVCaptureDeviceWasDisconnected, + object: nil + ) + } + + deinit { + NotificationCenter.default.removeObserver(self) } var permissionDenied: Bool { os_unfair_lock_lock(&lock) defer { os_unfair_lock_unlock(&lock) } - return _permissionDenied + // Treat disconnect as "permission denied" so renderer falls back to its + // existing "WAITING FOR CAMERA…" status path without modification. + return _permissionDenied || _deviceDisconnected + } + + var deviceDisconnected: Bool { + os_unfair_lock_lock(&lock) + defer { os_unfair_lock_unlock(&lock) } + return _deviceDisconnected } var width: Int { @@ -58,6 +81,12 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat return _height } + var activeDeviceUniqueID: String? { + os_unfair_lock_lock(&lock) + defer { os_unfair_lock_unlock(&lock) } + return _activeDeviceUniqueID + } + func latestTexture() -> MTLTexture? { os_unfair_lock_lock(&lock) defer { os_unfair_lock_unlock(&lock) } @@ -108,7 +137,7 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat } let discovery = AVCaptureDevice.DiscoverySession( - deviceTypes: [.builtInWideAngleCamera, .external], + deviceTypes: [.builtInWideAngleCamera, .external, .continuityCamera], mediaType: .video, position: .unspecified ) @@ -129,6 +158,7 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat let input = try AVCaptureDeviceInput(device: camera) if self.session.canAddInput(input) { self.session.addInput(input) + self.currentInput = input } } catch { self.session.commitConfiguration() @@ -146,6 +176,11 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat if self.session.canAddOutput(output) { self.session.addOutput(output) } + self.output = output + + os_unfair_lock_lock(&self.lock) + self._activeDeviceUniqueID = camera.uniqueID + os_unfair_lock_unlock(&self.lock) self.session.commitConfiguration() self.session.startRunning() @@ -154,6 +189,93 @@ final class CameraCapture: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat } } + /// Switch the running session over to a different camera device. Safe to call + /// from the main thread — actual session reconfiguration happens on sessionQueue. + func switchToDevice(_ device: AVCaptureDevice) { + sessionQueue.async { [weak self] in + guard let self else { return } + + if self.session.isRunning { + self.session.stopRunning() + } + + self.session.beginConfiguration() + + if let existing = self.currentInput { + self.session.removeInput(existing) + self.currentInput = nil + } + + do { + let input = try AVCaptureDeviceInput(device: device) + if self.session.canAddInput(input) { + self.session.addInput(input) + self.currentInput = input + } else { + self.session.commitConfiguration() + LogStream.shared.log("cannot add input for \(device.localizedName)", level: .error, source: .camera) + return + } + } catch { + self.session.commitConfiguration() + LogStream.shared.log("failed to create input for \(device.localizedName): \(error)", + level: .error, source: .camera) + return + } + + // If the output got dropped (e.g., never created yet), re-add it. + if self.output == nil { + let output = AVCaptureVideoDataOutput() + output.videoSettings = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferMetalCompatibilityKey as String: true + ] + output.alwaysDiscardsLateVideoFrames = true + output.setSampleBufferDelegate(self, queue: self.sampleQueue) + if self.session.canAddOutput(output) { + self.session.addOutput(output) + } + self.output = output + } + + os_unfair_lock_lock(&self.lock) + self._activeDeviceUniqueID = device.uniqueID + self._deviceDisconnected = false + // Best-effort default; real dimensions arrive with the first sample buffer. + self._width = 1920 + self._height = 1080 + os_unfair_lock_unlock(&self.lock) + + self.session.commitConfiguration() + self.session.startRunning() + + LogStream.shared.log("switched to \(device.localizedName)", level: .info, source: .camera) + } + } + + @objc private func handleDeviceDisconnected(_ note: Notification) { + guard let disconnected = note.object as? AVCaptureDevice else { return } + + os_unfair_lock_lock(&lock) + let activeID = _activeDeviceUniqueID + os_unfair_lock_unlock(&lock) + + guard disconnected.uniqueID == activeID else { return } + + let name = disconnected.localizedName + sessionQueue.async { [weak self] in + guard let self else { return } + if self.session.isRunning { + self.session.stopRunning() + } + os_unfair_lock_lock(&self.lock) + self._deviceDisconnected = true + self.latest = nil + os_unfair_lock_unlock(&self.lock) + LogStream.shared.log("camera \(name) disconnected", level: .warn, source: .camera) + } + } + // MARK: - AVCaptureVideoDataOutputSampleBufferDelegate func captureOutput( diff --git a/Sources/Process/ObjectDetector.swift b/Sources/Process/ObjectDetector.swift index ae08c9b..7fb14f7 100644 --- a/Sources/Process/ObjectDetector.swift +++ b/Sources/Process/ObjectDetector.swift @@ -43,6 +43,9 @@ final class ObjectDetector: @unchecked Sendable { private var lock = os_unfair_lock_s() private var _currentDetections: [Detection] = [] + private var _currentTextDetections: [TextDetection] = [] + private var _currentPoses: [PoseDetection] = [] + private var _lastDetectMode: DetectMode = .yolo private var _lastInferenceMs: Double = 0 private var _inFlight = false @@ -63,6 +66,53 @@ final class ObjectDetector: @unchecked Sendable { } else { LogStream.shared.log("loaded \(Theme.Performance.detectorModelName), tracker enabled (~5 Hz YOLO + ~60 Hz tracker)", level: .info, source: .detect) } + + // Warm up the YOLO model on the detector queue so the first user-visible + // detection doesn't pay ~200ms of CoreML/ANE compile latency. Dispatched + // async so init() returns immediately and app launch isn't blocked. + queue.async { [weak self] in + self?.warmUpYolo() + } + } + + private func warmUpYolo() { + guard let yolo = yoloRequest else { return } + let start = CFAbsoluteTimeGetCurrent() + + // 416x416 mid-gray bitmap — YOLO's typical input size. The model will + // rescale internally if needed; what matters is the CoreML graph gets + // compiled and the ANE picks up the weights. + let width = 416 + let height = 416 + let bytesPerRow = width * 4 + let colorSpace = CGColorSpaceCreateDeviceRGB() + guard let ctx = CGContext( + data: nil, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: bytesPerRow, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) else { + LogStream.shared.log("yolo warmup skipped: bitmap context alloc failed", level: .warn, source: .detect) + return + } + ctx.setFillColor(CGColor(red: 0.5, green: 0.5, blue: 0.5, alpha: 1.0)) + ctx.fill(CGRect(x: 0, y: 0, width: width, height: height)) + guard let cgImage = ctx.makeImage() else { + LogStream.shared.log("yolo warmup skipped: cgImage creation failed", level: .warn, source: .detect) + return + } + + let handler = VNImageRequestHandler(cgImage: cgImage, orientation: .up, options: [:]) + do { + try handler.perform([yolo]) + let ms = (CFAbsoluteTimeGetCurrent() - start) * 1000.0 + LogStream.shared.log("warmed up YOLO in \(String(format: "%.0f", ms))ms", level: .debug, source: .detect) + } catch { + LogStream.shared.log("yolo warmup failed: \(error)", level: .warn, source: .detect) + } } private static func makeYoloRequest(modelName: String) -> (request: VNCoreMLRequest?, failureReason: String?) { @@ -96,6 +146,24 @@ final class ObjectDetector: @unchecked Sendable { return _currentDetections } + func currentTextDetections() -> [TextDetection] { + os_unfair_lock_lock(&lock) + defer { os_unfair_lock_unlock(&lock) } + return _currentTextDetections + } + + func currentPoses() -> [PoseDetection] { + os_unfair_lock_lock(&lock) + defer { os_unfair_lock_unlock(&lock) } + return _currentPoses + } + + var lastDetectMode: DetectMode { + os_unfair_lock_lock(&lock) + defer { os_unfair_lock_unlock(&lock) } + return _lastDetectMode + } + func submit(pixelBuffer: CVPixelBuffer) { os_unfair_lock_lock(&lock) if _inFlight { @@ -118,11 +186,19 @@ final class ObjectDetector: @unchecked Sendable { let yoloInterval = 1.0 / max(0.5, Theme.Performance.yoloDetectionHz) let shouldDetect = (start - lastYoloAt) >= yoloInterval + var newTextDetections: [TextDetection] = [] + var newPoses: [PoseDetection] = [] + let detectMode: DetectMode + if shouldDetect { - detectAndBootstrap(pixelBuffer: pixelBuffer, now: start) + let extras = detectAndBootstrap(pixelBuffer: pixelBuffer, now: start) + newTextDetections = extras.text + newPoses = extras.poses lastYoloAt = start + detectMode = .yolo } else { trackOnly(pixelBuffer: pixelBuffer, now: start) + detectMode = .track } // Drop tracks that have aged out (no YOLO refresh recently). @@ -157,6 +233,14 @@ final class ObjectDetector: @unchecked Sendable { os_unfair_lock_lock(&lock) _currentDetections = snapshot + if shouldDetect { + // Only refresh text & poses on YOLO-cadence frames; tracker frames + // keep the most recent set so the renderer doesn't flicker between + // populated and empty in the ~50ms between YOLO refreshes. + _currentTextDetections = newTextDetections + _currentPoses = newPoses + } + _lastDetectMode = detectMode _lastInferenceMs = elapsedMs _inFlight = false os_unfair_lock_unlock(&lock) @@ -164,10 +248,19 @@ final class ObjectDetector: @unchecked Sendable { // MARK: - YOLO + supplementary detectors - private func detectAndBootstrap(pixelBuffer: CVPixelBuffer, now: CFAbsoluteTime) { + private func detectAndBootstrap(pixelBuffer: CVPixelBuffer, now: CFAbsoluteTime) -> (text: [TextDetection], poses: [PoseDetection]) { let faces = VNDetectFaceRectanglesRequest() let animals = VNRecognizeAnimalsRequest() - var requests: [VNRequest] = [faces, animals] + + let text = VNRecognizeTextRequest() + text.recognitionLevel = .accurate + text.usesLanguageCorrection = true + + let bodyPose = VNDetectHumanBodyPoseRequest() + let handPose = VNDetectHumanHandPoseRequest() + handPose.maximumHandCount = 2 + + var requests: [VNRequest] = [faces, animals, text, bodyPose, handPose] if let yolo = yoloRequest { requests.append(yolo) } @@ -248,6 +341,156 @@ final class ObjectDetector: @unchecked Sendable { level: .info, source: .track) } } + + // ---- Text recognition ---- + let textDetections: [TextDetection] = (text.results ?? []).compactMap { obs in + guard let top = obs.topCandidates(1).first else { return nil } + return TextDetection(rect: obs.boundingBox, text: top.string, confidence: top.confidence) + } + + // ---- Body pose ---- + var bodyPoses: [PoseDetection] = [] + if let bodyResults = bodyPose.results { + for obs in bodyResults { + if let pose = makeBodyPose(from: obs) { + bodyPoses.append(pose) + } + } + } + + // ---- Hand pose ---- + var handPoses: [PoseDetection] = [] + if let handResults = handPose.results { + for obs in handResults { + if let pose = makeHandPose(from: obs) { + handPoses.append(pose) + } + } + } + + return (text: textDetections, poses: bodyPoses + handPoses) + } + + // MARK: - Pose builders + + private static let poseConfidenceThreshold: Float = 0.3 + + private func makeBodyPose(from obs: VNHumanBodyPoseObservation) -> PoseDetection? { + // Apple's SDK joint names. We map each to its CGPoint via + // recognizedPoint(_:); points with confidence > threshold are kept. + typealias J = VNHumanBodyPoseObservation.JointName + let connections: [(J, J)] = [ + (.nose, .neck), + (.neck, .leftShoulder), + (.leftShoulder, .leftElbow), + (.leftElbow, .leftWrist), + (.neck, .rightShoulder), + (.rightShoulder, .rightElbow), + (.rightElbow, .rightWrist), + (.neck, .root), + (.root, .leftHip), + (.leftHip, .leftKnee), + (.leftKnee, .leftAnkle), + (.root, .rightHip), + (.rightHip, .rightKnee), + (.rightKnee, .rightAnkle) + ] + + var segments: [PoseDetection.Segment] = [] + // Use a small key->point map keyed by joint rawValue so points stay + // unique even if a joint appears in multiple connections. + var uniquePoints: [String: CGPoint] = [:] + var maxConf: Float = 0 + + for (a, b) in connections { + guard + let p1 = try? obs.recognizedPoint(a), + let p2 = try? obs.recognizedPoint(b) + else { continue } + if p1.confidence > ObjectDetector.poseConfidenceThreshold { + uniquePoints[a.rawValue.rawValue] = p1.location + maxConf = max(maxConf, p1.confidence) + } + if p2.confidence > ObjectDetector.poseConfidenceThreshold { + uniquePoints[b.rawValue.rawValue] = p2.location + maxConf = max(maxConf, p2.confidence) + } + if p1.confidence > ObjectDetector.poseConfidenceThreshold, + p2.confidence > ObjectDetector.poseConfidenceThreshold { + segments.append(PoseDetection.Segment(start: p1.location, end: p2.location)) + } + } + + if segments.isEmpty && uniquePoints.isEmpty { return nil } + return PoseDetection( + segments: segments, + points: Array(uniquePoints.values), + kind: .body, + confidence: maxConf + ) + } + + private func makeHandPose(from obs: VNHumanHandPoseObservation) -> PoseDetection? { + typealias J = VNHumanHandPoseObservation.JointName + + // Each finger chain: tip -> middle phalange -> proximal -> metacarpal/base. + // Apple's hand-pose joint names (verified against the macOS SDK): + // thumb: thumbTip, thumbIP, thumbMP, thumbCMC + // index: indexTip, indexDIP, indexPIP, indexMCP + // middle: middleTip, middleDIP,middlePIP,middleMCP + // ring: ringTip, ringDIP, ringPIP, ringMCP + // little: littleTip, littleDIP,littlePIP,littleMCP + let fingerChains: [[J]] = [ + [.thumbTip, .thumbIP, .thumbMP, .thumbCMC], + [.indexTip, .indexDIP, .indexPIP, .indexMCP], + [.middleTip, .middleDIP, .middlePIP, .middleMCP], + [.ringTip, .ringDIP, .ringPIP, .ringMCP], + [.littleTip, .littleDIP, .littlePIP, .littleMCP] + ] + // Palm ring: connect the 5 base/knuckle joints around the palm. + // Start at thumb base (CMC) and walk index→middle→ring→little MCPs, + // then close the ring back to the thumb base. + let palmRing: [J] = [.thumbCMC, .indexMCP, .middleMCP, .ringMCP, .littleMCP, .thumbCMC] + + var segments: [PoseDetection.Segment] = [] + var uniquePoints: [String: CGPoint] = [:] + var maxConf: Float = 0 + + func addPoint(_ name: J) -> (location: CGPoint, ok: Bool) { + guard let p = try? obs.recognizedPoint(name) else { return (.zero, false) } + if p.confidence > ObjectDetector.poseConfidenceThreshold { + uniquePoints[name.rawValue.rawValue] = p.location + maxConf = max(maxConf, p.confidence) + return (p.location, true) + } + return (.zero, false) + } + + for chain in fingerChains { + for i in 0..<(chain.count - 1) { + let a = addPoint(chain[i]) + let b = addPoint(chain[i + 1]) + if a.ok && b.ok { + segments.append(PoseDetection.Segment(start: a.location, end: b.location)) + } + } + } + + for i in 0..<(palmRing.count - 1) { + let a = addPoint(palmRing[i]) + let b = addPoint(palmRing[i + 1]) + if a.ok && b.ok { + segments.append(PoseDetection.Segment(start: a.location, end: b.location)) + } + } + + if segments.isEmpty && uniquePoints.isEmpty { return nil } + return PoseDetection( + segments: segments, + points: Array(uniquePoints.values), + kind: .hand, + confidence: maxConf + ) } // MARK: - Tracker-only update diff --git a/Sources/Process/ProcessedFrame.swift b/Sources/Process/ProcessedFrame.swift new file mode 100644 index 0000000..40b8531 --- /dev/null +++ b/Sources/Process/ProcessedFrame.swift @@ -0,0 +1,47 @@ +import Foundation +import CoreGraphics + +/// Shared frame-annotation types produced by `ObjectDetector` and consumed by +/// `Renderer`. All coordinates use Vision's normalized [0..1] image space +/// (origin bottom-left), matching the existing `Detection.rect`. The renderer +/// converts to viewport NDC the same way it does for detections, so adding a +/// new overlay type is just an array of rects-or-points and a draw call. + +/// Recognized text region — produced by `VNRecognizeTextRequest`. +struct TextDetection: Sendable { + var rect: CGRect // normalized, origin bottom-left + var text: String // the recognized string + var confidence: Float +} + +enum PoseKind: Sendable { + case body + case hand +} + +/// A pose detection — a set of normalized line segments (joint pairs) plus +/// the keypoint positions themselves. The renderer draws lines between the +/// segment endpoints and small dots at each unique point. +struct PoseDetection: Sendable { + /// Normalized line segments (each segment goes from start to end in + /// Vision's [0..1] image coords, origin bottom-left). + var segments: [Segment] + /// All unique joint points present in this pose, for dot rendering. + var points: [CGPoint] + /// Whether this pose is a body skeleton or a hand skeleton. + var kind: PoseKind + /// Overall confidence of the pose (max of constituent joint confidences). + var confidence: Float + + struct Segment: Sendable { + var start: CGPoint + var end: CGPoint + } +} + +/// Tells the renderer which detector mode produced the most recent +/// inference, so the footer can label `DET 28.4ms (yolo)` vs `DET 2.1ms (track)`. +enum DetectMode: String, Sendable { + case yolo = "yolo" + case track = "track" +} diff --git a/Sources/Render/ColorHash.swift b/Sources/Render/ColorHash.swift new file mode 100644 index 0000000..48f3682 --- /dev/null +++ b/Sources/Render/ColorHash.swift @@ -0,0 +1,53 @@ +import Foundation +import simd + +/// Deterministic per-trackId color generator. Hashes a UUID into a hue value +/// in [0, 1) and converts HSV(hue, 0.8, 1.0) to RGB. Used to give each tracked +/// object a stable, visually-distinct box + label color so it's clear which +/// label belongs to which box when several boxes overlap. +/// +/// Uniqueness caveat: with hue alone covering ~32 buckets of human-discernible +/// difference, ~6 simultaneous tracks of the same hue family will start to look +/// similar. For the POC's typical 3-8 tracks this is fine; if track count +/// regularly exceeds ~12 a second axis (saturation or value) would help. +enum ColorHash { + /// Returns a stable RGBA color for the given track id. + /// Saturation 0.8, value 1.0, alpha 1.0. + static func colorFor(trackId: UUID) -> SIMD4 { + // Spread the UUID's bytes across the hash so adjacent UUIDs (which the + // standard library doesn't actually generate adjacently, but be safe) + // map to far-apart hues. Mix several bytes from the uuid tuple. + let bytes = trackId.uuid + // Combine bytes 0, 3, 7, 11 with a multiplicative hash. The constant + // 2654435769 is Knuth's golden-ratio hash multiplier — distributes + // small input differences across the full 32-bit output range. + var h: UInt32 = UInt32(bytes.0) + h = h &* 2654435769 &+ UInt32(bytes.3) + h = h &* 2654435769 &+ UInt32(bytes.7) + h = h &* 2654435769 &+ UInt32(bytes.11) + let hue = Float(h % 360) / 360.0 + return hsvToRGB(h: hue, s: 0.8, v: 1.0) + } + + /// HSV → RGB, standard formula. Hue is in [0,1), s and v in [0,1]. + private static func hsvToRGB(h: Float, s: Float, v: Float) -> SIMD4 { + let hh = (h - floor(h)) * 6.0 + let i = Int(floor(hh)) + let f = hh - Float(i) + let p = v * (1 - s) + let q = v * (1 - s * f) + let t = v * (1 - s * (1 - f)) + let r: Float + let g: Float + let b: Float + switch i % 6 { + case 0: r = v; g = t; b = p + case 1: r = q; g = v; b = p + case 2: r = p; g = v; b = t + case 3: r = p; g = q; b = v + case 4: r = t; g = p; b = v + default: r = v; g = p; b = q + } + return SIMD4(r, g, b, 1.0) + } +} diff --git a/Sources/Render/Renderer.swift b/Sources/Render/Renderer.swift index 0827528..0c980ba 100644 --- a/Sources/Render/Renderer.swift +++ b/Sources/Render/Renderer.swift @@ -1,6 +1,9 @@ import MetalKit import AppKit import simd +import os +import ImageIO +import UniformTypeIdentifiers /// Camera textures coming from AVFoundation are stored origin-top-left. The MSL /// `quad_vertex` shader emits UVs with `v = 1.0 - v` so image samples appear right-side-up @@ -36,9 +39,23 @@ final class Renderer: NSObject, MTKViewDelegate { private let textRasterizer: TextRasterizer private var labelTextureCache: [String: MTLTexture] = [:] + /// Parallel array tracking access order for `labelTextureCache`. Head is + /// LRU, tail is MRU. On cache hit, move-to-end. On cache miss with the + /// cache at capacity, drop head before inserting. + private var labelTextureCacheKeys: [String] = [] + private let labelTextureCacheCapacity = 128 private let nearestSampler: MTLSamplerState private let linearSampler: MTLSamplerState + /// Set by `snapshotPNG()` on a non-render thread. The next `draw(in:)` + /// pass renders normally, then copies the drawable's texture into a + /// CPU-readable MTLTexture, encodes it as PNG, fires this completion, + /// and clears the flag. + private var pendingSnapshotCompletion: ((Data?) -> Void)? + /// Guards `pendingSnapshotCompletion` so the snapshot caller (background) + /// and the render loop (MainActor) don't race on read/write. + private var snapshotLock = os_unfair_lock_s() + private let startTime: CFAbsoluteTime = CFAbsoluteTimeGetCurrent() private var lastFrameTime: CFAbsoluteTime private var frameCounter: Int = 0 @@ -179,6 +196,10 @@ final class Renderer: NSObject, MTKViewDelegate { drawJarvis(encoder: encoder, texture: source, time: elapsed, paneSize: inner.size) drawDetections(encoder: encoder, detections: detections) drawDetectionLabels(encoder: encoder, detections: detections, paneRect: inner) + let texts = detector.currentTextDetections() + drawTextDetections(encoder: encoder, texts: texts, paneRect: inner) + let poses = detector.currentPoses() + drawPoses(encoder: encoder, poses: poses, paneRect: inner) case 2: drawImage(encoder: encoder, pipeline: pipelines.edges, texture: edgesTexture, sampler: nearestSampler) case 3: @@ -193,8 +214,148 @@ final class Renderer: NSObject, MTKViewDelegate { drawFooter(encoder: encoder, drawableSize: drawableSize) encoder.endEncoding() - commandBuffer.present(drawable) - commandBuffer.commit() + + // Snapshot path — piggyback on this frame's command buffer. We blit + // the drawable's texture into a CPU-readable shared-storage texture, + // then synchronously wait for the GPU to finish before reading back + // and encoding as PNG. ~10-30ms typical (dominated by waitUntilCompleted). + let snapshotCompletion = takePendingSnapshotCompletion() + if let completion = snapshotCompletion { + let readback = makeReadbackTexture(matching: drawable.texture) + if let readback, + let blit = commandBuffer.makeBlitCommandEncoder() { + blit.copy(from: drawable.texture, to: readback) + blit.endEncoding() + commandBuffer.present(drawable) + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + let data = encodePNG(from: readback) + completion(data) + } else { + // Couldn't allocate readback; fail gracefully. + commandBuffer.present(drawable) + commandBuffer.commit() + completion(nil) + } + } else { + commandBuffer.present(drawable) + commandBuffer.commit() + } + } + + // MARK: - Snapshot + + /// Public synchronous API. Sets the pending-completion flag, waits via a + /// semaphore for the next render loop to fulfill it, returns the PNG + /// data (or nil on failure). Safe to call off the MainActor; the wait is + /// at most ~30ms per shot. + func snapshotPNG() -> Data? { + let semaphore = DispatchSemaphore(value: 0) + // Capture into a class-level Sendable box so the completion can write + // it from whatever thread fires the completion. + let box = SnapshotBox() + let completion: (Data?) -> Void = { data in + box.data = data + semaphore.signal() + } + os_unfair_lock_lock(&snapshotLock) + // If a previous snapshot is still in flight, fail this one rather + // than stomp on it. The caller can retry. + if pendingSnapshotCompletion != nil { + os_unfair_lock_unlock(&snapshotLock) + return nil + } + pendingSnapshotCompletion = completion + os_unfair_lock_unlock(&snapshotLock) + + // 1 second is generous; if the render loop is stalled for longer than + // that, returning nil and letting the user retry is better than + // hanging the menu thread forever. + let result = semaphore.wait(timeout: .now() + .seconds(1)) + if result == .timedOut { + os_unfair_lock_lock(&snapshotLock) + pendingSnapshotCompletion = nil + os_unfair_lock_unlock(&snapshotLock) + return nil + } + return box.data + } + + private final class SnapshotBox: @unchecked Sendable { + var data: Data? + } + + private func takePendingSnapshotCompletion() -> ((Data?) -> Void)? { + os_unfair_lock_lock(&snapshotLock) + let c = pendingSnapshotCompletion + pendingSnapshotCompletion = nil + os_unfair_lock_unlock(&snapshotLock) + return c + } + + /// Allocates a shared-storage texture matching the given drawable texture's + /// dimensions and pixel format, so its bytes are CPU-addressable after the + /// blit completes. The drawable's own texture has private storage and + /// cannot be read back directly. + private func makeReadbackTexture(matching source: MTLTexture) -> MTLTexture? { + let descriptor = MTLTextureDescriptor.texture2DDescriptor( + pixelFormat: source.pixelFormat, + width: source.width, + height: source.height, + mipmapped: false + ) + descriptor.usage = [.shaderRead] + descriptor.storageMode = .shared + return device.makeTexture(descriptor: descriptor) + } + + private func encodePNG(from texture: MTLTexture) -> Data? { + let width = texture.width + let height = texture.height + let bytesPerRow = width * 4 + var bytes = [UInt8](repeating: 0, count: bytesPerRow * height) + let region = MTLRegionMake2D(0, 0, width, height) + bytes.withUnsafeMutableBytes { ptr in + if let base = ptr.baseAddress { + texture.getBytes(base, bytesPerRow: bytesPerRow, from: region, mipmapLevel: 0) + } + } + + // Drawable is bgra8Unorm. CGBitmapContext wants the byte order spelled + // out via CGBitmapInfo: byteOrder32Little + premultipliedFirst means + // memory layout BGRA, which matches the texture exactly — no swizzle. + let colorSpace = CGColorSpace(name: CGColorSpace.sRGB) ?? CGColorSpaceCreateDeviceRGB() + let bitmapInfo: CGBitmapInfo = [ + CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedFirst.rawValue), + .byteOrder32Little + ] + guard let context = bytes.withUnsafeMutableBytes({ ptr -> CGContext? in + CGContext( + data: ptr.baseAddress, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: bytesPerRow, + space: colorSpace, + bitmapInfo: bitmapInfo.rawValue + ) + }) else { + return nil + } + guard let cgImage = context.makeImage() else { return nil } + + let nsData = NSMutableData() + guard let destination = CGImageDestinationCreateWithData( + nsData as CFMutableData, + UTType.png.identifier as CFString, + 1, + nil + ) else { + return nil + } + CGImageDestinationAddImage(destination, cgImage, nil) + guard CGImageDestinationFinalize(destination) else { return nil } + return nsData as Data } // MARK: - No-source path @@ -316,7 +477,7 @@ final class Renderer: NSObject, MTKViewDelegate { let sizeNDC = SIMD2(Float(r.width) * 2, Float(r.height) * 2) var uniforms = BoxUniforms( rect: SIMD4(originNDC.x, originNDC.y, sizeNDC.x, sizeNDC.y), - color: Theme.Palette.boxStroke, + color: boxColor(for: detection), params: SIMD4(0, 0, 0, 0) // mode 0 = hollow stroke ) encoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 0) @@ -325,6 +486,28 @@ final class Renderer: NSObject, MTKViewDelegate { } } + /// Cyan for face/identity labels, per-track hue for everything else. + /// Face labels are recognizable enough on their own that a stable cyan + /// makes the UI feel less chaotic than randomized hues. + private func boxColor(for detection: Detection) -> SIMD4 { + if isFaceLabel(detection.label) { + return Theme.Palette.boxStroke + } + return ColorHash.colorFor(trackId: detection.trackId) + } + + private func isFaceLabel(_ label: String) -> Bool { + if label == "FACE" { return true } + // FaceRegistry names are uppercased before being assigned as a label + // (see ObjectDetector.detectAndBootstrap). Anything in the registry + // should keep the cyan stroke. Falling back to "all caps single word" + // would accidentally include YOLO labels like CUP; check the registry + // instead. The enrolledNames() call is cheap (dictionary keys snapshot + // behind a small lock); we run it at most once per detection per frame. + let names = detector.faceRegistry.enrolledNames() + return names.contains(where: { $0.uppercased() == label }) + } + private func drawCornerBrackets(encoder: MTLRenderCommandEncoder, drawableSize: CGSize, innerRects: [CGRect]) { let bracketLen = Theme.HUD.cornerBracketLength * backingScale let thickness = Theme.HUD.cornerBracketThickness * backingScale @@ -405,10 +588,12 @@ final class Renderer: NSObject, MTKViewDelegate { case .balanced: profile = "BALANCED" case .quality: profile = "QUALITY" } + let mode = detector.lastDetectMode.rawValue let text = String( - format: "VISIONPOC · FPS %d · DET %.1fms · %@", + format: "VISIONPOC · FPS %d · DET %.1fms (%@) · %@", Int(smoothedFPS.rounded()), detector.lastInferenceMs, + mode, profile ) footerTexture = makeLabelTexture(text, font: Theme.Font.body(Theme.Font.footerSize), wide: true) @@ -434,10 +619,28 @@ final class Renderer: NSObject, MTKViewDelegate { // MARK: - Detection labels & ASCII art + /// LRU-bounded label texture cache. Capacity capped at + /// `labelTextureCacheCapacity`. On hit, the key is moved to the tail + /// (most-recently-used). On miss with the cache full, the head (LRU) is + /// evicted before insertion. private func labelTexture(for text: String) -> MTLTexture? { - if let cached = labelTextureCache[text] { return cached } - let tex = makeLabelTexture(text, font: Theme.Font.body(Theme.Font.boxLabelSize)) - if let tex { labelTextureCache[text] = tex } + if let cached = labelTextureCache[text] { + // Move-to-end on hit. removeAll(where:) is O(n) for a 128-element + // array; trivially fast and avoids pulling in a real LRU library. + labelTextureCacheKeys.removeAll(where: { $0 == text }) + labelTextureCacheKeys.append(text) + return cached + } + guard let tex = makeLabelTexture(text, font: Theme.Font.body(Theme.Font.boxLabelSize)) else { + return nil + } + if labelTextureCacheKeys.count >= labelTextureCacheCapacity, + let oldest = labelTextureCacheKeys.first { + labelTextureCacheKeys.removeFirst() + labelTextureCache.removeValue(forKey: oldest) + } + labelTextureCache[text] = tex + labelTextureCacheKeys.append(text) return tex } @@ -449,7 +652,12 @@ final class Renderer: NSObject, MTKViewDelegate { let labelHeight: CGFloat = 18 * backingScale let inset: CGFloat = 4 * backingScale for det in detections { - guard let texture = labelTexture(for: det.label) else { continue } + // Confidence quantized to 10% buckets to keep label cache small. + // A CUP at 80% and a CUP at 90% get separate cached textures, + // but two CUPs at 85% share one (both round to 90%). + let bucket = Int((det.confidence * 10).rounded()) * 10 + let displayed = "\(det.label) \(bucket)%" + guard let texture = labelTexture(for: displayed) else { continue } let r = det.rect let boxLeftPx = paneRect.origin.x + CGFloat(r.origin.x) * paneRect.width // Box top edge in screen pixels (y-down). Vision y is origin @@ -458,14 +666,143 @@ final class Renderer: NSObject, MTKViewDelegate { let boxTopPx = paneRect.origin.y + (1.0 - (CGFloat(r.origin.y) + CGFloat(r.height))) * paneRect.height let labelWidth = CGFloat(texture.width) + // Clamp to pane edges so the label doesn't extend past the + // drawable when a box is hugging the right or top edge. + let rawX = boxLeftPx + inset + let rawY = boxTopPx + inset + let clampedX = min(rawX, paneRect.maxX - labelWidth - inset) + let clampedY = max(rawY, paneRect.minY + inset) let rect = CGRect( - x: boxLeftPx + inset, - y: boxTopPx + inset, + x: clampedX, + y: clampedY, width: labelWidth, height: labelHeight ) setViewport(encoder: encoder, rect: rect) - drawTextQuad(encoder: encoder, texture: texture, tint: Theme.Palette.boxStroke) + drawTextQuad(encoder: encoder, texture: texture, tint: boxColor(for: det)) + } + } + + // MARK: - OCR overlay + + private func drawTextDetections( + encoder: MTLRenderCommandEncoder, + texts: [TextDetection], + paneRect: CGRect + ) { + guard !texts.isEmpty else { return } + + // Hollow magenta box per recognized text region. + encoder.setRenderPipelineState(pipelines.boxes) + for t in texts { + let r = t.rect + let originNDC = SIMD2(Float(r.origin.x) * 2 - 1, Float(r.origin.y) * 2 - 1) + let sizeNDC = SIMD2(Float(r.width) * 2, Float(r.height) * 2) + var uniforms = BoxUniforms( + rect: SIMD4(originNDC.x, originNDC.y, sizeNDC.x, sizeNDC.y), + color: Theme.Palette.textStroke, + params: SIMD4(0, 0, 0, 0) + ) + encoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 0) + encoder.setFragmentBytes(&uniforms, length: MemoryLayout.stride, index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 6) + } + + // Magenta label INSIDE the top-left of each text rect, clamped to pane. + let labelHeight: CGFloat = 18 * backingScale + let inset: CGFloat = 4 * backingScale + for t in texts { + // OCR strings can be arbitrarily long; truncate to a sane prefix + // before caching so cache key cardinality stays bounded. The LRU + // cap also kicks in here. + let key = t.text.count > 32 + ? String(t.text.prefix(32)) + : t.text + guard let texture = labelTexture(for: key) else { continue } + let r = t.rect + let boxLeftPx = paneRect.origin.x + CGFloat(r.origin.x) * paneRect.width + let boxTopPx = paneRect.origin.y + + (1.0 - (CGFloat(r.origin.y) + CGFloat(r.height))) * paneRect.height + let labelWidth = CGFloat(texture.width) + let rawX = boxLeftPx + inset + let rawY = boxTopPx + inset + let clampedX = min(rawX, paneRect.maxX - labelWidth - inset) + let clampedY = max(rawY, paneRect.minY + inset) + let rect = CGRect( + x: clampedX, + y: clampedY, + width: labelWidth, + height: labelHeight + ) + setViewport(encoder: encoder, rect: rect) + drawTextQuad(encoder: encoder, texture: texture, tint: Theme.Palette.textStroke) + } + } + + // MARK: - Pose overlay + + private func drawPoses( + encoder: MTLRenderCommandEncoder, + poses: [PoseDetection], + paneRect: CGRect + ) { + guard !poses.isEmpty else { return } + encoder.setRenderPipelineState(pipelines.boxes) + + for pose in poses { + let color: SIMD4 + switch pose.kind { + case .body: color = Theme.Palette.bodyPose + case .hand: color = Theme.Palette.handPose + } + + // Segments — Option A: draw each as the axis-aligned bounding rect + // of its endpoints. Blocky for diagonals but a single draw call per + // segment and matches the existing shader contract exactly. A v2 + // could subdivide into mini-rects for smoother diagonals. + for seg in pose.segments { + let minX = Float(min(seg.start.x, seg.end.x)) + let maxX = Float(max(seg.start.x, seg.end.x)) + let minY = Float(min(seg.start.y, seg.end.y)) + let maxY = Float(max(seg.start.y, seg.end.y)) + // Give zero-extent (perfectly horizontal/vertical) segments a + // visible thickness — without this they'd render as a 0-pixel + // strip and vanish. + let thicknessNorm: Float = 0.003 + let w = max(maxX - minX, thicknessNorm) + let h = max(maxY - minY, thicknessNorm) + let originNDC = SIMD2(minX * 2 - 1, minY * 2 - 1) + let sizeNDC = SIMD2(w * 2, h * 2) + var uniforms = BoxUniforms( + rect: SIMD4(originNDC.x, originNDC.y, sizeNDC.x, sizeNDC.y), + color: color, + params: SIMD4(1, 0, 0, 0) // solid fill + ) + encoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 0) + encoder.setFragmentBytes(&uniforms, length: MemoryLayout.stride, index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 6) + } + + // Joint dots — 3 px squares. Convert 3 px to normalized image + // coords using the pane size so the dots stay roughly the same + // visual size regardless of pane. + let dotPx: CGFloat = 3 * backingScale + let dotNormX = Float(dotPx / max(paneRect.width, 1)) + let dotNormY = Float(dotPx / max(paneRect.height, 1)) + for p in pose.points { + let px = Float(p.x) - dotNormX * 0.5 + let py = Float(p.y) - dotNormY * 0.5 + let originNDC = SIMD2(px * 2 - 1, py * 2 - 1) + let sizeNDC = SIMD2(dotNormX * 2, dotNormY * 2) + var uniforms = BoxUniforms( + rect: SIMD4(originNDC.x, originNDC.y, sizeNDC.x, sizeNDC.y), + color: color, + params: SIMD4(1, 0, 0, 0) + ) + encoder.setVertexBytes(&uniforms, length: MemoryLayout.stride, index: 0) + encoder.setFragmentBytes(&uniforms, length: MemoryLayout.stride, index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 6) + } } } diff --git a/Sources/Theme.swift b/Sources/Theme.swift index e6737d8..a2e2039 100644 --- a/Sources/Theme.swift +++ b/Sources/Theme.swift @@ -25,6 +25,13 @@ enum Theme { static let boxStroke = SIMD4(0.20, 0.95, 1.00, 1.0) /// Bounding-box label background. static let boxLabelBg = SIMD4(0.0, 0.15, 0.18, 0.85) + + /// Text-recognition (OCR) overlay color — distinct from object boxes. + static let textStroke = SIMD4(1.00, 0.45, 0.85, 1.0) // magenta + /// Body-pose skeleton color. + static let bodyPose = SIMD4(0.30, 1.00, 0.80, 1.0) // mint + /// Hand-pose skeleton color. + static let handPose = SIMD4(1.00, 0.75, 0.20, 1.0) // amber } // MARK: - Layout