Coordinated three agents working in parallel on non-overlapping files,
with a shared `ProcessedFrame.swift` contract locking the API surface
between detector and renderer. All build clean on first integrate.
## Detector (ObjectDetector.swift)
- `VNRecognizeTextRequest` runs alongside YOLO at ~5 Hz; recognized
strings + boxes published via `currentTextDetections()`
- `VNDetectHumanBodyPoseRequest` produces a 14-segment skeleton (nose-
neck-shoulders-arms, neck-root-hips-legs); confidence-gated at 0.3
- `VNDetectHumanHandPoseRequest` produces 5 finger chains + palm ring
per hand, up to 2 hands; published as kind=.hand poses
- Body + hand published as `[PoseDetection]` via `currentPoses()`
- New `lastDetectMode: DetectMode { .yolo | .track }` so the footer can
distinguish the two latency regimes
- YOLO warmup runs one dummy inference at startup on the detector
queue — first user-visible detection no longer pays the ~200ms
model-compile latency
## Renderer (Renderer.swift + Render/ColorHash.swift)
- Box labels include 10%-bucketed confidence: "CUP 80%", "MOBILE PHONE
60%". Bucket keeps the label texture cache from exploding.
- Per-track stable color via UUID-byte → golden-ratio hash → HSV(h,
0.8, 1.0). Face labels stay cyan (calmer when JARVIS pane has
multiple labeled people in frame).
- Detection labels clamp to pane edges so boxes near the right margin
don't overflow into the next pane.
- Footer now reads "DET 28.4ms (yolo)" or "DET 2.1ms (track)" — the
flicker between the two values is informative, not a bug.
- LRU bound on `labelTextureCache` (128 entries, parallel access-order
array, drop-head-on-insert when at capacity).
- OCR text rendered as magenta boxes with truncated text labels (32
chars).
- Body pose drawn in mint, hand pose in amber; axis-aligned segment
rects with a 0.003 minimum thickness so horizontal/vertical lines
don't vanish at small viewport sizes. Joint dots at each unique
point.
- New `snapshotPNG() -> Data?` public method. Render loop piggybacks
one extra blit pass + waitUntilCompleted + CGImage encoding. <50ms
in practice, 1-second semaphore timeout if rendering stalls.
## App / Window / Capture
- Camera menu with a dynamic Devices submenu (rebuilt via
NSMenuDelegate.menuNeedsUpdate). Active device gets a checkmark,
Refresh List forces re-enumeration.
- CameraCapture.switchToDevice(_:) stops the session, swaps the input,
restarts. activeDeviceUniqueID exposed so the menu can match.
- Save Snapshot (⌘S) writes the 4-pane composite to
~/Desktop/VisionPOC-YYYY-MM-DD-HHMMSS.png
- Window-frame persistence for all three windows via UserDefaults
keys VPOC.MainWindowFrame / VPOC.SettingsWindowFrame /
VPOC.LogWindowFrame
- AVCaptureDevice.wasDisconnectedNotification → session stop + flag
set; permissionDenied returns true so the existing
"WAITING FOR CAMERA…" status path triggers without renderer changes
## Shared contract (new file)
- Sources/Process/ProcessedFrame.swift — TextDetection, PoseDetection,
PoseDetection.Segment, PoseKind (.body | .hand), DetectMode
(.yolo | .track). Sendable, value types, normalized [0..1] Vision
coords throughout.
## Theme
- Palette gains textStroke (magenta), bodyPose (mint), handPose (amber)
for the new overlays.
Note: Orbitron font and app icon (also Tier 2 items) deferred — no
clean source available for either without manual work.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
658 lines
26 KiB
Swift
658 lines
26 KiB
Swift
import Foundation
|
|
import Vision
|
|
import CoreML
|
|
import CoreVideo
|
|
import CoreGraphics
|
|
import os
|
|
|
|
struct Detection: Sendable {
|
|
var rect: CGRect // normalized image coords, Vision convention (origin bottom-left)
|
|
var label: String
|
|
var confidence: Float
|
|
var trackId: UUID // stable across frames once YOLO bootstraps the track
|
|
}
|
|
|
|
/// Combined detect-and-track. YOLO produces ground-truth detections at a
|
|
/// configurable cadence (`Theme.Performance.yoloDetectionHz`, default 5 Hz);
|
|
/// between YOLO runs, `VNTrackObjectRequest` updates each active track's box
|
|
/// from the latest frame so the rendered overlay tracks objects smoothly.
|
|
final class ObjectDetector: @unchecked Sendable {
|
|
private final class TrackedObject {
|
|
let id: UUID
|
|
var label: String
|
|
var confidence: Float
|
|
var observation: VNDetectedObjectObservation
|
|
var lastRefreshedAt: CFAbsoluteTime
|
|
|
|
init(label: String, confidence: Float, observation: VNDetectedObjectObservation, now: CFAbsoluteTime) {
|
|
self.id = UUID()
|
|
self.label = label
|
|
self.confidence = confidence
|
|
self.observation = observation
|
|
self.lastRefreshedAt = now
|
|
}
|
|
}
|
|
|
|
private let queue = DispatchQueue(label: "com.dmdbrands.VisionPOC.detector", qos: .userInitiated)
|
|
private let trackingHandler = VNSequenceRequestHandler()
|
|
let faceRegistry = FaceRegistry()
|
|
|
|
/// State for the in-flight "capture next N face prints for enrollment" flow.
|
|
/// Accessed exclusively from `queue`.
|
|
private var pendingEnrollment: (remaining: Int, collected: [VNFeaturePrintObservation], completion: @Sendable ([VNFeaturePrintObservation]) -> Void)?
|
|
|
|
private var lock = os_unfair_lock_s()
|
|
private var _currentDetections: [Detection] = []
|
|
private var _currentTextDetections: [TextDetection] = []
|
|
private var _currentPoses: [PoseDetection] = []
|
|
private var _lastDetectMode: DetectMode = .yolo
|
|
private var _lastInferenceMs: Double = 0
|
|
private var _inFlight = false
|
|
|
|
/// Mutated exclusively from `queue`. No lock needed inside the serial queue.
|
|
private var tracks: [TrackedObject] = []
|
|
private var lastYoloAt: CFAbsoluteTime = 0
|
|
private var lastStatsLogAt: CFAbsoluteTime = 0
|
|
|
|
private let yoloRequest: VNCoreMLRequest?
|
|
private let modelLoadFailure: String?
|
|
|
|
init() {
|
|
let result = ObjectDetector.makeYoloRequest(modelName: Theme.Performance.detectorModelName)
|
|
self.yoloRequest = result.request
|
|
self.modelLoadFailure = result.failureReason
|
|
if let failure = modelLoadFailure {
|
|
LogStream.shared.log(failure + " — falling back to Vision built-ins only.", level: .warn, source: .detect)
|
|
} else {
|
|
LogStream.shared.log("loaded \(Theme.Performance.detectorModelName), tracker enabled (~5 Hz YOLO + ~60 Hz tracker)", level: .info, source: .detect)
|
|
}
|
|
|
|
// Warm up the YOLO model on the detector queue so the first user-visible
|
|
// detection doesn't pay ~200ms of CoreML/ANE compile latency. Dispatched
|
|
// async so init() returns immediately and app launch isn't blocked.
|
|
queue.async { [weak self] in
|
|
self?.warmUpYolo()
|
|
}
|
|
}
|
|
|
|
private func warmUpYolo() {
|
|
guard let yolo = yoloRequest else { return }
|
|
let start = CFAbsoluteTimeGetCurrent()
|
|
|
|
// 416x416 mid-gray bitmap — YOLO's typical input size. The model will
|
|
// rescale internally if needed; what matters is the CoreML graph gets
|
|
// compiled and the ANE picks up the weights.
|
|
let width = 416
|
|
let height = 416
|
|
let bytesPerRow = width * 4
|
|
let colorSpace = CGColorSpaceCreateDeviceRGB()
|
|
guard let ctx = CGContext(
|
|
data: nil,
|
|
width: width,
|
|
height: height,
|
|
bitsPerComponent: 8,
|
|
bytesPerRow: bytesPerRow,
|
|
space: colorSpace,
|
|
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
|
|
) else {
|
|
LogStream.shared.log("yolo warmup skipped: bitmap context alloc failed", level: .warn, source: .detect)
|
|
return
|
|
}
|
|
ctx.setFillColor(CGColor(red: 0.5, green: 0.5, blue: 0.5, alpha: 1.0))
|
|
ctx.fill(CGRect(x: 0, y: 0, width: width, height: height))
|
|
guard let cgImage = ctx.makeImage() else {
|
|
LogStream.shared.log("yolo warmup skipped: cgImage creation failed", level: .warn, source: .detect)
|
|
return
|
|
}
|
|
|
|
let handler = VNImageRequestHandler(cgImage: cgImage, orientation: .up, options: [:])
|
|
do {
|
|
try handler.perform([yolo])
|
|
let ms = (CFAbsoluteTimeGetCurrent() - start) * 1000.0
|
|
LogStream.shared.log("warmed up YOLO in \(String(format: "%.0f", ms))ms", level: .debug, source: .detect)
|
|
} catch {
|
|
LogStream.shared.log("yolo warmup failed: \(error)", level: .warn, source: .detect)
|
|
}
|
|
}
|
|
|
|
private static func makeYoloRequest(modelName: String) -> (request: VNCoreMLRequest?, failureReason: String?) {
|
|
guard let compiledURL = Bundle.main.url(forResource: modelName, withExtension: "mlmodelc")
|
|
?? Bundle.main.url(forResource: modelName, withExtension: "mlpackage")
|
|
else {
|
|
return (nil, "Model \(modelName) not found in app bundle")
|
|
}
|
|
do {
|
|
let config = MLModelConfiguration()
|
|
config.computeUnits = .all
|
|
let mlModel = try MLModel(contentsOf: compiledURL, configuration: config)
|
|
let vnModel = try VNCoreMLModel(for: mlModel)
|
|
let request = VNCoreMLRequest(model: vnModel)
|
|
request.imageCropAndScaleOption = .scaleFill
|
|
return (request, nil)
|
|
} catch {
|
|
return (nil, "Failed to load \(modelName): \(error)")
|
|
}
|
|
}
|
|
|
|
var lastInferenceMs: Double {
|
|
os_unfair_lock_lock(&lock)
|
|
defer { os_unfair_lock_unlock(&lock) }
|
|
return _lastInferenceMs
|
|
}
|
|
|
|
func currentDetections() -> [Detection] {
|
|
os_unfair_lock_lock(&lock)
|
|
defer { os_unfair_lock_unlock(&lock) }
|
|
return _currentDetections
|
|
}
|
|
|
|
func currentTextDetections() -> [TextDetection] {
|
|
os_unfair_lock_lock(&lock)
|
|
defer { os_unfair_lock_unlock(&lock) }
|
|
return _currentTextDetections
|
|
}
|
|
|
|
func currentPoses() -> [PoseDetection] {
|
|
os_unfair_lock_lock(&lock)
|
|
defer { os_unfair_lock_unlock(&lock) }
|
|
return _currentPoses
|
|
}
|
|
|
|
var lastDetectMode: DetectMode {
|
|
os_unfair_lock_lock(&lock)
|
|
defer { os_unfair_lock_unlock(&lock) }
|
|
return _lastDetectMode
|
|
}
|
|
|
|
func submit(pixelBuffer: CVPixelBuffer) {
|
|
os_unfair_lock_lock(&lock)
|
|
if _inFlight {
|
|
os_unfair_lock_unlock(&lock)
|
|
return
|
|
}
|
|
_inFlight = true
|
|
os_unfair_lock_unlock(&lock)
|
|
|
|
nonisolated(unsafe) let buffer = pixelBuffer
|
|
queue.async { [weak self] in
|
|
self?.run(pixelBuffer: buffer)
|
|
}
|
|
}
|
|
|
|
// MARK: - Pipeline
|
|
|
|
private func run(pixelBuffer: CVPixelBuffer) {
|
|
let start = CFAbsoluteTimeGetCurrent()
|
|
let yoloInterval = 1.0 / max(0.5, Theme.Performance.yoloDetectionHz)
|
|
let shouldDetect = (start - lastYoloAt) >= yoloInterval
|
|
|
|
var newTextDetections: [TextDetection] = []
|
|
var newPoses: [PoseDetection] = []
|
|
let detectMode: DetectMode
|
|
|
|
if shouldDetect {
|
|
let extras = detectAndBootstrap(pixelBuffer: pixelBuffer, now: start)
|
|
newTextDetections = extras.text
|
|
newPoses = extras.poses
|
|
lastYoloAt = start
|
|
detectMode = .yolo
|
|
} else {
|
|
trackOnly(pixelBuffer: pixelBuffer, now: start)
|
|
detectMode = .track
|
|
}
|
|
|
|
// Drop tracks that have aged out (no YOLO refresh recently).
|
|
let maxAge = Theme.Performance.trackMaxAgeSeconds
|
|
let expired = tracks.filter { (start - $0.lastRefreshedAt) > maxAge }
|
|
for track in expired {
|
|
LogStream.shared.log("lost \(track.label.lowercased()) (id \(track.id.uuidString.prefix(4)))",
|
|
level: .info, source: .track)
|
|
}
|
|
tracks.removeAll { (start - $0.lastRefreshedAt) > maxAge }
|
|
|
|
// Periodic stats — once a second.
|
|
if (start - lastStatsLogAt) >= 1.0 {
|
|
lastStatsLogAt = start
|
|
let yoloMs = (CFAbsoluteTimeGetCurrent() - start) * 1000.0
|
|
let labels = tracks.map { $0.label.lowercased() }.sorted()
|
|
let summary = labels.isEmpty ? "no objects" : labels.joined(separator: ", ")
|
|
let kind = shouldDetect ? "yolo" : "track"
|
|
LogStream.shared.log("\(kind) \(String(format: "%.1fms", yoloMs)) — \(tracks.count) track\(tracks.count == 1 ? "" : "s"): \(summary)",
|
|
level: .debug, source: .detect)
|
|
}
|
|
|
|
let elapsedMs = (CFAbsoluteTimeGetCurrent() - start) * 1000.0
|
|
let snapshot = tracks.map {
|
|
Detection(
|
|
rect: $0.observation.boundingBox,
|
|
label: $0.label,
|
|
confidence: $0.confidence,
|
|
trackId: $0.id
|
|
)
|
|
}
|
|
|
|
os_unfair_lock_lock(&lock)
|
|
_currentDetections = snapshot
|
|
if shouldDetect {
|
|
// Only refresh text & poses on YOLO-cadence frames; tracker frames
|
|
// keep the most recent set so the renderer doesn't flicker between
|
|
// populated and empty in the ~50ms between YOLO refreshes.
|
|
_currentTextDetections = newTextDetections
|
|
_currentPoses = newPoses
|
|
}
|
|
_lastDetectMode = detectMode
|
|
_lastInferenceMs = elapsedMs
|
|
_inFlight = false
|
|
os_unfair_lock_unlock(&lock)
|
|
}
|
|
|
|
// MARK: - YOLO + supplementary detectors
|
|
|
|
private func detectAndBootstrap(pixelBuffer: CVPixelBuffer, now: CFAbsoluteTime) -> (text: [TextDetection], poses: [PoseDetection]) {
|
|
let faces = VNDetectFaceRectanglesRequest()
|
|
let animals = VNRecognizeAnimalsRequest()
|
|
|
|
let text = VNRecognizeTextRequest()
|
|
text.recognitionLevel = .accurate
|
|
text.usesLanguageCorrection = true
|
|
|
|
let bodyPose = VNDetectHumanBodyPoseRequest()
|
|
let handPose = VNDetectHumanHandPoseRequest()
|
|
handPose.maximumHandCount = 2
|
|
|
|
var requests: [VNRequest] = [faces, animals, text, bodyPose, handPose]
|
|
if let yolo = yoloRequest {
|
|
requests.append(yolo)
|
|
}
|
|
|
|
let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, orientation: .up, options: [:])
|
|
do {
|
|
try handler.perform(requests)
|
|
} catch {
|
|
NSLog("Vision perform error: \(error)")
|
|
}
|
|
|
|
// Generate feature prints for the face observations we just collected.
|
|
// We re-run a second Vision pass scoped to those face rects so the
|
|
// FeaturePrint network only processes the cropped face regions — ~5ms
|
|
// per face on the ANE, negligible.
|
|
let facePrints: [VNFeaturePrintObservation] = computeFacePrints(
|
|
for: faces.results ?? [],
|
|
pixelBuffer: pixelBuffer
|
|
)
|
|
|
|
// If an enrollment capture is pending, feed it the largest face's
|
|
// print from this frame.
|
|
consumeEnrollmentIfNeeded(faces: faces.results ?? [], prints: facePrints)
|
|
|
|
var newDetections: [(rect: CGRect, label: String, confidence: Float)] = []
|
|
|
|
let minConfidence = Theme.Performance.detectionMinConfidence
|
|
if let yoloResults = yoloRequest?.results as? [VNRecognizedObjectObservation] {
|
|
for obs in yoloResults {
|
|
let top = obs.labels.first
|
|
let confidence = top?.confidence ?? obs.confidence
|
|
if confidence < minConfidence { continue }
|
|
let label = (top?.identifier ?? "OBJECT").uppercased()
|
|
newDetections.append((obs.boundingBox, label, confidence))
|
|
}
|
|
}
|
|
if let faceResults = faces.results {
|
|
for (i, obs) in faceResults.enumerated() {
|
|
// Try to recognize the face against the registry. The prints
|
|
// array is index-aligned with faceResults from Vision's order.
|
|
var label = "FACE"
|
|
if i < facePrints.count,
|
|
let match = faceRegistry.bestMatch(for: facePrints[i]) {
|
|
label = match.name.uppercased()
|
|
LogStream.shared.log("matched \(match.name) (dist \(String(format: "%.1f", match.distance)))",
|
|
level: .debug, source: .face)
|
|
}
|
|
newDetections.append((obs.boundingBox, label, obs.confidence))
|
|
}
|
|
}
|
|
if let animalResults = animals.results {
|
|
for obs in animalResults {
|
|
let label = (obs.labels.first?.identifier ?? "ANIMAL").uppercased()
|
|
newDetections.append((obs.boundingBox, label, obs.confidence))
|
|
}
|
|
}
|
|
|
|
// Match each new detection against existing tracks. Same-label matches
|
|
// are preferred over cross-label matches so a YOLO "PERSON" doesn't
|
|
// accidentally inherit the label from a partially-overlapping "FACE"
|
|
// track. If nothing matches, bootstrap a fresh track.
|
|
var matchedTrackIDs = Set<UUID>()
|
|
for det in newDetections {
|
|
let matchIdx = bestMatchIndex(for: det.rect, label: det.label, excluding: matchedTrackIDs)
|
|
if let idx = matchIdx {
|
|
let track = tracks[idx]
|
|
track.label = det.label
|
|
track.confidence = det.confidence
|
|
track.observation = VNDetectedObjectObservation(boundingBox: det.rect)
|
|
track.lastRefreshedAt = now
|
|
matchedTrackIDs.insert(track.id)
|
|
} else {
|
|
let obs = VNDetectedObjectObservation(boundingBox: det.rect)
|
|
let track = TrackedObject(label: det.label, confidence: det.confidence, observation: obs, now: now)
|
|
tracks.append(track)
|
|
matchedTrackIDs.insert(track.id)
|
|
LogStream.shared.log("new \(det.label.lowercased()) (id \(track.id.uuidString.prefix(4)), conf \(String(format: "%.2f", det.confidence)))",
|
|
level: .info, source: .track)
|
|
}
|
|
}
|
|
|
|
// ---- Text recognition ----
|
|
let textDetections: [TextDetection] = (text.results ?? []).compactMap { obs in
|
|
guard let top = obs.topCandidates(1).first else { return nil }
|
|
return TextDetection(rect: obs.boundingBox, text: top.string, confidence: top.confidence)
|
|
}
|
|
|
|
// ---- Body pose ----
|
|
var bodyPoses: [PoseDetection] = []
|
|
if let bodyResults = bodyPose.results {
|
|
for obs in bodyResults {
|
|
if let pose = makeBodyPose(from: obs) {
|
|
bodyPoses.append(pose)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- Hand pose ----
|
|
var handPoses: [PoseDetection] = []
|
|
if let handResults = handPose.results {
|
|
for obs in handResults {
|
|
if let pose = makeHandPose(from: obs) {
|
|
handPoses.append(pose)
|
|
}
|
|
}
|
|
}
|
|
|
|
return (text: textDetections, poses: bodyPoses + handPoses)
|
|
}
|
|
|
|
// MARK: - Pose builders
|
|
|
|
private static let poseConfidenceThreshold: Float = 0.3
|
|
|
|
private func makeBodyPose(from obs: VNHumanBodyPoseObservation) -> PoseDetection? {
|
|
// Apple's SDK joint names. We map each to its CGPoint via
|
|
// recognizedPoint(_:); points with confidence > threshold are kept.
|
|
typealias J = VNHumanBodyPoseObservation.JointName
|
|
let connections: [(J, J)] = [
|
|
(.nose, .neck),
|
|
(.neck, .leftShoulder),
|
|
(.leftShoulder, .leftElbow),
|
|
(.leftElbow, .leftWrist),
|
|
(.neck, .rightShoulder),
|
|
(.rightShoulder, .rightElbow),
|
|
(.rightElbow, .rightWrist),
|
|
(.neck, .root),
|
|
(.root, .leftHip),
|
|
(.leftHip, .leftKnee),
|
|
(.leftKnee, .leftAnkle),
|
|
(.root, .rightHip),
|
|
(.rightHip, .rightKnee),
|
|
(.rightKnee, .rightAnkle)
|
|
]
|
|
|
|
var segments: [PoseDetection.Segment] = []
|
|
// Use a small key->point map keyed by joint rawValue so points stay
|
|
// unique even if a joint appears in multiple connections.
|
|
var uniquePoints: [String: CGPoint] = [:]
|
|
var maxConf: Float = 0
|
|
|
|
for (a, b) in connections {
|
|
guard
|
|
let p1 = try? obs.recognizedPoint(a),
|
|
let p2 = try? obs.recognizedPoint(b)
|
|
else { continue }
|
|
if p1.confidence > ObjectDetector.poseConfidenceThreshold {
|
|
uniquePoints[a.rawValue.rawValue] = p1.location
|
|
maxConf = max(maxConf, p1.confidence)
|
|
}
|
|
if p2.confidence > ObjectDetector.poseConfidenceThreshold {
|
|
uniquePoints[b.rawValue.rawValue] = p2.location
|
|
maxConf = max(maxConf, p2.confidence)
|
|
}
|
|
if p1.confidence > ObjectDetector.poseConfidenceThreshold,
|
|
p2.confidence > ObjectDetector.poseConfidenceThreshold {
|
|
segments.append(PoseDetection.Segment(start: p1.location, end: p2.location))
|
|
}
|
|
}
|
|
|
|
if segments.isEmpty && uniquePoints.isEmpty { return nil }
|
|
return PoseDetection(
|
|
segments: segments,
|
|
points: Array(uniquePoints.values),
|
|
kind: .body,
|
|
confidence: maxConf
|
|
)
|
|
}
|
|
|
|
private func makeHandPose(from obs: VNHumanHandPoseObservation) -> PoseDetection? {
|
|
typealias J = VNHumanHandPoseObservation.JointName
|
|
|
|
// Each finger chain: tip -> middle phalange -> proximal -> metacarpal/base.
|
|
// Apple's hand-pose joint names (verified against the macOS SDK):
|
|
// thumb: thumbTip, thumbIP, thumbMP, thumbCMC
|
|
// index: indexTip, indexDIP, indexPIP, indexMCP
|
|
// middle: middleTip, middleDIP,middlePIP,middleMCP
|
|
// ring: ringTip, ringDIP, ringPIP, ringMCP
|
|
// little: littleTip, littleDIP,littlePIP,littleMCP
|
|
let fingerChains: [[J]] = [
|
|
[.thumbTip, .thumbIP, .thumbMP, .thumbCMC],
|
|
[.indexTip, .indexDIP, .indexPIP, .indexMCP],
|
|
[.middleTip, .middleDIP, .middlePIP, .middleMCP],
|
|
[.ringTip, .ringDIP, .ringPIP, .ringMCP],
|
|
[.littleTip, .littleDIP, .littlePIP, .littleMCP]
|
|
]
|
|
// Palm ring: connect the 5 base/knuckle joints around the palm.
|
|
// Start at thumb base (CMC) and walk index→middle→ring→little MCPs,
|
|
// then close the ring back to the thumb base.
|
|
let palmRing: [J] = [.thumbCMC, .indexMCP, .middleMCP, .ringMCP, .littleMCP, .thumbCMC]
|
|
|
|
var segments: [PoseDetection.Segment] = []
|
|
var uniquePoints: [String: CGPoint] = [:]
|
|
var maxConf: Float = 0
|
|
|
|
func addPoint(_ name: J) -> (location: CGPoint, ok: Bool) {
|
|
guard let p = try? obs.recognizedPoint(name) else { return (.zero, false) }
|
|
if p.confidence > ObjectDetector.poseConfidenceThreshold {
|
|
uniquePoints[name.rawValue.rawValue] = p.location
|
|
maxConf = max(maxConf, p.confidence)
|
|
return (p.location, true)
|
|
}
|
|
return (.zero, false)
|
|
}
|
|
|
|
for chain in fingerChains {
|
|
for i in 0..<(chain.count - 1) {
|
|
let a = addPoint(chain[i])
|
|
let b = addPoint(chain[i + 1])
|
|
if a.ok && b.ok {
|
|
segments.append(PoseDetection.Segment(start: a.location, end: b.location))
|
|
}
|
|
}
|
|
}
|
|
|
|
for i in 0..<(palmRing.count - 1) {
|
|
let a = addPoint(palmRing[i])
|
|
let b = addPoint(palmRing[i + 1])
|
|
if a.ok && b.ok {
|
|
segments.append(PoseDetection.Segment(start: a.location, end: b.location))
|
|
}
|
|
}
|
|
|
|
if segments.isEmpty && uniquePoints.isEmpty { return nil }
|
|
return PoseDetection(
|
|
segments: segments,
|
|
points: Array(uniquePoints.values),
|
|
kind: .hand,
|
|
confidence: maxConf
|
|
)
|
|
}
|
|
|
|
// MARK: - Tracker-only update
|
|
|
|
private func trackOnly(pixelBuffer: CVPixelBuffer, now: CFAbsoluteTime) {
|
|
guard !tracks.isEmpty else { return }
|
|
|
|
// Build one tracking request per active track. Apple's tracker can
|
|
// handle a handful of objects in a single perform() call without
|
|
// measurable extra latency.
|
|
let trackRequests = tracks.map { track -> VNTrackObjectRequest in
|
|
let req = VNTrackObjectRequest(detectedObjectObservation: track.observation)
|
|
req.trackingLevel = .accurate
|
|
return req
|
|
}
|
|
|
|
do {
|
|
try trackingHandler.perform(trackRequests, on: pixelBuffer, orientation: .up)
|
|
} catch {
|
|
NSLog("Tracking perform error: \(error)")
|
|
return
|
|
}
|
|
|
|
let minConfidence = Theme.Performance.trackerMinConfidence
|
|
for (idx, req) in trackRequests.enumerated() where idx < tracks.count {
|
|
guard let updated = req.results?.first as? VNDetectedObjectObservation else { continue }
|
|
if updated.confidence < minConfidence {
|
|
// Tracker has lost the object; keep the old observation and
|
|
// let the age-out timer drop the track if YOLO doesn't see it
|
|
// again soon.
|
|
continue
|
|
}
|
|
tracks[idx].observation = updated
|
|
}
|
|
}
|
|
|
|
// MARK: - Matching
|
|
|
|
private func bestMatchIndex(for rect: CGRect, label: String, excluding excluded: Set<UUID>) -> Int? {
|
|
let threshold = CGFloat(Theme.Performance.trackMatchIoU)
|
|
var bestIdx: Int?
|
|
var bestScore: CGFloat = threshold
|
|
|
|
for (idx, track) in tracks.enumerated() where !excluded.contains(track.id) {
|
|
let score = iou(rect, track.observation.boundingBox)
|
|
// Prefer same-label matches by giving them a small score boost.
|
|
let adjusted: CGFloat = (track.label == label) ? score + 0.05 : score
|
|
if adjusted > bestScore {
|
|
bestScore = adjusted
|
|
bestIdx = idx
|
|
}
|
|
}
|
|
return bestIdx
|
|
}
|
|
|
|
// MARK: - Face feature prints & enrollment
|
|
|
|
/// Generates a per-face feature print by running `VNGenerateImageFeaturePrintRequest`
|
|
/// on a region-of-interest crop for each face. Vision's public API doesn't
|
|
/// expose a dedicated face-embedding model, but the general-purpose image
|
|
/// FeaturePrint applied to a tight face crop produces distances that
|
|
/// reliably separate "same person" from "different person" for the small
|
|
/// number of enrolled identities typical of a POC.
|
|
private func computeFacePrints(
|
|
for faces: [VNFaceObservation],
|
|
pixelBuffer: CVPixelBuffer
|
|
) -> [VNFeaturePrintObservation] {
|
|
guard !faces.isEmpty else { return [] }
|
|
let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, orientation: .up, options: [:])
|
|
|
|
var requests: [VNGenerateImageFeaturePrintRequest] = []
|
|
requests.reserveCapacity(faces.count)
|
|
for face in faces {
|
|
let req = VNGenerateImageFeaturePrintRequest()
|
|
// Slightly pad the face rect so we capture some surrounding context
|
|
// (hairline, jaw). Clamp to [0,1].
|
|
req.regionOfInterest = padFaceRect(face.boundingBox)
|
|
requests.append(req)
|
|
}
|
|
|
|
do {
|
|
try handler.perform(requests)
|
|
} catch {
|
|
NSLog("Face FeaturePrint perform error: \(error)")
|
|
return []
|
|
}
|
|
|
|
var out: [VNFeaturePrintObservation] = []
|
|
for req in requests {
|
|
if let obs = req.results?.first as? VNFeaturePrintObservation {
|
|
out.append(obs)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
private func padFaceRect(_ r: CGRect) -> CGRect {
|
|
let pad: CGFloat = 0.08
|
|
let x = max(0, r.origin.x - r.width * pad)
|
|
let y = max(0, r.origin.y - r.height * pad)
|
|
let w = min(1 - x, r.width * (1 + 2 * pad))
|
|
let h = min(1 - y, r.height * (1 + 2 * pad))
|
|
return CGRect(x: x, y: y, width: w, height: h)
|
|
}
|
|
|
|
private func consumeEnrollmentIfNeeded(
|
|
faces: [VNFaceObservation],
|
|
prints: [VNFeaturePrintObservation]
|
|
) {
|
|
guard var enrollment = pendingEnrollment else { return }
|
|
guard !faces.isEmpty, !prints.isEmpty else {
|
|
// Wait for a frame that actually contains a face.
|
|
return
|
|
}
|
|
|
|
// Use the largest face by bounding-box area — most likely to be the
|
|
// person actively enrolling rather than a bystander in the background.
|
|
let pairs = zip(faces, prints)
|
|
let chosen = pairs.max(by: { lhs, rhs in
|
|
let la = lhs.0.boundingBox.width * lhs.0.boundingBox.height
|
|
let ra = rhs.0.boundingBox.width * rhs.0.boundingBox.height
|
|
return la < ra
|
|
})
|
|
guard let (_, print) = chosen else { return }
|
|
|
|
enrollment.collected.append(print)
|
|
enrollment.remaining -= 1
|
|
|
|
if enrollment.remaining <= 0 {
|
|
let captured = enrollment.collected
|
|
let completion = enrollment.completion
|
|
pendingEnrollment = nil
|
|
completion(captured)
|
|
} else {
|
|
pendingEnrollment = enrollment
|
|
}
|
|
}
|
|
|
|
/// Captures `count` feature prints from upcoming detection frames and
|
|
/// delivers them to `completion` on an arbitrary queue. The caller is
|
|
/// responsible for hopping back to main if it touches UI.
|
|
func captureFeaturePrints(count: Int, completion: @escaping @Sendable ([VNFeaturePrintObservation]) -> Void) {
|
|
let n = max(1, count)
|
|
queue.async { [weak self] in
|
|
guard let self else { return }
|
|
self.pendingEnrollment = (remaining: n, collected: [], completion: completion)
|
|
}
|
|
}
|
|
|
|
func cancelEnrollment() {
|
|
queue.async { [weak self] in
|
|
self?.pendingEnrollment = nil
|
|
}
|
|
}
|
|
|
|
private func iou(_ a: CGRect, _ b: CGRect) -> CGFloat {
|
|
let inter = a.intersection(b)
|
|
if inter.isNull || inter.isEmpty { return 0 }
|
|
let interArea = inter.width * inter.height
|
|
let union = a.width * a.height + b.width * b.height - interArea
|
|
guard union > 0 else { return 0 }
|
|
return interArea / union
|
|
}
|
|
}
|