diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f726426 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +Sources/Resources/Models/*.mlmodel filter=lfs diff=lfs merge=lfs -text +Sources/Resources/Models/*.mlpackage filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 7f77c87..8bab860 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ xcuserdata/ # Regenerated from project.yml — do not commit VisionPOC.xcodeproj/ + +# Models are stored via Git LFS (see .gitattributes). Nothing to ignore here. diff --git a/README.md b/README.md index ac2d2bb..75f11d9 100644 --- a/README.md +++ b/README.md @@ -23,14 +23,19 @@ Sibling to [MetalPOC](https://github.com/KofTwentyTwo/MetalPOC) (Jarvis HUD rend ## Build & Run +Prerequisite: [Git LFS](https://git-lfs.com) is required because the bundled YOLOv3 model (~248 MB) is tracked via LFS. Install once on your machine: `brew install git-lfs && git lfs install`. + ```bash git clone https://github.com/KofTwentyTwo/VisionPOC.git cd VisionPOC +git lfs pull # ensure the YOLOv3 model is materialized xcodegen generate xcodebuild -project VisionPOC.xcodeproj -scheme VisionPOC -configuration Debug build open build/Debug/VisionPOC.app ``` +If you cloned without LFS installed, run `git lfs install && git lfs pull` and the model will be fetched into `Sources/Resources/Models/`. A fallback script (`scripts/fetch-model.sh`) is also provided for environments where LFS isn't available — it pulls the same model directly from Apple's Core ML catalog. + On first launch, macOS will prompt for camera access. Grant it and the four-pane grid will activate. ## Architecture diff --git a/Sources/Process/ObjectDetector.swift b/Sources/Process/ObjectDetector.swift index c28e4b3..f5b7694 100644 --- a/Sources/Process/ObjectDetector.swift +++ b/Sources/Process/ObjectDetector.swift @@ -1,5 +1,6 @@ import Foundation import Vision +import CoreML import CoreVideo import CoreGraphics import os @@ -19,6 +20,39 @@ final class ObjectDetector: @unchecked Sendable { private var _inFlight = false private var _lastFinishedAt: CFAbsoluteTime = 0 + private let yoloRequest: VNCoreMLRequest? + private let modelLoadFailure: String? + + init() { + let result = ObjectDetector.makeYoloRequest(modelName: Theme.Performance.detectorModelName) + self.yoloRequest = result.request + self.modelLoadFailure = result.failureReason + if let failure = modelLoadFailure { + NSLog("ObjectDetector: \(failure) — falling back to Vision built-ins only.") + } else { + NSLog("ObjectDetector: loaded \(Theme.Performance.detectorModelName) (\(yoloRequest != nil ? "ANE/GPU" : "n/a")).") + } + } + + private static func makeYoloRequest(modelName: String) -> (request: VNCoreMLRequest?, failureReason: String?) { + guard let compiledURL = Bundle.main.url(forResource: modelName, withExtension: "mlmodelc") + ?? Bundle.main.url(forResource: modelName, withExtension: "mlpackage") + else { + return (nil, "Model \(modelName) not found in app bundle") + } + do { + let config = MLModelConfiguration() + config.computeUnits = .all + let mlModel = try MLModel(contentsOf: compiledURL, configuration: config) + let vnModel = try VNCoreMLModel(for: mlModel) + let request = VNCoreMLRequest(model: vnModel) + request.imageCropAndScaleOption = .scaleFill + return (request, nil) + } catch { + return (nil, "Failed to load \(modelName): \(error)") + } + } + var lastInferenceMs: Double { os_unfair_lock_lock(&lock) defer { os_unfair_lock_unlock(&lock) } @@ -43,8 +77,6 @@ final class ObjectDetector: @unchecked Sendable { _inFlight = true os_unfair_lock_unlock(&lock) - // CVPixelBuffer is not Sendable but we only read from it on the detector queue - // and AVFoundation has already handed it off to us — safe to capture. nonisolated(unsafe) let buffer = pixelBuffer queue.async { [weak self] in self?.run(pixelBuffer: buffer) @@ -54,45 +86,52 @@ final class ObjectDetector: @unchecked Sendable { private func run(pixelBuffer: CVPixelBuffer) { let start = CFAbsoluteTimeGetCurrent() - let saliency = VNGenerateObjectnessBasedSaliencyImageRequest() let faces = VNDetectFaceRectanglesRequest() - let humans = VNDetectHumanRectanglesRequest() let animals = VNRecognizeAnimalsRequest() + var requests: [VNRequest] = [faces, animals] + if let yolo = yoloRequest { + requests.append(yolo) + } - // Vision picks the optimal compute unit (ANE on Apple Silicon) by default. let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, orientation: .up, options: [:]) var collected: [Detection] = [] do { - try handler.perform([saliency, faces, humans, animals]) + try handler.perform(requests) } catch { NSLog("Vision perform error: \(error)") } - if let salient = (saliency.results?.first as? VNSaliencyImageObservation)?.salientObjects { - for obj in salient { - collected.append(Detection(rect: obj.boundingBox, label: "OBJECT", confidence: obj.confidence)) + if let yoloResults = yoloRequest?.results as? [VNRecognizedObjectObservation] { + let minConfidence = Theme.Performance.detectionMinConfidence + for obs in yoloResults { + let top = obs.labels.first + let topConfidence = top?.confidence ?? obs.confidence + if topConfidence < minConfidence { continue } + let label = (top?.identifier ?? "OBJECT").uppercased() + collected.append(Detection(rect: obs.boundingBox, + label: label, + confidence: topConfidence)) } } - if let face = faces.results { - for obs in face { - collected.append(Detection(rect: obs.boundingBox, label: "FACE", confidence: obs.confidence)) - } - } - if let human = humans.results { - for obs in human { - collected.append(Detection(rect: obs.boundingBox, label: "HUMAN", confidence: obs.confidence)) + + if let faceResults = faces.results { + for obs in faceResults { + collected.append(Detection(rect: obs.boundingBox, + label: "FACE", + confidence: obs.confidence)) } } if let animalResults = animals.results { for obs in animalResults { - let label = obs.labels.first?.identifier.uppercased() ?? "ANIMAL" - collected.append(Detection(rect: obs.boundingBox, label: label, confidence: obs.confidence)) + let label = (obs.labels.first?.identifier ?? "ANIMAL").uppercased() + collected.append(Detection(rect: obs.boundingBox, + label: label, + confidence: obs.confidence)) } } let filtered = filterAndMerge(collected) - let elapsed = (CFAbsoluteTimeGetCurrent() - start) * 1000.0 os_unfair_lock_lock(&lock) @@ -105,21 +144,26 @@ final class ObjectDetector: @unchecked Sendable { private func filterAndMerge(_ all: [Detection]) -> [Detection] { let minArea = Theme.Performance.minBoxArea - let sizeFiltered = all.filter { $0.rect.width * $0.rect.height >= minArea } + let sized = all.filter { $0.rect.width * $0.rect.height >= minArea } - let labeled = sizeFiltered.filter { $0.label != "OBJECT" } - let generic = sizeFiltered.filter { $0.label == "OBJECT" } - - let prunedGeneric = generic.filter { g in - !labeled.contains(where: { iou($0.rect, g.rect) > 0.5 }) + // Suppress strongly-overlapping detections that share the same label. + // YOLO already applies NMS internally so cross-class duplicates are rare; + // a small same-label pass handles overlap between YOLO + face/animal. + let byLabel = Dictionary(grouping: sized, by: { $0.label }) + var kept: [Detection] = [] + for (_, group) in byLabel { + let sortedGroup = group.sorted { $0.confidence > $1.confidence } + var accepted: [Detection] = [] + for det in sortedGroup { + if accepted.allSatisfy({ iou($0.rect, det.rect) < 0.5 }) { + accepted.append(det) + } + } + kept.append(contentsOf: accepted) } - var merged = labeled + prunedGeneric - merged.sort { $0.confidence > $1.confidence } - if merged.count > Theme.Performance.maxDetectionsPerFrame { - merged = Array(merged.prefix(Theme.Performance.maxDetectionsPerFrame)) - } - return merged + let sorted = kept.sorted { $0.confidence > $1.confidence } + return Array(sorted.prefix(Theme.Performance.maxDetectionsPerFrame)) } private func iou(_ a: CGRect, _ b: CGRect) -> CGFloat { diff --git a/Sources/Resources/Models/YOLOv3.mlmodel b/Sources/Resources/Models/YOLOv3.mlmodel new file mode 100644 index 0000000..ca6c17c --- /dev/null +++ b/Sources/Resources/Models/YOLOv3.mlmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fea1f2b83b303e7fc6acf08c05a062793229de474587da336d8724e5bf16e38 +size 248381690 diff --git a/Sources/Theme.swift b/Sources/Theme.swift index a82c4e2..b242f18 100644 --- a/Sources/Theme.swift +++ b/Sources/Theme.swift @@ -102,6 +102,15 @@ enum Theme { /// ASCII art pane: number of character cells across the pane horizontally. /// The vertical cell count is derived to match the source aspect ratio. static let asciiColumns: Int = 120 + + /// Minimum top-label confidence (0..1) for a YOLO detection to be drawn. + /// Lower values surface more objects but include weaker guesses. + static let detectionMinConfidence: Float = 0.30 + + /// Bundled object detector model. The .mlmodel file must live under + /// `Sources/Resources/Models/.mlmodel` so Xcode compiles it into + /// the app bundle as `.mlmodelc`. + static let detectorModelName: String = "YOLOv3" } // MARK: - Tick diff --git a/scripts/fetch-model.sh b/scripts/fetch-model.sh new file mode 100755 index 0000000..1c5023d --- /dev/null +++ b/scripts/fetch-model.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Fetches the bundled object-detection model from Apple's Core ML model catalog. +# The model is too large to commit to git (~248 MB), so we pull it on demand. +# +# Run once after cloning, before the first `xcodegen generate && xcodebuild`: +# ./scripts/fetch-model.sh +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +MODELS_DIR="$ROOT/Sources/Resources/Models" +MODEL_NAME="YOLOv3" +MODEL_URL="https://docs-assets.developer.apple.com/coreml/models/Image/ObjectDetection/YOLOv3/YOLOv3.mlmodel" +DEST="$MODELS_DIR/$MODEL_NAME.mlmodel" + +mkdir -p "$MODELS_DIR" + +if [[ -f "$DEST" ]]; then + size=$(stat -f%z "$DEST") + if [[ "$size" -gt 100000000 ]]; then + echo "$MODEL_NAME.mlmodel already present (${size} bytes). Skipping download." + exit 0 + fi + echo "Existing $MODEL_NAME.mlmodel looks truncated (${size} bytes); re-downloading." +fi + +echo "Downloading $MODEL_NAME from Apple's Core ML catalog…" +curl -fL --progress-bar -o "$DEST" "$MODEL_URL" + +actual=$(stat -f%z "$DEST") +echo "Saved $DEST (${actual} bytes)." + +if [[ "$actual" -lt 100000000 ]]; then + echo "Download appears incomplete. Expected ~248 MB; got ${actual} bytes." >&2 + exit 1 +fi