Add YOLOv3 object detection (80 COCO classes) via Git LFS

Drops Apple's full YOLOv3 (~248 MB) into Sources/Resources/Models/ as the
primary object detector. Xcode auto-compiles the .mlmodel into a .mlmodelc
inside the app bundle; the model is a Vision-friendly pipeline so
VNCoreMLRequest returns VNRecognizedObjectObservation directly — no manual
NMS or output decoding required.

Detection pipeline
- YOLOv3 is the primary detector, returning bounding boxes labeled with
  COCO classes: person, bicycle, car, ..., bottle, wine glass, cup, fork,
  knife, spoon, bowl, ... laptop, mouse, remote, keyboard, cell phone, ...
  book, clock, vase. 80 classes total — covers the "is that a coffee mug"
  question and many others.
- VNDetectFaceRectanglesRequest stays as a supplementary FACE labeler
  (more precise than YOLO's "person" for the face region).
- VNRecognizeAnimalsRequest stays for finer-grained cat/dog labels.
- Saliency and human-rectangle requests removed — YOLO supersedes them.
- Per-label NMS with IoU>0.5 inside filterAndMerge drops same-label
  overlaps. Cross-label overlaps (e.g. person + face) are kept since they
  carry different information.
- Min-confidence filter (Theme.Performance.detectionMinConfidence, 0.30)
  prunes weak guesses.

Distribution
- Model is tracked via Git LFS (.gitattributes) — it exceeds GitHub's
  100 MB per-file limit.
- README documents the `git lfs install && git lfs pull` prerequisite.
- scripts/fetch-model.sh remains as a no-LFS fallback that downloads the
  same model from Apple's Core ML catalog.

Theme knobs
- Performance.detectionMinConfidence (default 0.30)
- Performance.detectorModelName (default "YOLOv3")

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

.gitattributes

@@ -0,0 +1,2 @@
+Sources/Resources/Models/*.mlmodel filter=lfs diff=lfs merge=lfs -text
+Sources/Resources/Models/*.mlpackage filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -7,3 +7,5 @@ xcuserdata/
 
 # Regenerated from project.yml — do not commit
 VisionPOC.xcodeproj/
+
+# Models are stored via Git LFS (see .gitattributes). Nothing to ignore here.

README.md

@@ -23,14 +23,19 @@ Sibling to [MetalPOC](https://github.com/KofTwentyTwo/MetalPOC) (Jarvis HUD rend
 
 ## Build & Run
 
+Prerequisite: [Git LFS](https://git-lfs.com) is required because the bundled YOLOv3 model (~248 MB) is tracked via LFS. Install once on your machine: `brew install git-lfs && git lfs install`.
+
 ```bash
 git clone https://github.com/KofTwentyTwo/VisionPOC.git
 cd VisionPOC
+git lfs pull                                  # ensure the YOLOv3 model is materialized
 xcodegen generate
 xcodebuild -project VisionPOC.xcodeproj -scheme VisionPOC -configuration Debug build
 open build/Debug/VisionPOC.app
 ```
 
+If you cloned without LFS installed, run `git lfs install && git lfs pull` and the model will be fetched into `Sources/Resources/Models/`. A fallback script (`scripts/fetch-model.sh`) is also provided for environments where LFS isn't available — it pulls the same model directly from Apple's Core ML catalog.
+
 On first launch, macOS will prompt for camera access. Grant it and the four-pane grid will activate.
 
 ## Architecture

Sources/Process/ObjectDetector.swift

@@ -1,5 +1,6 @@
 import Foundation
 import Vision
+import CoreML
 import CoreVideo
 import CoreGraphics
 import os
@@ -19,6 +20,39 @@ final class ObjectDetector: @unchecked Sendable {
     private var _inFlight = false
     private var _lastFinishedAt: CFAbsoluteTime = 0
 
+    private let yoloRequest: VNCoreMLRequest?
+    private let modelLoadFailure: String?
+
+    init() {
+        let result = ObjectDetector.makeYoloRequest(modelName: Theme.Performance.detectorModelName)
+        self.yoloRequest = result.request
+        self.modelLoadFailure = result.failureReason
+        if let failure = modelLoadFailure {
+            NSLog("ObjectDetector: \(failure) — falling back to Vision built-ins only.")
+        } else {
+            NSLog("ObjectDetector: loaded \(Theme.Performance.detectorModelName) (\(yoloRequest != nil ? "ANE/GPU" : "n/a")).")
+        }
+    }
+
+    private static func makeYoloRequest(modelName: String) -> (request: VNCoreMLRequest?, failureReason: String?) {
+        guard let compiledURL = Bundle.main.url(forResource: modelName, withExtension: "mlmodelc")
+            ?? Bundle.main.url(forResource: modelName, withExtension: "mlpackage")
+        else {
+            return (nil, "Model \(modelName) not found in app bundle")
+        }
+        do {
+            let config = MLModelConfiguration()
+            config.computeUnits = .all
+            let mlModel = try MLModel(contentsOf: compiledURL, configuration: config)
+            let vnModel = try VNCoreMLModel(for: mlModel)
+            let request = VNCoreMLRequest(model: vnModel)
+            request.imageCropAndScaleOption = .scaleFill
+            return (request, nil)
+        } catch {
+            return (nil, "Failed to load \(modelName): \(error)")
+        }
+    }
+
     var lastInferenceMs: Double {
         os_unfair_lock_lock(&lock)
         defer { os_unfair_lock_unlock(&lock) }
@@ -43,8 +77,6 @@ final class ObjectDetector: @unchecked Sendable {
         _inFlight = true
         os_unfair_lock_unlock(&lock)
 
-        // CVPixelBuffer is not Sendable but we only read from it on the detector queue
-        // and AVFoundation has already handed it off to us — safe to capture.
         nonisolated(unsafe) let buffer = pixelBuffer
         queue.async { [weak self] in
             self?.run(pixelBuffer: buffer)
@@ -54,45 +86,52 @@ final class ObjectDetector: @unchecked Sendable {
     private func run(pixelBuffer: CVPixelBuffer) {
         let start = CFAbsoluteTimeGetCurrent()
 
-        let saliency = VNGenerateObjectnessBasedSaliencyImageRequest()
         let faces = VNDetectFaceRectanglesRequest()
-        let humans = VNDetectHumanRectanglesRequest()
         let animals = VNRecognizeAnimalsRequest()
+        var requests: [VNRequest] = [faces, animals]
+        if let yolo = yoloRequest {
+            requests.append(yolo)
+        }
 
-        // Vision picks the optimal compute unit (ANE on Apple Silicon) by default.
         let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, orientation: .up, options: [:])
         var collected: [Detection] = []
 
         do {
-            try handler.perform([saliency, faces, humans, animals])
+            try handler.perform(requests)
         } catch {
             NSLog("Vision perform error: \(error)")
         }
 
-        if let salient = (saliency.results?.first as? VNSaliencyImageObservation)?.salientObjects {
-            for obj in salient {
-                collected.append(Detection(rect: obj.boundingBox, label: "OBJECT", confidence: obj.confidence))
+        if let yoloResults = yoloRequest?.results as? [VNRecognizedObjectObservation] {
+            let minConfidence = Theme.Performance.detectionMinConfidence
+            for obs in yoloResults {
+                let top = obs.labels.first
+                let topConfidence = top?.confidence ?? obs.confidence
+                if topConfidence < minConfidence { continue }
+                let label = (top?.identifier ?? "OBJECT").uppercased()
+                collected.append(Detection(rect: obs.boundingBox,
+                                           label: label,
+                                           confidence: topConfidence))
             }
         }
-        if let face = faces.results {
-            for obs in face {
-                collected.append(Detection(rect: obs.boundingBox, label: "FACE", confidence: obs.confidence))
-            }
-        }
-        if let human = humans.results {
-            for obs in human {
-                collected.append(Detection(rect: obs.boundingBox, label: "HUMAN", confidence: obs.confidence))
+
+        if let faceResults = faces.results {
+            for obs in faceResults {
+                collected.append(Detection(rect: obs.boundingBox,
+                                           label: "FACE",
+                                           confidence: obs.confidence))
             }
         }
         if let animalResults = animals.results {
             for obs in animalResults {
-                let label = obs.labels.first?.identifier.uppercased() ?? "ANIMAL"
-                collected.append(Detection(rect: obs.boundingBox, label: label, confidence: obs.confidence))
+                let label = (obs.labels.first?.identifier ?? "ANIMAL").uppercased()
+                collected.append(Detection(rect: obs.boundingBox,
+                                           label: label,
+                                           confidence: obs.confidence))
             }
         }
 
         let filtered = filterAndMerge(collected)
-
         let elapsed = (CFAbsoluteTimeGetCurrent() - start) * 1000.0
 
         os_unfair_lock_lock(&lock)
@@ -105,21 +144,26 @@ final class ObjectDetector: @unchecked Sendable {
 
     private func filterAndMerge(_ all: [Detection]) -> [Detection] {
         let minArea = Theme.Performance.minBoxArea
-        let sizeFiltered = all.filter { $0.rect.width * $0.rect.height >= minArea }
+        let sized = all.filter { $0.rect.width * $0.rect.height >= minArea }
 
-        let labeled = sizeFiltered.filter { $0.label != "OBJECT" }
-        let generic = sizeFiltered.filter { $0.label == "OBJECT" }
-
-        let prunedGeneric = generic.filter { g in
-            !labeled.contains(where: { iou($0.rect, g.rect) > 0.5 })
+        // Suppress strongly-overlapping detections that share the same label.
+        // YOLO already applies NMS internally so cross-class duplicates are rare;
+        // a small same-label pass handles overlap between YOLO + face/animal.
+        let byLabel = Dictionary(grouping: sized, by: { $0.label })
+        var kept: [Detection] = []
+        for (_, group) in byLabel {
+            let sortedGroup = group.sorted { $0.confidence > $1.confidence }
+            var accepted: [Detection] = []
+            for det in sortedGroup {
+                if accepted.allSatisfy({ iou($0.rect, det.rect) < 0.5 }) {
+                    accepted.append(det)
+                }
+            }
+            kept.append(contentsOf: accepted)
         }
 
-        var merged = labeled + prunedGeneric
-        merged.sort { $0.confidence > $1.confidence }
-        if merged.count > Theme.Performance.maxDetectionsPerFrame {
-            merged = Array(merged.prefix(Theme.Performance.maxDetectionsPerFrame))
-        }
-        return merged
+        let sorted = kept.sorted { $0.confidence > $1.confidence }
+        return Array(sorted.prefix(Theme.Performance.maxDetectionsPerFrame))
     }
 
     private func iou(_ a: CGRect, _ b: CGRect) -> CGFloat {

Sources/Theme.swift

@@ -102,6 +102,15 @@ enum Theme {
         /// ASCII art pane: number of character cells across the pane horizontally.
         /// The vertical cell count is derived to match the source aspect ratio.
         static let asciiColumns: Int = 120
+
+        /// Minimum top-label confidence (0..1) for a YOLO detection to be drawn.
+        /// Lower values surface more objects but include weaker guesses.
+        static let detectionMinConfidence: Float = 0.30
+
+        /// Bundled object detector model. The .mlmodel file must live under
+        /// `Sources/Resources/Models/<name>.mlmodel` so Xcode compiles it into
+        /// the app bundle as `<name>.mlmodelc`.
+        static let detectorModelName: String = "YOLOv3"
     }
 
     // MARK: - Tick

scripts/fetch-model.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Fetches the bundled object-detection model from Apple's Core ML model catalog.
+# The model is too large to commit to git (~248 MB), so we pull it on demand.
+#
+# Run once after cloning, before the first `xcodegen generate && xcodebuild`:
+#   ./scripts/fetch-model.sh
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+MODELS_DIR="$ROOT/Sources/Resources/Models"
+MODEL_NAME="YOLOv3"
+MODEL_URL="https://docs-assets.developer.apple.com/coreml/models/Image/ObjectDetection/YOLOv3/YOLOv3.mlmodel"
+DEST="$MODELS_DIR/$MODEL_NAME.mlmodel"
+
+mkdir -p "$MODELS_DIR"
+
+if [[ -f "$DEST" ]]; then
+  size=$(stat -f%z "$DEST")
+  if [[ "$size" -gt 100000000 ]]; then
+    echo "$MODEL_NAME.mlmodel already present (${size} bytes). Skipping download."
+    exit 0
+  fi
+  echo "Existing $MODEL_NAME.mlmodel looks truncated (${size} bytes); re-downloading."
+fi
+
+echo "Downloading $MODEL_NAME from Apple's Core ML catalog…"
+curl -fL --progress-bar -o "$DEST" "$MODEL_URL"
+
+actual=$(stat -f%z "$DEST")
+echo "Saved $DEST (${actual} bytes)."
+
+if [[ "$actual" -lt 100000000 ]]; then
+  echo "Download appears incomplete. Expected ~248 MB; got ${actual} bytes." >&2
+  exit 1
+fi