fix: better detect windows from other spaces (closes #1324)

We found a workaround after more than 3 years of searching!

alt-tab-macos.xcodeproj/project.pbxproj

@@ -127,6 +127,7 @@
 		BF0C8879F979414A87AB2958 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = BF0C89348E07FC05870FEE62 /* InfoPlist.strings */; };
 		BF0C88C371B5550E7D23E4F1 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = BF0C8BF5835F97923BF3E8DA /* InfoPlist.strings */; };
 		BF0C88D7E209C788558BD943 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = BF0C8CC9692E6E4A32846D72 /* InfoPlist.strings */; };
+		BF0C88FFEE7645A1966C7C4E /* AXUIElementTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF0C855CD9DBCAEFE48F9EA5 /* AXUIElementTests.swift */; };
 		BF0C894199BB262BDB2D8D37 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = BF0C8221101CD179DB1121D5 /* InfoPlist.strings */; };
 		BF0C894325A54CF0B6B53464 /* Localizable.strings in Resources */ = {isa = PBXBuildFile; fileRef = BF0C8B8B31D2F421AC258351 /* Localizable.strings */; };
 		BF0C896B5D08B7DA8F2024AC /* KeyRepeatTimer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF0C8D606B53D315F2D9927B /* KeyRepeatTimer.swift */; };
@@ -386,6 +387,7 @@
 		BF0C84B9B7A41E4A4534E519 /* app-icons-style-full.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = "app-icons-style-full.jpg"; sourceTree = "<group>"; };
 		BF0C84E11784B3B0900D0963 /* app.icns */ = {isa = PBXFileReference; lastKnownFileType = image.icns; path = app.icns; sourceTree = "<group>"; };
 		BF0C850C46CCD8B978168CCD /* sv */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.strings; name = sv; path = Localizable.strings; sourceTree = "<group>"; };
+		BF0C855CD9DBCAEFE48F9EA5 /* AXUIElementTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AXUIElementTests.swift; sourceTree = "<group>"; };
 		BF0C85C20516C4210508F349 /* tsconfig.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = tsconfig.json; sourceTree = "<group>"; };
 		BF0C861255F42BF7B23408EF /* sq */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.strings; name = sq; path = InfoPlist.strings; sourceTree = "<group>"; };
 		BF0C863EDD60D925950B223B /* convert_iconset_to_icns.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; path = convert_iconset_to_icns.sh; sourceTree = "<group>"; };
@@ -972,6 +974,7 @@
 				BF0C899905524253DC170B56 /* KeyboardEventsTests.swift */,
 				BF0C829EAA93A4C4ADB42CF8 /* Mocks.swift */,
 				BF0C8C6810FF50908E637A48 /* ConcurrentScreenshots.swift */,
+				BF0C855CD9DBCAEFE48F9EA5 /* AXUIElementTests.swift */,
 			);
 			path = "unit-tests";
 			sourceTree = "<group>";
@@ -2242,6 +2245,7 @@
 				BF0C875A28B98904F8F4A66A /* ATShortcut.swift in Sources */,
 				BF0C84E5AC1D423625ABF3E6 /* Mocks.swift in Sources */,
 				BF0C8CBED8D88DFBADE7EC03 /* ConcurrentScreenshots.swift in Sources */,
+				BF0C88FFEE7645A1966C7C4E /* AXUIElementTests.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

src/api-wrappers/AXUIElement.swift

@@ -7,22 +7,20 @@ import ApplicationServices.HIServices.AXAttributeConstants
 import ApplicationServices.HIServices.AXActionConstants
 
 // if the window server is busy, it may not reply to AX calls. We retry right before the call times-out and returns a bogus value
-func retryAxCallUntilTimeout(_ group: DispatchGroup? = nil, _ timeoutInSeconds: Double = Double(AXUIElement.globalTimeoutInSeconds), _ fn: @escaping () throws -> Void, _ startTime: DispatchTime = DispatchTime.now()) {
-    group?.enter()
+func retryAxCallUntilTimeout(_ timeoutInSeconds: Double = Double(AXUIElement.globalTimeoutInSeconds), _ fn: @escaping () throws -> Void, _ startTime: DispatchTime = DispatchTime.now()) {
     BackgroundWork.axCallsQueue.async {
-        retryAxCallUntilTimeout_(group, timeoutInSeconds, fn, startTime)
+        retryAxCallUntilTimeout_(timeoutInSeconds, fn, startTime)
     }
 }
 
-func retryAxCallUntilTimeout_(_ group: DispatchGroup?, _ timeoutInSeconds: Double, _ fn: @escaping () throws -> Void, _ startTime: DispatchTime = DispatchTime.now()) {
+func retryAxCallUntilTimeout_(_ timeoutInSeconds: Double, _ fn: @escaping () throws -> Void, _ startTime: DispatchTime = DispatchTime.now()) {
     do {
         try fn()
-        group?.leave()
     } catch {
         let timePassedInSeconds = Double(DispatchTime.now().uptimeNanoseconds - startTime.uptimeNanoseconds) / 1_000_000_000
         if timePassedInSeconds < timeoutInSeconds {
             BackgroundWork.axCallsQueue.asyncAfter(deadline: .now() + .milliseconds(AXUIElement.retryDelayInMilliseconds)) {
-                retryAxCallUntilTimeout_(group, timeoutInSeconds, fn, startTime)
+                retryAxCallUntilTimeout_(timeoutInSeconds, fn, startTime)
             }
         }
     }
@@ -50,6 +48,15 @@ extension AXUIElement {
         }
     }
 
+    // periphery:ignore
+    func id() -> AXUIElementID? {
+        let pointer = UnsafeRawPointer(Unmanaged.passUnretained(self).toOpaque()).advanced(by: 0x20)
+        let cfDataPointer = pointer.load(as: CFData?.self)
+        let cfData = cfDataPointer
+        let bytePtr = CFDataGetBytePtr(cfData)
+        return bytePtr?.withMemoryRebound(to: AXUIElementID.self, capacity: 1) { $0.pointee }
+    }
+
     func cgWindowId() throws -> CGWindowID? {
         var id = CGWindowID(0)
         return try axCallWhichCanThrow(_AXUIElementGetWindow(self, &id), &id)
@@ -261,8 +268,49 @@ extension AXUIElement {
         return try attribute(kAXChildrenAttribute, [AXUIElement].self)
     }
 
-    func windows() throws -> [AXUIElement]? {
-        return try attribute(kAXWindowsAttribute, [AXUIElement].self)
+    /// we combine both the normal approach and brute-force to get all possible windows
+    /// with only normal approach: we miss other-Spaces windows
+    /// with only brute-force approach: we miss windows when the app launches (e.g. launch Note.app: first window is not found by brute-force)
+    func allWindows(_ pid: pid_t) throws -> [AXUIElement] {
+        let aWindows = try! windows()
+        let bWindows = AXUIElement.windowsByBruteForce(pid)
+        return Array(Set(aWindows + bWindows))
+    }
+
+    /// doesn't return windows on other Spaces
+    /// use windowsByBruteForce if you want those
+    func windows() throws -> [AXUIElement] {
+        let windows = try attribute(kAXWindowsAttribute, [AXUIElement].self)
+        if let windows,
+           !windows.isEmpty {
+            // bug in macOS: sometimes the OS returns multiple duplicate windows (e.g. Mail.app starting at login)
+            let uniqueWindows = Array(Set(windows))
+            if !uniqueWindows.isEmpty {
+                return uniqueWindows
+            }
+        }
+        return []
+    }
+
+    /// brute-force getting the windows of a process by iterating over AXUIElementID one by one
+    private static func windowsByBruteForce(_ pid: pid_t) -> [AXUIElement] {
+        // we use this to call _AXUIElementCreateWithRemoteToken; we reuse the object for performance
+        // tests showed that this remoteToken is 20 bytes: 4 + 4 + 4 + 8; the order of bytes matters
+        var remoteToken = Data(count: 20)
+        remoteToken.replaceSubrange(0..<4, with: withUnsafeBytes(of: pid) { Data($0) })
+        remoteToken.replaceSubrange(4..<8, with: withUnsafeBytes(of: Int32(0)) { Data($0) })
+        remoteToken.replaceSubrange(8..<12, with: withUnsafeBytes(of: Int32(0x636f636f)) { Data($0) })
+        var axWindows = [AXUIElement]()
+        // we iterate to 1000 as a tradeoff between performance, and missing windows of long-lived processes
+        for axUiElementId: AXUIElementID in 0..<1000 {
+            remoteToken.replaceSubrange(12..<20, with: withUnsafeBytes(of: axUiElementId) { Data($0) })
+            if let axUiElement = _AXUIElementCreateWithRemoteToken(remoteToken as CFData)?.takeRetainedValue(),
+               let subrole = try? axUiElement.subrole(),
+               [kAXStandardWindowSubrole, kAXDialogSubrole].contains(subrole) {
+                axWindows.append(axUiElement)
+            }
+        }
+        return axWindows
     }
 
     func isMinimized() throws -> Bool {
@@ -318,3 +366,9 @@ extension AXUIElement {
 enum AxError: Error {
     case runtimeError
 }
+
+/// tests have shown that this ID has a range going from 0 to probably UInt.MAX
+/// it starts at 0 for each app, and increments over time, for each new UI element
+/// this means that long-lived apps (e.g. Finder) may have high IDs
+/// we don't know how high it can go, and if it wraps around
+typealias AXUIElementID = UInt

src/api-wrappers/CGWindow.swift

@@ -26,8 +26,11 @@ extension CGWindow {
     }
 
     // periphery:ignore
-    func bounds() -> CFDictionary? {
-        return value(kCGWindowBounds, CFDictionary.self)
+    func bounds() -> NSRect? {
+        if let cfDictionary = value(kCGWindowBounds, CFDictionary.self) {
+            return NSRect(dictionaryRepresentation: cfDictionary)
+        }
+        return nil
     }
 
     // periphery:ignore

src/api-wrappers/private-apis/ApplicationServices.HIServices.framework.swift

@@ -4,6 +4,14 @@
 @_silgen_name("_AXUIElementGetWindow") @discardableResult
 func _AXUIElementGetWindow(_ axUiElement: AXUIElement, _ wid: inout CGWindowID) -> AXError
 
+/// returns an AXUIElement given a Data object. Data object should hold:
+/// - pid (4 bytes)
+/// - 0 (4 bytes)
+/// - 0x636f636f (4 bytes)
+/// - AXUIElementID (8 bytes)
+@_silgen_name("_AXUIElementCreateWithRemoteToken") @discardableResult
+func _AXUIElementCreateWithRemoteToken(_ data: CFData) -> Unmanaged<AXUIElement>?
+
 /// for some reason, this constant is missing from ApplicationServices.HIServices.AXRoleConstants
 let kAXDocumentWindowSubrole = "AXDocumentWindow"
 

src/logic/Application.swift

@@ -67,39 +67,34 @@ class Application: NSObject {
         }
     }
 
-    func manuallyUpdateWindows(_ group: DispatchGroup? = nil) {
-        // TODO: this method manually checks windows, but will not find windows on other Spaces
-        retryAxCallUntilTimeout(group, 5) { [weak self] in
+    func manuallyUpdateWindows() {
+        retryAxCallUntilTimeout(5) { [weak self] in
             guard let self else { return }
             var atLeastOneActualWindow = false
-            if let axWindows_ = try self.axUiElement!.windows(), !axWindows_.isEmpty {
-                // bug in macOS: sometimes the OS returns multiple duplicate windows (e.g. Mail.app starting at login)
-                let uniqueWindows = Array(Set(axWindows_))
-                if uniqueWindows.isEmpty { return }
-                for axWindow in uniqueWindows {
-                    if let wid = try axWindow.cgWindowId() {
-                        let title = try axWindow.title()
-                        let subrole = try axWindow.subrole()
-                        let role = try axWindow.role()
-                        let size = try axWindow.size()
-                        let level = try wid.level()
-                        if AXUIElement.isActualWindow(self, wid, level, title, subrole, role, size) {
-                            let isFullscreen = try axWindow.isFullscreen()
-                            let isMinimized = try axWindow.isMinimized()
-                            let position = try axWindow.position()
-                            atLeastOneActualWindow = true
-                            DispatchQueue.main.async { [weak self] in
-                                guard let self else { return }
-                                if let window = (Windows.list.first { $0.isEqualRobust(axWindow, wid) }) {
-                                    window.title = window.bestEffortTitle(title)
-                                    window.size = size
-                                    window.isFullscreen = isFullscreen
-                                    window.isMinimized = isMinimized
-                                    window.position = position
-                                } else {
-                                    let window = self.addWindow(axWindow, wid, title, isFullscreen, isMinimized, position, size)
-                                    App.app.refreshOpenUi([window], .refreshUiAfterExternalEvent)
-                                }
+            let axWindows = try self.axUiElement!.allWindows(self.pid)
+            for axWindow in axWindows {
+                if let wid = try axWindow.cgWindowId() {
+                    let title = try axWindow.title()
+                    let subrole = try axWindow.subrole()
+                    let role = try axWindow.role()
+                    let size = try axWindow.size()
+                    let level = try wid.level()
+                    if AXUIElement.isActualWindow(self, wid, level, title, subrole, role, size) {
+                        let isFullscreen = try axWindow.isFullscreen()
+                        let isMinimized = try axWindow.isMinimized()
+                        let position = try axWindow.position()
+                        atLeastOneActualWindow = true
+                        DispatchQueue.main.async { [weak self] in
+                            guard let self else { return }
+                            if let window = (Windows.list.first { $0.isEqualRobust(axWindow, wid) }) {
+                                window.title = window.bestEffortTitle(title)
+                                window.size = size
+                                window.isFullscreen = isFullscreen
+                                window.isMinimized = isMinimized
+                                window.position = position
+                            } else {
+                                let window = self.addWindow(axWindow, wid, title, isFullscreen, isMinimized, position, size)
+                                App.app.refreshOpenUi([window], .refreshUiAfterExternalEvent)
                             }
                         }
                     }

src/logic/Applications.swift

@@ -4,23 +4,8 @@ import ApplicationServices
 class Applications {
     static var list = [Application]()
 
-    static func manuallyUpdateWindowsFor2s() {
-        let group = DispatchGroup()
-        manuallyUpdateWindows(group)
-        _ = group.wait(wallTimeout: .now() + .seconds(2))
-    }
-
-    static func manuallyUpdateWindows(_ group: DispatchGroup? = nil) {
-        for app in list {
-            if app.runningApplication.isFinishedLaunching && app.runningApplication.activationPolicy != .prohibited {
-                app.manuallyUpdateWindows(group)
-            }
-        }
-    }
-
     static func initialDiscovery() {
         addInitialRunningApplications()
-        addInitialRunningApplicationsWindows()
         RunningApplicationsEvents.observe()
     }
 
@@ -28,25 +13,6 @@ class Applications {
         addRunningApplications(NSWorkspace.shared.runningApplications)
     }
 
-    static func addInitialRunningApplicationsWindows() {
-        let otherSpaces = Spaces.otherSpaces()
-        // the CGSAddWindowsToSpaces trick stopped working starting with macOS 12.2
-        // see https://github.com/lwouis/alt-tab-macos/issues/1324
-        if otherSpaces.count > 0, #unavailable(macOS 12.2) {
-            let windowsOnCurrentSpace = Spaces.windowsInSpaces([Spaces.currentSpaceId])
-            let windowsOnOtherSpaces = Spaces.windowsInSpaces(otherSpaces)
-            let windowsOnlyOnOtherSpaces = Array(Set(windowsOnOtherSpaces).subtracting(windowsOnCurrentSpace))
-            if windowsOnlyOnOtherSpaces.count > 0 {
-                // on initial launch, we use private APIs to bring windows from other spaces into the current space, observe them, then remove them from the current space
-                CGSAddWindowsToSpaces(CGS_CONNECTION, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId])
-                Applications.manuallyUpdateWindowsFor2s()
-                CGSRemoveWindowsFromSpaces(CGS_CONNECTION, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId])
-            }
-        } else {
-            Applications.manuallyUpdateWindows()
-        }
-    }
-
     static func addRunningApplications(_ runningApps: [NSRunningApplication]) {
         runningApps.forEach {
             let bundleIdentifier = $0.bundleIdentifier

src/logic/events/AccessibilityEvents.swift

@@ -118,16 +118,6 @@ fileprivate func focusedWindowChanged(_ element: AXUIElement, _ pid: pid_t) thro
                 }
             }
         }
-        DispatchQueue.main.async {
-            if let app = Applications.find(pid) {
-                // work-around for apps started "hidden" like in Login Items with the "Hide" checkbox, or with `open -j`
-                // these apps report isHidden=false, don't generate windowCreated events initially, and have a delay before their windows are created
-                // our only recourse is to manually check their windows once they emit
-                if (!app.hasBeenActiveOnce) {
-                    app.manuallyUpdateWindows()
-                }
-            }
-        }
     } else {
         DispatchQueue.main.async {
             Applications.find(pid)?.focusedWindow = nil

src/logic/events/SpacesEvents.swift

@@ -14,6 +14,6 @@ class SpacesEvents {
         // also, updating windows on Space transition works around an issue with Safari where its fullscreen windows spawn not in fullscreen.
         // resize/move events happen and the window is still not fullscreen. AltTab doesn't get informed that the window is later fullscreen.
         // updating on Space change helps correct the window to being fullscreen
-        Applications.manuallyUpdateWindows()
+        Applications.manuallyUpdateAllAppsWindows()
     }
 }

unit-tests/AXUIElementTests.swift

@@ -0,0 +1,66 @@
+//import XCTest
+//
+//final class ImageScalingTests: XCTestCase {
+//    /// 20ms
+//    func testWindowsByBruteForce() throws {
+//        measure(options: options) {
+//            let _ = windowsByBruteForce(41414)
+//        }
+//    }
+//
+//    private var options: XCTMeasureOptions = {
+//        let o = XCTMeasureOptions()
+//        o.iterationCount = 10
+//        return o
+//    }()
+//}
+//
+//func windowsByBruteForce(_ pid: pid_t) -> [AXUIElement] {
+//    // we use this to call _AXUIElementCreateWithRemoteToken; we reuse the object for performance
+//    // tests showed that this remoteToken is 20 bytes: 4 + 4 + 4 + 8; the order of bytes matters
+//    var remoteToken = Data(count: 20)
+//    remoteToken.replaceSubrange(0..<4, with: withUnsafeBytes(of: pid) { Data($0) })
+//    remoteToken.replaceSubrange(4..<8, with: withUnsafeBytes(of: Int32(0)) { Data($0) })
+//    remoteToken.replaceSubrange(8..<12, with: withUnsafeBytes(of: Int32(0x636f636f)) { Data($0) })
+//    var axWindows = [AXUIElement]()
+//    // we iterate to 1000 as a tradeoff between performance, and missing windows of long-lived processes
+//    for axUiElementId: AXUIElementID in 0..<1000 {
+//        remoteToken.replaceSubrange(12..<20, with: withUnsafeBytes(of: axUiElementId) { Data($0) })
+//        if let axUiElement = _AXUIElementCreateWithRemoteToken(remoteToken as CFData)?.takeRetainedValue(),
+//           let subrole = try? axUiElement.subrole(),
+//           [kAXStandardWindowSubrole, kAXDialogSubrole].contains(subrole) {
+//            axWindows.append(axUiElement)
+//        }
+//    }
+//    return axWindows
+//}
+//
+//typealias AXUIElementID = UInt
+//
+//enum AxError: Error {
+//    case runtimeError
+//}
+//
+//@_silgen_name("_AXUIElementCreateWithRemoteToken") @discardableResult
+//func _AXUIElementCreateWithRemoteToken(_ data: CFData) -> Unmanaged<AXUIElement>?
+//
+//extension AXUIElement {
+//    func axCallWhichCanThrow<T>(_ result: AXError, _ successValue: inout T) throws -> T? {
+//        switch result {
+//            case .success: return successValue
+//            // .cannotComplete can happen if the app is unresponsive; we throw in that case to retry until the call succeeds
+//            case .cannotComplete: throw AxError.runtimeError
+//            // for other errors it's pointless to retry
+//            default: return nil
+//        }
+//    }
+//
+//    func attribute<T>(_ key: String, _ _: T.Type) throws -> T? {
+//        var value: AnyObject?
+//        return try axCallWhichCanThrow(AXUIElementCopyAttributeValue(self, key as CFString, &value), &value) as? T
+//    }
+//
+//    func subrole() throws -> String? {
+//        return try attribute(kAXSubroleAttribute, String.self)
+//    }
+//}