fix: improve detection of windows (closes #4405)

src/api-wrappers/AXUIElement.swift

@@ -9,37 +9,35 @@ import ApplicationServices.HIServices.AXActionConstants
 extension AXUIElement {
     // default timeout for AX calls is 6s
     // we reduce to 1s to avoid AX calls blocking threads, thus too many threads getting created to make the next AX calls
-    static let globalMessagingTimeoutInSeconds = Float(1)
-    // we retry for 2 minutes at most
-    static let retryAXCallQuickQueueTimeoutInSeconds = Float(6)
-    static let retryAXCallSlowQueueTimeoutInSeconds = Float(120)
-    // 250ms is similar to human delay in processing changes on screen
-    // See https://humanbenchmark.com/tests/reactiontime
-    static let retryDelayInMilliseconds = DispatchTimeInterval.milliseconds(250)
+    private static let globalMessagingTimeoutInSeconds = Float(1)
+    // if an app times out our AX calls, we retry for 6s then give up
+    private static let axCallsRetriesQueueTimeoutInSeconds = Float(6)
+    // once an app is unresponsive, let's ignore other AX calls for it to avoid congestion
+    private static var axCallsRetriesQueueUnresponsiveAppsMap = ConcurrentMap<pid_t, UInt64>()
+    // some events like window resizing trigger in quick succession. We debounce those
+    private static var eventsDebounceMap = ConcurrentMap<String, DispatchWorkItem>()
 
     static func setGlobalTimeout() {
         AXUIElementSetMessagingTimeout(AXUIElementCreateSystemWide(), globalMessagingTimeoutInSeconds)
     }
 
-    private static var debounceMap = DebounceMap()
-
     /// if the window server is busy, it may not reply to AX calls. We retry right before the call times-out and returns a bogus value
-    static func retryAxCallUntilTimeout(file: String = #file, function: String = #function, line: Int = #line, context: String = "", after: DispatchTime? = nil, debounceType: DebounceType? = nil, wid: CGWindowID? = nil, slowQueue: Bool = false, startTimeInNanoseconds: UInt64 = DispatchTime.now().uptimeNanoseconds, block: @escaping () throws -> Void) {
-        let closure = { retryAxCallUntilTimeout_(file: file, function: function, line: line, context: context, after: after, debounceType: debounceType, wid: wid, slowQueue: slowQueue, startTimeInNanoseconds: startTimeInNanoseconds, block: block) }
-        let queue = slowQueue ? BackgroundWork.axCallsSlowQueue : BackgroundWork.axCallsQuickQueue
+    static func retryAxCallUntilTimeout(file: String = #file, function: String = #function, line: Int = #line, context: String = "", after: DispatchTime? = nil, debounceType: DebounceType? = nil, pid: pid_t? = nil, wid: CGWindowID? = nil, retriesQueue: Bool = false, startTimeInNanoseconds: UInt64 = DispatchTime.now().uptimeNanoseconds, block: @escaping () throws -> Void) {
+        let closure = { retryAxCallUntilTimeout_(file: file, function: function, line: line, context: context, after: after, debounceType: debounceType, pid: pid, wid: wid, retriesQueue: retriesQueue, startTimeInNanoseconds: startTimeInNanoseconds, block: block) }
+        let queue = retriesQueue ? BackgroundWork.axCallsRetriesQueue : BackgroundWork.axCallsFirstAttemptQueue
         if let after {
             queue!.addOperationAfter(deadline: after, block: closure)
         } else if let debounceType, let wid {
             let debounceMapKey = "\(debounceType.rawValue)\(wid)"
             let workItem = DispatchWorkItem {
                 closure()
-                debounceMap.withLock { $0[debounceMapKey] = nil }
+                eventsDebounceMap.withLock { $0[debounceMapKey] = nil }
             }
-            debounceMap.withLock {
+            eventsDebounceMap.withLock {
                 $0[debounceMapKey]?.cancel()
                 $0[debounceMapKey] = workItem
             }
-            queue!.addOperationAfter(deadline: .now() + retryDelayInMilliseconds) {
+            queue!.addOperationAfter(deadline: .now() + humanPerceptionDelay) {
                 workItem.perform()
             }
         } else {
@@ -47,24 +45,43 @@ extension AXUIElement {
         }
     }
 
-    private static func retryAxCallUntilTimeout_(file: String, function: String, line: Int, context: String, after: DispatchTime?, debounceType: DebounceType?, wid: CGWindowID?, slowQueue: Bool, startTimeInNanoseconds: UInt64, block: @escaping () throws -> Void) {
+    private static func retryAxCallUntilTimeout_(file: String, function: String, line: Int, context: String, after: DispatchTime?, debounceType: DebounceType?, pid: pid_t?, wid: CGWindowID?, retriesQueue: Bool, startTimeInNanoseconds: UInt64, block: @escaping () throws -> Void) {
+        // attempt the AX call
         if (try? block()) != nil {
             return
         }
+        // do we already have ongoing retries for this pid?
+        if let pid {
+            axCallsRetriesQueueUnresponsiveAppsMap.lock.lock()
+            let time = axCallsRetriesQueueUnresponsiveAppsMap.map[pid]
+            axCallsRetriesQueueUnresponsiveAppsMap.lock.unlock()
+            if let time {
+                if startTimeInNanoseconds > time {
+                    // new most recent call; replace and retry
+                    axCallsRetriesQueueUnresponsiveAppsMap.withLock { $0[pid] = startTimeInNanoseconds }
+                } else if startTimeInNanoseconds == time {
+                    // most recent call; retry
+                } else {
+                    // old call which has been replaced; ignore
+                    return
+                }
+            } else {
+                // first call; set and retry
+                axCallsRetriesQueueUnresponsiveAppsMap.withLock { $0[pid] = startTimeInNanoseconds }
+            }
+        }
+        // should we give up?
         let timePassedInSeconds = Float(DispatchTime.now().uptimeNanoseconds - startTimeInNanoseconds) / 1_000_000_000
-        if timePassedInSeconds >= retryAXCallSlowQueueTimeoutInSeconds {
-            Logger.info("AX call failed for more than \(Int(retryAXCallSlowQueueTimeoutInSeconds))s. Giving up on it", logFromContext(file, function, line, context))
-            // if we keep the "fix: improve detection of windows" commit, we need to blacklist such apps (or windows?) for the runtime of AltTab to avoid trying over and over
-            return
-        }
-        if timePassedInSeconds >= retryAXCallQuickQueueTimeoutInSeconds {
-            Logger.info(logFromContext(file, function, line, context))
-            retryAxCallUntilTimeout(file: file, function: function, line: line, context: context, after: .now() + .seconds(1), debounceType: debounceType, wid: wid, slowQueue: true, startTimeInNanoseconds: startTimeInNanoseconds, block: block)
+        if timePassedInSeconds >= axCallsRetriesQueueTimeoutInSeconds {
+            Logger.info("AX call failed for more than \(Int(axCallsRetriesQueueTimeoutInSeconds))s. Giving up on it", logFromContext(file, function, line, context))
+            if let pid {
+                axCallsRetriesQueueUnresponsiveAppsMap.withLock { $0.removeValue(forKey: pid) }
+            }
             return
         }
+        // retry
         Logger.info(logFromContext(file, function, line, context))
-        retryAxCallUntilTimeout(file: file, function: function, line: line, context: context, after: .now() + retryDelayInMilliseconds, debounceType: debounceType, wid: wid, slowQueue: false, startTimeInNanoseconds: startTimeInNanoseconds, block: block)
-
+        retryAxCallUntilTimeout(file: file, function: function, line: line, context: context, after: .now() + humanPerceptionDelay, debounceType: debounceType, pid: pid, wid: wid, retriesQueue: true, startTimeInNanoseconds: startTimeInNanoseconds, block: block)
     }
 
     private static func logFromContext(_ file: String, _ function: String, _ line: Int, _ context: String) -> String {
@@ -450,12 +467,12 @@ enum DebounceType: Int {
     case windowTitleChanged = 1
 }
 
-class DebounceMap {
-    typealias DebounceMapType = [String: DispatchWorkItem]
-    private var map: DebounceMapType = [:]
-    private let lock = NSLock()
+class ConcurrentMap<K: Hashable, V> {
+    var map = [K: V]()
+    let lock = NSLock()
 
-    func withLock<T>(_ block: (inout DebounceMapType) -> T) -> T {
+    @discardableResult
+    func withLock<T>(_ block: (inout [K: V]) -> T) -> T {
         lock.lock()
         defer { lock.unlock() }
         return block(&map)

src/api-wrappers/HelperExtensions.swift

@@ -328,3 +328,7 @@ extension DispatchTimeInterval {
 extension NSRunningApplication {
     var id: String { "pid:\(String(describing: processIdentifier)) app:\(bundleIdentifier ?? bundleURL?.absoluteString ?? executableURL?.absoluteString ?? localizedName ?? "nil")"  }
 }
+
+// 250ms is similar to human delay in processing changes on screen
+// See https://humanbenchmark.com/tests/reactiontime
+let humanPerceptionDelay = DispatchTimeInterval.milliseconds(250)

src/logic/Application.swift

@@ -69,7 +69,7 @@ class Application: NSObject {
     }
 
     func manuallyUpdateWindows() {
-        AXUIElement.retryAxCallUntilTimeout(context: debugId) { [weak self] in
+        AXUIElement.retryAxCallUntilTimeout(context: debugId, pid: pid) { [weak self] in
             guard let self else { return }
             var atLeastOneActualWindow = false
             guard let axWindows = try self.axUiElement?.allWindows(self.pid) else { return }
@@ -171,7 +171,7 @@ class Application: NSObject {
             kAXApplicationHiddenNotification,
             kAXApplicationShownNotification,
         ] {
-            AXUIElement.retryAxCallUntilTimeout(context: debugId) { [weak self] in
+            AXUIElement.retryAxCallUntilTimeout(context: debugId, pid: pid) { [weak self] in
                 guard let self else { return }
                 if try self.axUiElement!.subscribeToNotification(axObserver, notification) {
                     if notification == kAXApplicationActivatedNotification {

src/logic/Applications.swift

@@ -13,6 +13,12 @@ class Applications {
         addRunningApplications(NSWorkspace.shared.runningApplications)
     }
 
+    static func manuallyRefreshAllWindows() {
+        for app in list {
+            app.manuallyUpdateWindows()
+        }
+    }
+
     static func addRunningApplications(_ runningApps: [NSRunningApplication]) {
         runningApps.forEach {
             let bundleIdentifier = $0.bundleIdentifier

src/logic/BackgroundWork.swift

@@ -5,8 +5,8 @@ class BackgroundWork {
     // we use an OperationQueue tied to the global DispatchQueue to call blocking APIs in parallel
     static var screenshotsQueue: LabeledOperationQueue!
     static var accessibilityCommandsQueue: LabeledOperationQueue!
-    static var axCallsQuickQueue: LabeledOperationQueue!
-    static var axCallsSlowQueue: LabeledOperationQueue!
+    static var axCallsFirstAttemptQueue: LabeledOperationQueue!
+    static var axCallsRetriesQueue: LabeledOperationQueue!
     static var crashReportsQueue: LabeledOperationQueue!
     // we use Threads to observe events in sequence
     static var accessibilityEventsThread: BackgroundThreadWithRunLoop!
@@ -27,9 +27,9 @@ class BackgroundWork {
         // They are tried once and if they timeout we don't retry. The OS seems to still execute them even if the call timed out
         accessibilityCommandsQueue = LabeledOperationQueue("accessibilityCommandsQueue", .userInteractive, 4)
         // calls to the AX APIs can block for a long time (e.g. if an app is unresponsive)
-        // We first try the AX calls on the Quick queue. If we get a timeout, we move to the Slow queue and retry for a while there
-        axCallsQuickQueue = LabeledOperationQueue("axCallsQuickQueue", .userInteractive, 8)
-        axCallsSlowQueue = LabeledOperationQueue("axCallsSlowQueue", .userInteractive, 8)
+        // We first try the AX calls on axCallsFirstAttemptQueue. If we get a timeout, we move to axCallsRetriesQueue and retry there for a while
+        axCallsFirstAttemptQueue = LabeledOperationQueue("axCallsFirstAttemptQueue", .userInteractive, 8)
+        axCallsRetriesQueue = LabeledOperationQueue("axCallsRetriesQueue", .userInteractive, 8)
         // we observe app and windows notifications. They arrive on this thread, and are handled off the main thread initially
         accessibilityEventsThread = BackgroundThreadWithRunLoop("accessibilityEventsThread", .userInteractive)
         // we listen to as any keyboard events as possible on a background thread, as it's more available/reliable than the main thread
@@ -70,7 +70,7 @@ class BackgroundWork {
     }
 
     private static func logQueues() -> Void {
-        let queues = [screenshotsQueue, accessibilityCommandsQueue, axCallsQuickQueue, axCallsSlowQueue, crashReportsQueue].compactMap { $0 }
+        let queues = [screenshotsQueue, accessibilityCommandsQueue, axCallsFirstAttemptQueue, axCallsRetriesQueue, crashReportsQueue].compactMap { $0 }
         var map = [String:Int]()
         for queue in queues {
             map[queue.underlyingQueue!.label] = queue.operations.reduce(0) { $1.isExecuting ? $0 + 1 : $0 }

src/logic/Window.swift

@@ -68,7 +68,7 @@ class Window {
     /// some apps will not trigger AXApplicationActivated, where we usually update application.focusedWindow
     /// workaround: we check and possibly do it here
     func checkIfFocused(_ application: Application, _ wid: CGWindowID) {
-        AXUIElement.retryAxCallUntilTimeout(context: debugId) {
+        AXUIElement.retryAxCallUntilTimeout(context: debugId, pid: application.pid) {
             let focusedWid = try application.axUiElement?.focusedWindow()?.cgWindowId()
             if wid == focusedWid {
                 application.focusedWindow = self
@@ -86,7 +86,7 @@ class Window {
         AXObserverCreate(application.pid, axObserverCallback, &axObserver)
         guard let axObserver else { return }
         for notification in Window.notifications {
-            AXUIElement.retryAxCallUntilTimeout(context: debugId) { [weak self] in
+            AXUIElement.retryAxCallUntilTimeout(context: debugId, pid: application.pid) { [weak self] in
                 guard let self else { return }
                 if try self.axUiElement!.subscribeToNotification(axObserver, notification) {
                     if notification == kAXUIElementDestroyedNotification {

src/logic/Windows.swift

@@ -74,7 +74,7 @@ class Windows {
     static func updateIsFullscreenOnCurrentSpace() {
         let windowsOnCurrentSpace = Windows.list.filter { !$0.isWindowlessApp }
         for window in windowsOnCurrentSpace {
-            AXUIElement.retryAxCallUntilTimeout(context: window.debugId, after: .now() + AXUIElement.retryDelayInMilliseconds) { [weak window] in
+            AXUIElement.retryAxCallUntilTimeout(context: window.debugId, after: .now() + humanPerceptionDelay) { [weak window] in
                 guard let window else { return }
                 try updateWindowSizeAndPositionAndFullscreen(window.axUiElement!, window.cgWindowId!, window)
             }

src/logic/events/AccessibilityEvents.swift

@@ -23,9 +23,9 @@ fileprivate func handleEvent(_ type: String, _ element: AXUIElement) throws {
             case kAXUIElementDestroyedNotification: try windowDestroyed(element, pid)
             case kAXWindowMiniaturizedNotification,
                  kAXWindowDeminiaturizedNotification: try windowMiniaturizedOrDeminiaturized(element, type)
-            case kAXTitleChangedNotification: try windowTitleChanged(element)
+            case kAXTitleChangedNotification: try windowTitleChanged(element, pid)
             case kAXWindowResizedNotification,
-                 kAXWindowMovedNotification: try windowResizedOrMoved(element)
+                 kAXWindowMovedNotification: try windowResizedOrMoved(element, pid)
             default: return
         }
     }
@@ -100,7 +100,7 @@ fileprivate func focusedWindowChanged(_ element: AXUIElement, _ pid: pid_t) thro
                 if let windows = Windows.updateLastFocus(element, wid) {
                     App.app.refreshOpenUi(windows, .refreshUiAfterExternalEvent)
                 } else {
-                    AXUIElement.retryAxCallUntilTimeout(context: "wid:\(wid) pid:\(pid)") {
+                    AXUIElement.retryAxCallUntilTimeout(context: "wid:\(wid) pid:\(pid)", pid: pid) {
                         if let (title, role, subrole, isMinimized, isFullscreen) = try element.windowAttributes() {
                             let position = try element.position()
                             let size = try element.size()
@@ -154,9 +154,9 @@ fileprivate func windowMiniaturizedOrDeminiaturized(_ element: AXUIElement, _ ty
     }
 }
 
-fileprivate func windowTitleChanged(_ element: AXUIElement) throws {
+fileprivate func windowTitleChanged(_ element: AXUIElement, _ pid: pid_t) throws {
     let wid = try element.cgWindowId()
-    AXUIElement.retryAxCallUntilTimeout(context: "\(wid)", debounceType: .windowTitleChanged, wid: wid) {
+    AXUIElement.retryAxCallUntilTimeout(context: "\(wid)", debounceType: .windowTitleChanged, pid: pid, wid: wid) {
         if let (title, _, _, isMinimized, isFullscreen) = try element.windowAttributes() {
             DispatchQueue.main.async {
                 if let window = (Windows.list.first { $0.isEqualRobust(element, wid) }), title != window.title {
@@ -170,9 +170,9 @@ fileprivate func windowTitleChanged(_ element: AXUIElement) throws {
     }
 }
 
-fileprivate func windowResizedOrMoved(_ element: AXUIElement) throws {
+fileprivate func windowResizedOrMoved(_ element: AXUIElement, _ pid: pid_t) throws {
     let wid = try element.cgWindowId()
-    AXUIElement.retryAxCallUntilTimeout(context: "\(wid)", debounceType: .windowResizedOrMoved, wid: wid) {
+    AXUIElement.retryAxCallUntilTimeout(context: "\(wid)", debounceType: .windowResizedOrMoved, pid: pid, wid: wid) {
         try updateWindowSizeAndPositionAndFullscreen(element, wid, nil)
     }
 }

src/ui/App.swift

@@ -237,13 +237,14 @@ class App: AppCenterApplication {
         Logger.debug(shortcutIndex, self.shortcutIndex, isFirstSummon)
         App.app.appIsBeingUsed = true
         if isFirstSummon || shortcutIndex != self.shortcutIndex {
+            NSScreen.updatePreferred()
+            Applications.manuallyRefreshAllWindows()
             if isVeryFirstSummon {
                 Windows.sortByLevel()
                 isVeryFirstSummon = false
             }
             isFirstSummon = false
             self.shortcutIndex = shortcutIndex
-            NSScreen.updatePreferred()
             if !Windows.updatesBeforeShowing() { hideUi(); return }
             Windows.setInitialFocusedAndHoveredWindowIndex()
             if Preferences.windowDisplayDelay == DispatchTimeInterval.milliseconds(0) {