From 2cd8b96d389004b41ce2aad5667d0a11be36dabf Mon Sep 17 00:00:00 2001 From: lwouis Date: Mon, 3 Feb 2025 23:42:40 +0100 Subject: [PATCH] fix: better detect windows from other spaces (closes #1324) We found a workaround after more than 3 years of searching! --- alt-tab-macos.xcodeproj/project.pbxproj | 4 ++ src/api-wrappers/AXUIElement.swift | 70 ++++++++++++++++--- src/api-wrappers/CGWindow.swift | 7 +- ...icationServices.HIServices.framework.swift | 8 +++ src/logic/Application.swift | 57 +++++++-------- src/logic/Applications.swift | 34 --------- src/logic/events/AccessibilityEvents.swift | 10 --- src/logic/events/SpacesEvents.swift | 2 +- unit-tests/AXUIElementTests.swift | 66 +++++++++++++++++ 9 files changed, 172 insertions(+), 86 deletions(-) create mode 100644 unit-tests/AXUIElementTests.swift diff --git a/alt-tab-macos.xcodeproj/project.pbxproj b/alt-tab-macos.xcodeproj/project.pbxproj index 79677cf8..2195b2d0 100644 --- a/alt-tab-macos.xcodeproj/project.pbxproj +++ b/alt-tab-macos.xcodeproj/project.pbxproj @@ -127,6 +127,7 @@ BF0C8879F979414A87AB2958 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = BF0C89348E07FC05870FEE62 /* InfoPlist.strings */; }; BF0C88C371B5550E7D23E4F1 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = BF0C8BF5835F97923BF3E8DA /* InfoPlist.strings */; }; BF0C88D7E209C788558BD943 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = BF0C8CC9692E6E4A32846D72 /* InfoPlist.strings */; }; + BF0C88FFEE7645A1966C7C4E /* AXUIElementTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF0C855CD9DBCAEFE48F9EA5 /* AXUIElementTests.swift */; }; BF0C894199BB262BDB2D8D37 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = BF0C8221101CD179DB1121D5 /* InfoPlist.strings */; }; BF0C894325A54CF0B6B53464 /* Localizable.strings in Resources */ = {isa = PBXBuildFile; fileRef = BF0C8B8B31D2F421AC258351 /* Localizable.strings */; }; BF0C896B5D08B7DA8F2024AC /* KeyRepeatTimer.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF0C8D606B53D315F2D9927B /* KeyRepeatTimer.swift */; }; @@ -386,6 +387,7 @@ BF0C84B9B7A41E4A4534E519 /* app-icons-style-full.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = "app-icons-style-full.jpg"; sourceTree = ""; }; BF0C84E11784B3B0900D0963 /* app.icns */ = {isa = PBXFileReference; lastKnownFileType = image.icns; path = app.icns; sourceTree = ""; }; BF0C850C46CCD8B978168CCD /* sv */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.strings; name = sv; path = Localizable.strings; sourceTree = ""; }; + BF0C855CD9DBCAEFE48F9EA5 /* AXUIElementTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AXUIElementTests.swift; sourceTree = ""; }; BF0C85C20516C4210508F349 /* tsconfig.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = tsconfig.json; sourceTree = ""; }; BF0C861255F42BF7B23408EF /* sq */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.strings; name = sq; path = InfoPlist.strings; sourceTree = ""; }; BF0C863EDD60D925950B223B /* convert_iconset_to_icns.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; path = convert_iconset_to_icns.sh; sourceTree = ""; }; @@ -972,6 +974,7 @@ BF0C899905524253DC170B56 /* KeyboardEventsTests.swift */, BF0C829EAA93A4C4ADB42CF8 /* Mocks.swift */, BF0C8C6810FF50908E637A48 /* ConcurrentScreenshots.swift */, + BF0C855CD9DBCAEFE48F9EA5 /* AXUIElementTests.swift */, ); path = "unit-tests"; sourceTree = ""; @@ -2242,6 +2245,7 @@ BF0C875A28B98904F8F4A66A /* ATShortcut.swift in Sources */, BF0C84E5AC1D423625ABF3E6 /* Mocks.swift in Sources */, BF0C8CBED8D88DFBADE7EC03 /* ConcurrentScreenshots.swift in Sources */, + BF0C88FFEE7645A1966C7C4E /* AXUIElementTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/src/api-wrappers/AXUIElement.swift b/src/api-wrappers/AXUIElement.swift index e97a2e74..a7813478 100644 --- a/src/api-wrappers/AXUIElement.swift +++ b/src/api-wrappers/AXUIElement.swift @@ -7,22 +7,20 @@ import ApplicationServices.HIServices.AXAttributeConstants import ApplicationServices.HIServices.AXActionConstants // if the window server is busy, it may not reply to AX calls. We retry right before the call times-out and returns a bogus value -func retryAxCallUntilTimeout(_ group: DispatchGroup? = nil, _ timeoutInSeconds: Double = Double(AXUIElement.globalTimeoutInSeconds), _ fn: @escaping () throws -> Void, _ startTime: DispatchTime = DispatchTime.now()) { - group?.enter() +func retryAxCallUntilTimeout(_ timeoutInSeconds: Double = Double(AXUIElement.globalTimeoutInSeconds), _ fn: @escaping () throws -> Void, _ startTime: DispatchTime = DispatchTime.now()) { BackgroundWork.axCallsQueue.async { - retryAxCallUntilTimeout_(group, timeoutInSeconds, fn, startTime) + retryAxCallUntilTimeout_(timeoutInSeconds, fn, startTime) } } -func retryAxCallUntilTimeout_(_ group: DispatchGroup?, _ timeoutInSeconds: Double, _ fn: @escaping () throws -> Void, _ startTime: DispatchTime = DispatchTime.now()) { +func retryAxCallUntilTimeout_(_ timeoutInSeconds: Double, _ fn: @escaping () throws -> Void, _ startTime: DispatchTime = DispatchTime.now()) { do { try fn() - group?.leave() } catch { let timePassedInSeconds = Double(DispatchTime.now().uptimeNanoseconds - startTime.uptimeNanoseconds) / 1_000_000_000 if timePassedInSeconds < timeoutInSeconds { BackgroundWork.axCallsQueue.asyncAfter(deadline: .now() + .milliseconds(AXUIElement.retryDelayInMilliseconds)) { - retryAxCallUntilTimeout_(group, timeoutInSeconds, fn, startTime) + retryAxCallUntilTimeout_(timeoutInSeconds, fn, startTime) } } } @@ -50,6 +48,15 @@ extension AXUIElement { } } + // periphery:ignore + func id() -> AXUIElementID? { + let pointer = UnsafeRawPointer(Unmanaged.passUnretained(self).toOpaque()).advanced(by: 0x20) + let cfDataPointer = pointer.load(as: CFData?.self) + let cfData = cfDataPointer + let bytePtr = CFDataGetBytePtr(cfData) + return bytePtr?.withMemoryRebound(to: AXUIElementID.self, capacity: 1) { $0.pointee } + } + func cgWindowId() throws -> CGWindowID? { var id = CGWindowID(0) return try axCallWhichCanThrow(_AXUIElementGetWindow(self, &id), &id) @@ -261,8 +268,49 @@ extension AXUIElement { return try attribute(kAXChildrenAttribute, [AXUIElement].self) } - func windows() throws -> [AXUIElement]? { - return try attribute(kAXWindowsAttribute, [AXUIElement].self) + /// we combine both the normal approach and brute-force to get all possible windows + /// with only normal approach: we miss other-Spaces windows + /// with only brute-force approach: we miss windows when the app launches (e.g. launch Note.app: first window is not found by brute-force) + func allWindows(_ pid: pid_t) throws -> [AXUIElement] { + let aWindows = try! windows() + let bWindows = AXUIElement.windowsByBruteForce(pid) + return Array(Set(aWindows + bWindows)) + } + + /// doesn't return windows on other Spaces + /// use windowsByBruteForce if you want those + func windows() throws -> [AXUIElement] { + let windows = try attribute(kAXWindowsAttribute, [AXUIElement].self) + if let windows, + !windows.isEmpty { + // bug in macOS: sometimes the OS returns multiple duplicate windows (e.g. Mail.app starting at login) + let uniqueWindows = Array(Set(windows)) + if !uniqueWindows.isEmpty { + return uniqueWindows + } + } + return [] + } + + /// brute-force getting the windows of a process by iterating over AXUIElementID one by one + private static func windowsByBruteForce(_ pid: pid_t) -> [AXUIElement] { + // we use this to call _AXUIElementCreateWithRemoteToken; we reuse the object for performance + // tests showed that this remoteToken is 20 bytes: 4 + 4 + 4 + 8; the order of bytes matters + var remoteToken = Data(count: 20) + remoteToken.replaceSubrange(0..<4, with: withUnsafeBytes(of: pid) { Data($0) }) + remoteToken.replaceSubrange(4..<8, with: withUnsafeBytes(of: Int32(0)) { Data($0) }) + remoteToken.replaceSubrange(8..<12, with: withUnsafeBytes(of: Int32(0x636f636f)) { Data($0) }) + var axWindows = [AXUIElement]() + // we iterate to 1000 as a tradeoff between performance, and missing windows of long-lived processes + for axUiElementId: AXUIElementID in 0..<1000 { + remoteToken.replaceSubrange(12..<20, with: withUnsafeBytes(of: axUiElementId) { Data($0) }) + if let axUiElement = _AXUIElementCreateWithRemoteToken(remoteToken as CFData)?.takeRetainedValue(), + let subrole = try? axUiElement.subrole(), + [kAXStandardWindowSubrole, kAXDialogSubrole].contains(subrole) { + axWindows.append(axUiElement) + } + } + return axWindows } func isMinimized() throws -> Bool { @@ -318,3 +366,9 @@ extension AXUIElement { enum AxError: Error { case runtimeError } + +/// tests have shown that this ID has a range going from 0 to probably UInt.MAX +/// it starts at 0 for each app, and increments over time, for each new UI element +/// this means that long-lived apps (e.g. Finder) may have high IDs +/// we don't know how high it can go, and if it wraps around +typealias AXUIElementID = UInt diff --git a/src/api-wrappers/CGWindow.swift b/src/api-wrappers/CGWindow.swift index 5dfdc9a4..6d327050 100644 --- a/src/api-wrappers/CGWindow.swift +++ b/src/api-wrappers/CGWindow.swift @@ -26,8 +26,11 @@ extension CGWindow { } // periphery:ignore - func bounds() -> CFDictionary? { - return value(kCGWindowBounds, CFDictionary.self) + func bounds() -> NSRect? { + if let cfDictionary = value(kCGWindowBounds, CFDictionary.self) { + return NSRect(dictionaryRepresentation: cfDictionary) + } + return nil } // periphery:ignore diff --git a/src/api-wrappers/private-apis/ApplicationServices.HIServices.framework.swift b/src/api-wrappers/private-apis/ApplicationServices.HIServices.framework.swift index 75904be8..eb53985a 100644 --- a/src/api-wrappers/private-apis/ApplicationServices.HIServices.framework.swift +++ b/src/api-wrappers/private-apis/ApplicationServices.HIServices.framework.swift @@ -4,6 +4,14 @@ @_silgen_name("_AXUIElementGetWindow") @discardableResult func _AXUIElementGetWindow(_ axUiElement: AXUIElement, _ wid: inout CGWindowID) -> AXError +/// returns an AXUIElement given a Data object. Data object should hold: +/// - pid (4 bytes) +/// - 0 (4 bytes) +/// - 0x636f636f (4 bytes) +/// - AXUIElementID (8 bytes) +@_silgen_name("_AXUIElementCreateWithRemoteToken") @discardableResult +func _AXUIElementCreateWithRemoteToken(_ data: CFData) -> Unmanaged? + /// for some reason, this constant is missing from ApplicationServices.HIServices.AXRoleConstants let kAXDocumentWindowSubrole = "AXDocumentWindow" diff --git a/src/logic/Application.swift b/src/logic/Application.swift index f323824f..96d09f92 100644 --- a/src/logic/Application.swift +++ b/src/logic/Application.swift @@ -67,39 +67,34 @@ class Application: NSObject { } } - func manuallyUpdateWindows(_ group: DispatchGroup? = nil) { - // TODO: this method manually checks windows, but will not find windows on other Spaces - retryAxCallUntilTimeout(group, 5) { [weak self] in + func manuallyUpdateWindows() { + retryAxCallUntilTimeout(5) { [weak self] in guard let self else { return } var atLeastOneActualWindow = false - if let axWindows_ = try self.axUiElement!.windows(), !axWindows_.isEmpty { - // bug in macOS: sometimes the OS returns multiple duplicate windows (e.g. Mail.app starting at login) - let uniqueWindows = Array(Set(axWindows_)) - if uniqueWindows.isEmpty { return } - for axWindow in uniqueWindows { - if let wid = try axWindow.cgWindowId() { - let title = try axWindow.title() - let subrole = try axWindow.subrole() - let role = try axWindow.role() - let size = try axWindow.size() - let level = try wid.level() - if AXUIElement.isActualWindow(self, wid, level, title, subrole, role, size) { - let isFullscreen = try axWindow.isFullscreen() - let isMinimized = try axWindow.isMinimized() - let position = try axWindow.position() - atLeastOneActualWindow = true - DispatchQueue.main.async { [weak self] in - guard let self else { return } - if let window = (Windows.list.first { $0.isEqualRobust(axWindow, wid) }) { - window.title = window.bestEffortTitle(title) - window.size = size - window.isFullscreen = isFullscreen - window.isMinimized = isMinimized - window.position = position - } else { - let window = self.addWindow(axWindow, wid, title, isFullscreen, isMinimized, position, size) - App.app.refreshOpenUi([window], .refreshUiAfterExternalEvent) - } + let axWindows = try self.axUiElement!.allWindows(self.pid) + for axWindow in axWindows { + if let wid = try axWindow.cgWindowId() { + let title = try axWindow.title() + let subrole = try axWindow.subrole() + let role = try axWindow.role() + let size = try axWindow.size() + let level = try wid.level() + if AXUIElement.isActualWindow(self, wid, level, title, subrole, role, size) { + let isFullscreen = try axWindow.isFullscreen() + let isMinimized = try axWindow.isMinimized() + let position = try axWindow.position() + atLeastOneActualWindow = true + DispatchQueue.main.async { [weak self] in + guard let self else { return } + if let window = (Windows.list.first { $0.isEqualRobust(axWindow, wid) }) { + window.title = window.bestEffortTitle(title) + window.size = size + window.isFullscreen = isFullscreen + window.isMinimized = isMinimized + window.position = position + } else { + let window = self.addWindow(axWindow, wid, title, isFullscreen, isMinimized, position, size) + App.app.refreshOpenUi([window], .refreshUiAfterExternalEvent) } } } diff --git a/src/logic/Applications.swift b/src/logic/Applications.swift index 8332c8ea..45462fdc 100644 --- a/src/logic/Applications.swift +++ b/src/logic/Applications.swift @@ -4,23 +4,8 @@ import ApplicationServices class Applications { static var list = [Application]() - static func manuallyUpdateWindowsFor2s() { - let group = DispatchGroup() - manuallyUpdateWindows(group) - _ = group.wait(wallTimeout: .now() + .seconds(2)) - } - - static func manuallyUpdateWindows(_ group: DispatchGroup? = nil) { - for app in list { - if app.runningApplication.isFinishedLaunching && app.runningApplication.activationPolicy != .prohibited { - app.manuallyUpdateWindows(group) - } - } - } - static func initialDiscovery() { addInitialRunningApplications() - addInitialRunningApplicationsWindows() RunningApplicationsEvents.observe() } @@ -28,25 +13,6 @@ class Applications { addRunningApplications(NSWorkspace.shared.runningApplications) } - static func addInitialRunningApplicationsWindows() { - let otherSpaces = Spaces.otherSpaces() - // the CGSAddWindowsToSpaces trick stopped working starting with macOS 12.2 - // see https://github.com/lwouis/alt-tab-macos/issues/1324 - if otherSpaces.count > 0, #unavailable(macOS 12.2) { - let windowsOnCurrentSpace = Spaces.windowsInSpaces([Spaces.currentSpaceId]) - let windowsOnOtherSpaces = Spaces.windowsInSpaces(otherSpaces) - let windowsOnlyOnOtherSpaces = Array(Set(windowsOnOtherSpaces).subtracting(windowsOnCurrentSpace)) - if windowsOnlyOnOtherSpaces.count > 0 { - // on initial launch, we use private APIs to bring windows from other spaces into the current space, observe them, then remove them from the current space - CGSAddWindowsToSpaces(CGS_CONNECTION, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId]) - Applications.manuallyUpdateWindowsFor2s() - CGSRemoveWindowsFromSpaces(CGS_CONNECTION, windowsOnlyOnOtherSpaces as NSArray, [Spaces.currentSpaceId]) - } - } else { - Applications.manuallyUpdateWindows() - } - } - static func addRunningApplications(_ runningApps: [NSRunningApplication]) { runningApps.forEach { let bundleIdentifier = $0.bundleIdentifier diff --git a/src/logic/events/AccessibilityEvents.swift b/src/logic/events/AccessibilityEvents.swift index 32b89fdb..b9338ab0 100644 --- a/src/logic/events/AccessibilityEvents.swift +++ b/src/logic/events/AccessibilityEvents.swift @@ -118,16 +118,6 @@ fileprivate func focusedWindowChanged(_ element: AXUIElement, _ pid: pid_t) thro } } } - DispatchQueue.main.async { - if let app = Applications.find(pid) { - // work-around for apps started "hidden" like in Login Items with the "Hide" checkbox, or with `open -j` - // these apps report isHidden=false, don't generate windowCreated events initially, and have a delay before their windows are created - // our only recourse is to manually check their windows once they emit - if (!app.hasBeenActiveOnce) { - app.manuallyUpdateWindows() - } - } - } } else { DispatchQueue.main.async { Applications.find(pid)?.focusedWindow = nil diff --git a/src/logic/events/SpacesEvents.swift b/src/logic/events/SpacesEvents.swift index e74adc5b..1dc75f7e 100644 --- a/src/logic/events/SpacesEvents.swift +++ b/src/logic/events/SpacesEvents.swift @@ -14,6 +14,6 @@ class SpacesEvents { // also, updating windows on Space transition works around an issue with Safari where its fullscreen windows spawn not in fullscreen. // resize/move events happen and the window is still not fullscreen. AltTab doesn't get informed that the window is later fullscreen. // updating on Space change helps correct the window to being fullscreen - Applications.manuallyUpdateWindows() + Applications.manuallyUpdateAllAppsWindows() } } diff --git a/unit-tests/AXUIElementTests.swift b/unit-tests/AXUIElementTests.swift new file mode 100644 index 00000000..2c05ffde --- /dev/null +++ b/unit-tests/AXUIElementTests.swift @@ -0,0 +1,66 @@ +//import XCTest +// +//final class ImageScalingTests: XCTestCase { +// /// 20ms +// func testWindowsByBruteForce() throws { +// measure(options: options) { +// let _ = windowsByBruteForce(41414) +// } +// } +// +// private var options: XCTMeasureOptions = { +// let o = XCTMeasureOptions() +// o.iterationCount = 10 +// return o +// }() +//} +// +//func windowsByBruteForce(_ pid: pid_t) -> [AXUIElement] { +// // we use this to call _AXUIElementCreateWithRemoteToken; we reuse the object for performance +// // tests showed that this remoteToken is 20 bytes: 4 + 4 + 4 + 8; the order of bytes matters +// var remoteToken = Data(count: 20) +// remoteToken.replaceSubrange(0..<4, with: withUnsafeBytes(of: pid) { Data($0) }) +// remoteToken.replaceSubrange(4..<8, with: withUnsafeBytes(of: Int32(0)) { Data($0) }) +// remoteToken.replaceSubrange(8..<12, with: withUnsafeBytes(of: Int32(0x636f636f)) { Data($0) }) +// var axWindows = [AXUIElement]() +// // we iterate to 1000 as a tradeoff between performance, and missing windows of long-lived processes +// for axUiElementId: AXUIElementID in 0..<1000 { +// remoteToken.replaceSubrange(12..<20, with: withUnsafeBytes(of: axUiElementId) { Data($0) }) +// if let axUiElement = _AXUIElementCreateWithRemoteToken(remoteToken as CFData)?.takeRetainedValue(), +// let subrole = try? axUiElement.subrole(), +// [kAXStandardWindowSubrole, kAXDialogSubrole].contains(subrole) { +// axWindows.append(axUiElement) +// } +// } +// return axWindows +//} +// +//typealias AXUIElementID = UInt +// +//enum AxError: Error { +// case runtimeError +//} +// +//@_silgen_name("_AXUIElementCreateWithRemoteToken") @discardableResult +//func _AXUIElementCreateWithRemoteToken(_ data: CFData) -> Unmanaged? +// +//extension AXUIElement { +// func axCallWhichCanThrow(_ result: AXError, _ successValue: inout T) throws -> T? { +// switch result { +// case .success: return successValue +// // .cannotComplete can happen if the app is unresponsive; we throw in that case to retry until the call succeeds +// case .cannotComplete: throw AxError.runtimeError +// // for other errors it's pointless to retry +// default: return nil +// } +// } +// +// func attribute(_ key: String, _ _: T.Type) throws -> T? { +// var value: AnyObject? +// return try axCallWhichCanThrow(AXUIElementCopyAttributeValue(self, key as CFString, &value), &value) as? T +// } +// +// func subrole() throws -> String? { +// return try attribute(kAXSubroleAttribute, String.self) +// } +//} \ No newline at end of file