From 9d78700ab7f46e2487fe0eb225cb458b72d8426c Mon Sep 17 00:00:00 2001 From: lwouis Date: Wed, 12 Feb 2025 17:52:04 +0100 Subject: [PATCH] feat: improve performance and lower resources consumption avoid unnecessary AX calls and batch calls when possible --- src/api-wrappers/AXUIElement.swift | 229 ++++++++++++--------- src/api-wrappers/CGWindowID.swift | 2 +- src/logic/Application.swift | 10 +- src/logic/Window.swift | 1 - src/logic/events/AccessibilityEvents.swift | 82 ++++---- 5 files changed, 175 insertions(+), 149 deletions(-) diff --git a/src/api-wrappers/AXUIElement.swift b/src/api-wrappers/AXUIElement.swift index fc6ae975..842e39ff 100644 --- a/src/api-wrappers/AXUIElement.swift +++ b/src/api-wrappers/AXUIElement.swift @@ -99,6 +99,29 @@ extension AXUIElement { return try axCallWhichCanThrow(AXUIElementCopyAttributeValue(self, key as CFString, &value), &value) as? T } + func windowAttributes() throws -> (String?, String?, String?, Bool, Bool)? { + let attributes = [ + kAXTitleAttribute, + kAXRoleAttribute, + kAXSubroleAttribute, + kAXMinimizedAttribute, + kAXFullscreenAttribute, + ] + var values: CFArray? + if let array = ((try axCallWhichCanThrow(AXUIElementCopyMultipleAttributeValues(self, attributes as CFArray, [], &values), &values)) as? Array) { + return ( + array[0] as? String, + array[1] as? String, + array[2] as? String, + // if the value is nil, we return false. This avoid returning Bool?; simplifies things + (array[3] as? Bool) ?? false, + // if the value is nil, we return false. This avoid returning Bool?; simplifies things + (array[4] as? Bool) ?? false + ) + } + return nil + } + private func value(_ key: String, _ target: T, _ type: AXValueType) throws -> T? { if let a = try attribute(key, AXValue.self) { var value = target @@ -110,6 +133,102 @@ extension AXUIElement { return nil } + func title() throws -> String? { + return try attribute(kAXTitleAttribute, String.self) + } + + // periphery:ignore + func parent() throws -> AXUIElement? { + return try attribute(kAXParentAttribute, AXUIElement.self) + } + + func children() throws -> [AXUIElement]? { + return try attribute(kAXChildrenAttribute, [AXUIElement].self) + } + + func isMinimized() throws -> Bool { + // if the AX call doesn't return, we return false. This avoid returning Bool?; simplifies things + return try attribute(kAXMinimizedAttribute, Bool.self) == true + } + + func isFullscreen() throws -> Bool { + // if the AX call doesn't return, we return false. This avoid returning Bool?; simplifies things + return try attribute(kAXFullscreenAttribute, Bool.self) == true + } + + func focusedWindow() throws -> AXUIElement? { + return try attribute(kAXFocusedWindowAttribute, AXUIElement.self) + } + + func role() throws -> String? { + return try attribute(kAXRoleAttribute, String.self) + } + + func subrole() throws -> String? { + return try attribute(kAXSubroleAttribute, String.self) + } + + func closeButton() throws -> AXUIElement? { + return try attribute(kAXCloseButtonAttribute, AXUIElement.self) + } + + func appIsRunning() throws -> Bool? { + return try attribute(kAXIsApplicationRunningAttribute, Bool.self) + } + + /// doesn't return windows on other Spaces + /// use windowsByBruteForce if you want those + func windows() throws -> [AXUIElement] { + let windows = try attribute(kAXWindowsAttribute, [AXUIElement].self) + if let windows, + !windows.isEmpty { + // bug in macOS: sometimes the OS returns multiple duplicate windows (e.g. Mail.app starting at login) + let uniqueWindows = Array(Set(windows)) + if !uniqueWindows.isEmpty { + return uniqueWindows + } + } + return [] + } + + func position() throws -> CGPoint? { + return try value(kAXPositionAttribute, CGPoint.zero, .cgPoint) + } + + func size() throws -> CGSize? { + return try value(kAXSizeAttribute, CGSize.zero, .cgSize) + } + + /// we combine both the normal approach and brute-force to get all possible windows + /// with only normal approach: we miss other-Spaces windows + /// with only brute-force approach: we miss windows when the app launches (e.g. launch Note.app: first window is not found by brute-force) + func allWindows(_ pid: pid_t) throws -> [AXUIElement] { + let aWindows = try! windows() + let bWindows = AXUIElement.windowsByBruteForce(pid) + return Array(Set(aWindows + bWindows)) + } + + /// brute-force getting the windows of a process by iterating over AXUIElementID one by one + private static func windowsByBruteForce(_ pid: pid_t) -> [AXUIElement] { + // we use this to call _AXUIElementCreateWithRemoteToken; we reuse the object for performance + // tests showed that this remoteToken is 20 bytes: 4 + 4 + 4 + 8; the order of bytes matters + var remoteToken = Data(count: 20) + remoteToken.replaceSubrange(0..<4, with: withUnsafeBytes(of: pid) { Data($0) }) + remoteToken.replaceSubrange(4..<8, with: withUnsafeBytes(of: Int32(0)) { Data($0) }) + remoteToken.replaceSubrange(8..<12, with: withUnsafeBytes(of: Int32(0x636f636f)) { Data($0) }) + var axWindows = [AXUIElement]() + // we iterate to 1000 as a tradeoff between performance, and missing windows of long-lived processes + for axUiElementId: AXUIElementID in 0..<1000 { + remoteToken.replaceSubrange(12..<20, with: withUnsafeBytes(of: axUiElementId) { Data($0) }) + if let axUiElement = _AXUIElementCreateWithRemoteToken(remoteToken as CFData)?.takeRetainedValue(), + let subrole = try? axUiElement.subrole(), + [kAXStandardWindowSubrole, kAXDialogSubrole].contains(subrole) { + axWindows.append(axUiElement) + } + } + return axWindows + } + static func isActualWindow(_ app: Application, _ wid: CGWindowID, _ level: CGWindowLevel, _ title: String?, _ subrole: String?, _ role: String?, _ size: CGSize?) -> Bool { // Some non-windows have title: nil (e.g. some OS elements) // Some non-windows have subrole: nil (e.g. some OS elements), "AXUnknown" (e.g. Bartender), "AXSystemDialog" (e.g. Intellij tooltips) @@ -274,104 +393,18 @@ extension AXUIElement { return (app.bundleIdentifier?.hasPrefix("com.autodesk.AutoCAD") ?? false) && subrole == kAXDocumentWindowSubrole } - func position() throws -> CGPoint? { - return try value(kAXPositionAttribute, CGPoint.zero, .cgPoint) - } - - func size() throws -> CGSize? { - return try value(kAXSizeAttribute, CGSize.zero, .cgSize) - } - - func title() throws -> String? { - return try attribute(kAXTitleAttribute, String.self) - } - - // periphery:ignore - func parent() throws -> AXUIElement? { - return try attribute(kAXParentAttribute, AXUIElement.self) - } - - func children() throws -> [AXUIElement]? { - return try attribute(kAXChildrenAttribute, [AXUIElement].self) - } - - /// we combine both the normal approach and brute-force to get all possible windows - /// with only normal approach: we miss other-Spaces windows - /// with only brute-force approach: we miss windows when the app launches (e.g. launch Note.app: first window is not found by brute-force) - func allWindows(_ pid: pid_t) throws -> [AXUIElement] { - let aWindows = try! windows() - let bWindows = AXUIElement.windowsByBruteForce(pid) - return Array(Set(aWindows + bWindows)) - } - - /// doesn't return windows on other Spaces - /// use windowsByBruteForce if you want those - func windows() throws -> [AXUIElement] { - let windows = try attribute(kAXWindowsAttribute, [AXUIElement].self) - if let windows, - !windows.isEmpty { - // bug in macOS: sometimes the OS returns multiple duplicate windows (e.g. Mail.app starting at login) - let uniqueWindows = Array(Set(windows)) - if !uniqueWindows.isEmpty { - return uniqueWindows - } - } - return [] - } - - /// brute-force getting the windows of a process by iterating over AXUIElementID one by one - private static func windowsByBruteForce(_ pid: pid_t) -> [AXUIElement] { - // we use this to call _AXUIElementCreateWithRemoteToken; we reuse the object for performance - // tests showed that this remoteToken is 20 bytes: 4 + 4 + 4 + 8; the order of bytes matters - var remoteToken = Data(count: 20) - remoteToken.replaceSubrange(0..<4, with: withUnsafeBytes(of: pid) { Data($0) }) - remoteToken.replaceSubrange(4..<8, with: withUnsafeBytes(of: Int32(0)) { Data($0) }) - remoteToken.replaceSubrange(8..<12, with: withUnsafeBytes(of: Int32(0x636f636f)) { Data($0) }) - var axWindows = [AXUIElement]() - // we iterate to 1000 as a tradeoff between performance, and missing windows of long-lived processes - for axUiElementId: AXUIElementID in 0..<1000 { - remoteToken.replaceSubrange(12..<20, with: withUnsafeBytes(of: axUiElementId) { Data($0) }) - if let axUiElement = _AXUIElementCreateWithRemoteToken(remoteToken as CFData)?.takeRetainedValue(), - let subrole = try? axUiElement.subrole(), - [kAXStandardWindowSubrole, kAXDialogSubrole].contains(subrole) { - axWindows.append(axUiElement) - } - } - return axWindows - } - - func isMinimized() throws -> Bool { - return try attribute(kAXMinimizedAttribute, Bool.self) == true - } - - func isFullscreen() throws -> Bool { - return try attribute(kAXFullscreenAttribute, Bool.self) == true - } - - func focusedWindow() throws -> AXUIElement? { - return try attribute(kAXFocusedWindowAttribute, AXUIElement.self) - } - - func role() throws -> String? { - return try attribute(kAXRoleAttribute, String.self) - } - - func subrole() throws -> String? { - return try attribute(kAXSubroleAttribute, String.self) - } - - func appIsRunning() throws -> Bool? { - return try attribute(kAXIsApplicationRunningAttribute, Bool.self) - } - - func closeButton() throws -> AXUIElement? { - return try attribute(kAXCloseButtonAttribute, AXUIElement.self) - } - func focusWindow() { performAction(kAXRaiseAction) } + func setAttribute(_ key: String, _ value: Any) { + AXUIElementSetAttributeValue(self, key as CFString, value as CFTypeRef) + } + + func performAction(_ action: String) { + AXUIElementPerformAction(self, action as CFString) + } + func subscribeToNotification(_ axObserver: AXObserver, _ notification: String, _ callback: (() -> Void)? = nil) throws { let result = AXObserverAddNotification(axObserver, self, notification as CFString, nil) if result == .success || result == .notificationAlreadyRegistered { @@ -380,14 +413,6 @@ extension AXUIElement { throw AxError.runtimeError } } - - func setAttribute(_ key: String, _ value: Any) { - AXUIElementSetAttributeValue(self, key as CFString, value as CFTypeRef) - } - - func performAction(_ action: String) { - AXUIElementPerformAction(self, action as CFString) - } } enum AxError: Error { diff --git a/src/api-wrappers/CGWindowID.swift b/src/api-wrappers/CGWindowID.swift index 206db046..9ea27197 100644 --- a/src/api-wrappers/CGWindowID.swift +++ b/src/api-wrappers/CGWindowID.swift @@ -5,7 +5,7 @@ extension CGWindowID { cgProperty("kCGSWindowTitle", String.self) } - func level() throws -> CGWindowLevel { + func level() -> CGWindowLevel { var level = CGWindowLevel(0) CGSGetWindowLevel(CGS_CONNECTION, self, &level) return level diff --git a/src/logic/Application.swift b/src/logic/Application.swift index b96a8675..6c4e641e 100644 --- a/src/logic/Application.swift +++ b/src/logic/Application.swift @@ -73,15 +73,11 @@ class Application: NSObject { var atLeastOneActualWindow = false let axWindows = try self.axUiElement!.allWindows(self.pid) for axWindow in axWindows { - if let wid = try axWindow.cgWindowId() { - let title = try axWindow.title() - let subrole = try axWindow.subrole() - let role = try axWindow.role() + if let wid = try axWindow.cgWindowId(), + let (title, role, subrole, isMinimized, isFullscreen) = try axWindow.windowAttributes() { let size = try axWindow.size() - let level = try wid.level() + let level = wid.level() if AXUIElement.isActualWindow(self, wid, level, title, subrole, role, size) { - let isFullscreen = try axWindow.isFullscreen() - let isMinimized = try axWindow.isMinimized() let position = try axWindow.position() atLeastOneActualWindow = true DispatchQueue.main.async { [weak self] in diff --git a/src/logic/Window.swift b/src/logic/Window.swift index e3ba29f7..604daba2 100644 --- a/src/logic/Window.swift +++ b/src/logic/Window.swift @@ -35,7 +35,6 @@ class Window { ] init(_ axUiElement: AXUIElement, _ application: Application, _ wid: CGWindowID, _ axTitle: String?, _ isFullscreen: Bool, _ isMinimized: Bool, _ position: CGPoint?, _ size: CGSize?) { - // TODO: make a efficient batched AXUIElementCopyMultipleAttributeValues call once for each window, and store the values self.axUiElement = axUiElement self.application = application cgWindowId = wid diff --git a/src/logic/events/AccessibilityEvents.swift b/src/logic/events/AccessibilityEvents.swift index acf470e2..bcf04929 100644 --- a/src/logic/events/AccessibilityEvents.swift +++ b/src/logic/events/AccessibilityEvents.swift @@ -65,20 +65,16 @@ fileprivate func applicationHiddenOrShown(_ pid: pid_t, _ type: String) throws { } fileprivate func windowCreated(_ element: AXUIElement, _ pid: pid_t) throws { - if let wid = try element.cgWindowId() { - let axTitle = try element.title() - let subrole = try element.subrole() - let role = try element.role() - let isFullscreen = try element.isFullscreen() - let isMinimized = try element.isMinimized() - let level = try wid.level() + if let wid = try element.cgWindowId(), + let (title, role, subrole, isMinimized, isFullscreen) = try element.windowAttributes() { let position = try element.position() let size = try element.size() + let level = wid.level() DispatchQueue.main.async { if let app = Applications.find(pid), NSRunningApplication(processIdentifier: pid) != nil { if (!Windows.list.contains { $0.isEqualRobust(element, wid) }) && - AXUIElement.isActualWindow(app, wid, level, axTitle, subrole, role, size) { - let window = Window(element, app, wid, axTitle, isFullscreen, isMinimized, position, size) + AXUIElement.isActualWindow(app, wid, level, title, subrole, role, size) { + let window = Window(element, app, wid, title, isFullscreen, isMinimized, position, size) Windows.appendAndUpdateFocus(window) Windows.cycleFocusedWindowIndex(1) App.app.refreshOpenUi([window], .refreshUiAfterExternalEvent) @@ -94,27 +90,31 @@ fileprivate func focusedWindowChanged(_ element: AXUIElement, _ pid: pid_t) thro // photoshop will focus a window *after* you focus another app // we check that a focused window happens within an active app if runningApp.isActive { - let axTitle = try element.title() - let subrole = try element.subrole() - let role = try element.role() - let isFullscreen = try element.isFullscreen() - let isMinimized = try element.isMinimized() - let level = try wid.level() - let position = try element.position() - let size = try element.size() DispatchQueue.main.async { guard let app = Applications.find(pid) else { return } - // if the window is shown by alt-tab, we mark her as focused for this app + // if the window is shown by alt-tab, we mark it as focused for this app // this avoids issues with dialogs, quicklook, etc (see scenarios from #1044 and #2003) if let w = (Windows.list.first { $0.isEqualRobust(element, wid) }) { app.focusedWindow = w } if let windows = Windows.updateLastFocus(element, wid) { App.app.refreshOpenUi(windows, .refreshUiAfterExternalEvent) - } else if AXUIElement.isActualWindow(app, wid, level, axTitle, subrole, role, size) { - let window = Window(element, app, wid, axTitle, isFullscreen, isMinimized, position, size) - Windows.appendAndUpdateFocus(window) - App.app.refreshOpenUi([window], .refreshUiAfterExternalEvent) + } else { + AXUIElement.retryAxCallUntilTimeout { + if let (title, role, subrole, isMinimized, isFullscreen) = try element.windowAttributes() { + let position = try element.position() + let size = try element.size() + let level = wid.level() + DispatchQueue.main.async { + if (!Windows.list.contains { $0.isEqualRobust(element, wid) }), + AXUIElement.isActualWindow(app, wid, level, title, subrole, role, size) { + let window = Window(element, app, wid, title, isFullscreen, isMinimized, position, size) + Windows.appendAndUpdateFocus(window) + App.app.refreshOpenUi([window], .refreshUiAfterExternalEvent) + } + } + } + } } } } @@ -158,11 +158,14 @@ fileprivate func windowMiniaturizedOrDeminiaturized(_ element: AXUIElement, _ ty fileprivate func windowTitleChanged(_ element: AXUIElement) throws { if let wid = try element.cgWindowId() { AXUIElement.retryAxCallUntilTimeoutDebounced(.windowTitleChanged, wid) { - let newTitle = try element.title() - DispatchQueue.main.async { - if let window = (Windows.list.first { $0.isEqualRobust(element, wid) }), newTitle != window.title { - window.title = window.bestEffortTitle(newTitle) - App.app.refreshOpenUi([window], .refreshUiAfterExternalEvent) + if let (title, _, _, isMinimized, isFullscreen) = try element.windowAttributes() { + DispatchQueue.main.async { + if let window = (Windows.list.first { $0.isEqualRobust(element, wid) }), title != window.title { + window.title = window.bestEffortTitle(title) + window.isMinimized = isMinimized + window.isFullscreen = isFullscreen + App.app.refreshOpenUi([window], .refreshUiAfterExternalEvent) + } } } } @@ -178,18 +181,21 @@ fileprivate func windowResizedOrMoved(_ element: AXUIElement) throws { } func updateWindowSizeAndPositionAndFullscreen(_ element: AXUIElement, _ wid: CGWindowID, _ window: Window?) throws { - let isFullscreen = try element.isFullscreen() - let size = try element.size() - let position = try element.position() - DispatchQueue.main.async { - if let window = (window != nil ? window : (Windows.list.first { $0.isEqualRobust(element, wid) })) { - window.size = size - window.position = position - if window.isFullscreen != isFullscreen { - window.isFullscreen = isFullscreen - App.app.checkIfShortcutsShouldBeDisabled(window, nil) + if let (title, _, _, isMinimized, isFullscreen) = try element.windowAttributes() { + let size = try element.size() + let position = try element.position() + DispatchQueue.main.async { + if let window = (window != nil ? window : (Windows.list.first { $0.isEqualRobust(element, wid) })) { + window.title = title + window.size = size + window.position = position + window.isMinimized = isMinimized + if window.isFullscreen != isFullscreen { + window.isFullscreen = isFullscreen + App.app.checkIfShortcutsShouldBeDisabled(window, nil) + } + App.app.refreshOpenUi([window], .refreshUiAfterExternalEvent) } - App.app.refreshOpenUi([window], .refreshUiAfterExternalEvent) } } }