From bb8a5ceaa913587ac273707f9ff395789255de63 Mon Sep 17 00:00:00 2001 From: Louis Pontoise Date: Tue, 16 Jun 2020 13:00:08 +0900 Subject: [PATCH] fix: show android emulator window (closes #376) --- src/api-wrappers/AXUIElement.swift | 40 ++++++++++++++++------ src/api-wrappers/Sysctl.swift | 6 ++++ src/logic/Application.swift | 2 +- src/logic/events/AccessibilityEvents.swift | 6 ++-- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/api-wrappers/AXUIElement.swift b/src/api-wrappers/AXUIElement.swift index 523d5d03..5ee1c38f 100644 --- a/src/api-wrappers/AXUIElement.swift +++ b/src/api-wrappers/AXUIElement.swift @@ -45,23 +45,43 @@ extension AXUIElement { return nil } - func isActualWindow(_ bundleIdentifier: String?, _ wid: CGWindowID, _ isOnNormalLevel: Bool, _ title: String?, _ subrole: String?, _ role: String?) -> Bool { - // Some non-windows have cgWindowId == 0 (e.g. windows of apps starting at login with the checkbox "Hidden" checked) + func isActualWindow(_ runningApp: NSRunningApplication, _ wid: CGWindowID, _ isOnNormalLevel: Bool, _ title: String?, _ subrole: String?, _ role: String?) -> Bool { // Some non-windows have title: nil (e.g. some OS elements) // Some non-windows have subrole: nil (e.g. some OS elements), "AXUnknown" (e.g. Bartender), "AXSystemDialog" (e.g. Intellij tooltips) // Minimized windows or windows of a hidden app have subrole "AXDialog" // Activity Monitor main window subrole is "AXDialog" for a brief moment at launch; it then becomes "AXStandardWindow" - // CGWindowLevel == .normalWindow helps filter out iStats Pro and other top-level pop-overs + + // Some non-windows have cgWindowId == 0 (e.g. windows of apps starting at login with the checkbox "Hidden" checked) return wid != 0 && - // don't show floating windows + // CGWindowLevel == .normalWindow helps filter out iStats Pro and other top-level pop-overs, and floating windows isOnNormalLevel && (["AXStandardWindow", "AXDialog"].contains(subrole) || - // All Steam windows have subrole == AXUnknown - // some dropdown menus are not desirable; they have title == "", or sometimes role == nil when switching between menus quickly - (bundleIdentifier == "com.valvesoftware.steam" && title != "" && role != nil) || - // Firefox fullscreen video have subrole == AXUnknown if fullscreen'ed when the base window is not fullscreen - (bundleIdentifier == "org.mozilla.firefox" && role == "AXWindow") - ) + steam(runningApp, title, role) || + firefoxFullscreenVideo(runningApp, role) || + androidEmulator(runningApp, title)) + } + + private func steam(_ runningApp: NSRunningApplication, _ title: String?, _ role: String?) -> Bool { + // All Steam windows have subrole == AXUnknown + // some dropdown menus are not desirable; they have title == "", or sometimes role == nil when switching between menus quickly + return runningApp.bundleIdentifier == "com.valvesoftware.steam" && title != "" && role != nil + } + + private func firefoxFullscreenVideo(_ runningApp: NSRunningApplication, _ role: String?) -> Bool { + // Firefox fullscreen video have subrole == AXUnknown if fullscreen'ed when the base window is not fullscreen + return runningApp.bundleIdentifier == "org.mozilla.firefox" && role == "AXWindow" + } + + private func androidEmulator(_ runningApp: NSRunningApplication, _ title: String?) -> Bool { + // android emulator small vertical menu is a "window" with empty title; we exclude it + if title != "", + // NSRunningApplication provides no way to identify the emulator; we pattern match on its KERN_PROCARGS + runningApp.bundleIdentifier == nil, + let executablePath = Sysctl.run([CTL_KERN, KERN_PROCARGS, runningApp.processIdentifier]) { + // example path: ~/Library/Android/sdk/emulator/qemu/darwin-x86_64/qemu-system-x86_64 + return executablePath.range(of: "qemu-system[^/]*$", options: .regularExpression, range: nil, locale: nil) != nil + } + return false } func isOnNormalLevel(_ wid: CGWindowID) -> Bool { diff --git a/src/api-wrappers/Sysctl.swift b/src/api-wrappers/Sysctl.swift index a3732db3..8f82879d 100644 --- a/src/api-wrappers/Sysctl.swift +++ b/src/api-wrappers/Sysctl.swift @@ -9,6 +9,12 @@ public struct Sysctl { return run(name, { $0.baseAddress?.withMemoryRebound(to: T.self, capacity: 1) { $0.pointee } }) } + static func run(_ keys: [Int32]) -> String? { + return data(keys)?.withUnsafeBufferPointer() { dataPointer -> String? in + dataPointer.baseAddress.flatMap { String(validatingUTF8: $0) } + } + } + private static func run(_ name: String, _ fn: (UnsafeBufferPointer) -> R?) -> R? { return keys(name).flatMap { keys in data(keys)?.withUnsafeBufferPointer() { fn($0) } } } diff --git a/src/logic/Application.swift b/src/logic/Application.swift index f71b4cf2..09b8a082 100644 --- a/src/logic/Application.swift +++ b/src/logic/Application.swift @@ -60,7 +60,7 @@ class Application: NSObject { let subrole = try $0.subrole() let role = try $0.role() let isOnNormalLevel = $0.isOnNormalLevel(wid) - if $0.isActualWindow(self.runningApplication.bundleIdentifier, wid, isOnNormalLevel, title, subrole, role) { + if $0.isActualWindow(self.runningApplication, wid, isOnNormalLevel, title, subrole, role) { return ($0, wid, title, try $0.isFullscreen(), try $0.isMinimized(), try $0.position()) } } diff --git a/src/logic/events/AccessibilityEvents.swift b/src/logic/events/AccessibilityEvents.swift index bafd76aa..89422903 100644 --- a/src/logic/events/AccessibilityEvents.swift +++ b/src/logic/events/AccessibilityEvents.swift @@ -21,7 +21,7 @@ func retryUntilTimeout(_ fn: @escaping () throws -> Void, _ startTime: DispatchT } func handleEvent(_ type: String, _ element: AXUIElement) throws { - debugPrint("Accessibility event", type, element) + debugPrint("Accessibility event", type, try element.title()) // events are handled concurrently, thus we check that the app is still running if let pid = try element.pid() { switch type { @@ -102,7 +102,7 @@ private func windowCreated(_ element: AXUIElement, _ pid: pid_t) throws { // a window being un-minimized can trigger kAXWindowCreatedNotification if Windows.list.firstIndexThatMatches(element, wid) == nil, let runningApp = NSRunningApplication(processIdentifier: pid), - element.isActualWindow(runningApp.bundleIdentifier, wid, isOnNormalLevel, axTitle, subrole, role), + element.isActualWindow(runningApp, wid, isOnNormalLevel, axTitle, subrole, role), let app = (Applications.list.first { $0.runningApplication.processIdentifier == pid }) { let window = Window(element, app, wid, axTitle, isFullscreen, isMinimized, position) Windows.list.insertAndScaleRecycledPool(window, at: 0) @@ -127,7 +127,7 @@ private func focusedWindowChanged(_ element: AXUIElement, _ pid: pid_t) throws { Windows.list.insertAndScaleRecycledPool(Windows.list.remove(at: existingIndex), at: 0) App.app.refreshOpenUi([Windows.list[0], Windows.list[existingIndex]]) } else if let runningApp = NSRunningApplication(processIdentifier: pid), - element.isActualWindow(runningApp.bundleIdentifier, wid, isOnNormalLevel, axTitle, subrole, role), + element.isActualWindow(runningApp, wid, isOnNormalLevel, axTitle, subrole, role), let app = (Applications.list.first { $0.runningApplication.processIdentifier == pid }) { Windows.list.insertAndScaleRecycledPool(Window(element, app, wid, axTitle, isFullscreen, isMinimized, position), at: 0) App.app.refreshOpenUi([Windows.list[0]])