DroidBridge 0.2.0: device choice, rotation, foldable cover screen, faster entry

README.md

@@ -9,10 +9,14 @@ The text clipboard is shared both ways. An open-source take on DeskDock, for mac
 - Real Android pointer and keyboard: the device sees a USB mouse and keyboard (UHID).
 - Nothing to install on the device: the Mac starts a small server over adb.
 - Text clipboard sync in both directions.
-- Works with several displays: only edges that don't lead to another display count.
+- Works with several displays: arrange the device against one or two of them, as in System
+  Settings > Displays; only the stretches it touches lead to it.
+- Follows the device when it turns, and foldables when they switch screens (e.g. a Galaxy Z Fold
+  closed, on its cover screen).
+- Several Android devices on adb: pick the one to use in the menu.
 - English and Brazilian Portuguese.
 
-Status: early (0.1). USB only for now; Wi-Fi and file drag-and-drop are planned.
+Status: early (0.2). USB only for now; Wi-Fi and file drag-and-drop are planned.
 
 ## Requirements
 

docs/architecture.md

@@ -18,7 +18,9 @@ The mouse is relative and Android accelerates it, so the Mac can't compute where
 is. The shell user can't monitor input events, and the system's cursor-position API is internal
 to `system_server`. What the shell user can do is ask the input service for a dump: it lists each
 device's hovering pointer in display coordinates, e.g.
-`DeviceId(15):[... hoveringPointers=[Pointer(id=0, MOUSE) at (0, 1835.57)]]`. A dump takes
+`DeviceId(15):[... hoveringPointers=[Pointer(id=0, MOUSE) at (0, 1835.57)]]`. Those
+coordinates are the display's unrotated (physical) ones, while the frame next to them is the rotated
+one, so the server turns them using the display orientation from the same dump. A dump takes
 about 20 ms on a recent phone, so the server only takes one (at most every 30 ms) while the pointer
 moves toward the edge that leads back to the Mac. Once the pointer touches that edge and keeps
 being pushed against it, the server tells the Mac, with the position along the edge.
@@ -36,7 +38,9 @@ When the Mac cursor is pushed out of a free screen edge, the Mac freezes its cur
 (`CGAssociateMouseAndMouseCursorPosition(false)`), hides it and sends `ENTER` with the position
 along the edge. The server pushes the pointer into the corner of the entry edge, then walks it along
 the edge, measuring with the dump, until it is within about 12 px of the matching position
-(0.3-0.5 s). From then on the Mac forwards raw mouse deltas (times the speed setting), buttons,
+(0.1-0.2 s; the pixels-per-count gain of each axis is learned on earlier entries). When the
+screen turns or a foldable switches screens, the server sends the new size again and the Mac
+reshapes the device in its arrangement, keeping the sides that touch displays. From then on the Mac forwards raw mouse deltas (times the speed setting), buttons,
 scrolling and keys, and swallows them locally through a `CGEventTap`.
 
 ## The Mac side

mac/Resources/en.lproj/Localizable.strings

@@ -1,8 +1,8 @@
 "status.noAdb" = "adb not found: install Android platform-tools";
 "status.waiting" = "Waiting for an Android device (USB debugging)";
 "status.connecting" = "Connecting…";
-"status.connected" = "Connected: %@";
-"status.onDevice" = "Controlling %@";
+"status.connected" = "Connected over %2$@: %1$@";
+"status.onDevice" = "Controlling %1$@ (%2$@)";
 "status.failed" = "Could not connect: %@";
 "menu.grantAccessibility" = "Allow Accessibility access…";
 "menu.share" = "Share mouse and keyboard";
@@ -31,3 +31,7 @@
 "passage.drop" = "Drop to attach to %1$@: %2$@ edge, from %3$d%% to %4$d%% of its %5$@.";
 "passage.enters" = "The pointer enters the Android device through the side that touches the display, at the matching position.";
 "passage.dropNone" = "Not touching any display: dropping it here puts it back where it was.";
+"status.chooseDevice" = "Several Android devices: choose one in Device";
+"transport.usb" = "USB";
+"transport.wifi" = "Wi-Fi";
+"menu.device" = "Device";

mac/Resources/pt-BR.lproj/Localizable.strings

@@ -1,8 +1,8 @@
 "status.noAdb" = "adb não encontrado: instale o Android platform-tools";
 "status.waiting" = "Aguardando um Android (depuração USB)";
 "status.connecting" = "Conectando…";
-"status.connected" = "Conectado: %@";
-"status.onDevice" = "Controlando %@";
+"status.connected" = "Conectado por %2$@: %1$@";
+"status.onDevice" = "Controlando %1$@ (%2$@)";
 "status.failed" = "Não foi possível conectar: %@";
 "menu.grantAccessibility" = "Permitir acesso de Acessibilidade…";
 "menu.share" = "Compartilhar mouse e teclado";
@@ -31,3 +31,7 @@
 "passage.drop" = "Solte para encostar em %1$@: borda %2$@, de %3$d%% a %4$d%% da %5$@.";
 "passage.enters" = "O ponteiro entra no Android pelo lado que encosta no monitor, na posição correspondente.";
 "passage.dropNone" = "Longe dos monitores: soltando aqui, ele volta para onde estava.";
+"status.chooseDevice" = "Vários Androids conectados: escolha um em Aparelho";
+"transport.usb" = "USB";
+"transport.wifi" = "Wi-Fi";
+"menu.device" = "Aparelho";

mac/Sources/DroidBridge/AppDelegate.swift

@@ -145,6 +145,20 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate {
         share.state = Settings.enabled ? .on : .off
         menu.addItem(share)
 
+        if link.devices.count > 1 || Settings.deviceSerial != nil {
+            let devices = NSMenuItem(title: L("menu.device"), action: nil, keyEquivalent: "")
+            let deviceMenu = NSMenu()
+            for d in link.devices {
+                let via = [d.usbSerial != nil ? L("transport.usb") : nil, d.wirelessSerial != nil ? L("transport.wifi") : nil]
+                    .compactMap { $0 }.joined(separator: " + ")
+                let i = item("\(d.model) (\(via))", #selector(chooseDevice(_:)))
+                i.representedObject = d.id
+                i.state = (Settings.deviceSerial ?? (link.devices.count == 1 ? d.id : nil)) == d.id ? .on : .off
+                deviceMenu.addItem(i)
+            }
+            devices.submenu = deviceMenu
+            menu.addItem(devices)
+        }
         menu.addItem(item(L("menu.arrange"), #selector(openArrangement)))
 
         let speed = NSMenuItem(title: L("menu.speed"), action: nil, keyEquivalent: "")
@@ -177,7 +191,9 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate {
         case .waiting: return L("status.waiting")
         case .connecting: return L("status.connecting")
         case let .connected(model, _, _):
-            return String(format: L(capture.isRemote ? "status.onDevice" : "status.connected"), model)
+            let key = capture.isRemote ? "status.onDevice" : "status.connected"
+            return String(format: L(key), model, L(link.transport == .usb ? "transport.usb" : "transport.wifi"))
+        case .chooseDevice: return L("status.chooseDevice")
         case let .failed(reason): return String(format: L("status.failed"), reason)
         }
     }
@@ -199,6 +215,12 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate {
         if !Settings.enabled { capture.returnToMac() }
     }
 
+    @objc private func chooseDevice(_ sender: NSMenuItem) {
+        guard let id = sender.representedObject as? String else { return }
+        capture.returnToMac()
+        link.select(id)
+    }
+
     @objc private func openArrangement() {
         arrangementWindow.show()
     }

mac/Sources/DroidBridge/ArrangementWindow.swift

@@ -9,8 +9,15 @@ final class ArrangementModel: ObservableObject {
     @Published var displays: [DisplayInfo] = []
     @Published var device: CGRect?
     @Published var deviceName: String?
+    /// The device screen as it is now; when it turns or a foldable changes screens, its rectangle follows.
     @Published var deviceSize: CGSize? {
-        didSet { if oldValue != deviceSize { reload() } }
+        didSet {
+            guard oldValue != deviceSize else { return }
+            reload()
+            if let r = device, let turned = EdgeGeometry.reshaped(device: r, aspect: aspect, displays: bounds) {
+                commit(turned)
+            }
+        }
     }
     /// Drag offset of the device, in global points.
     @Published var drag: CGSize?

mac/Sources/DroidBridge/DeviceLink.swift

@@ -10,15 +10,32 @@ final class DeviceLink {
         case waiting
         case connecting(String)
         case connected(model: String, width: Int, height: Int)
+        case chooseDevice
         case failed(String)
     }
 
-    static let serverVersion = "0.1.0"
+    /// An Android device, possibly reachable both by USB and over Wi-Fi.
+    struct Device: Equatable {
+        /// The hardware serial (ro.serialno), the same over USB and Wi-Fi.
+        let id: String
+        let model: String
+        var usbSerial: String?
+        var wirelessSerial: String?
+    }
+
+    enum Transport { case usb, wifi }
+
+    static let serverVersion = "0.2.0"
     private static let remoteJar = "/data/local/tmp/droidbridge-server.jar"
     private let log = Logger(subsystem: "dev.droidbridge", category: "link")
 
     var onState: ((State) -> Void)?
     var onMessage: ((Wire.Message) -> Void)?
+    /// The devices adb sees now (main queue).
+    private(set) var devices: [Device] = []
+    /// How the current connection reaches the device.
+    private(set) var transport: Transport = .usb
+    private var hardwareSerials: [String: String] = [:]
 
     private(set) var state: State = .waiting {
         didSet { if state != oldValue { let s = state; DispatchQueue.main.async { self.onState?(s) } } }
@@ -66,23 +83,79 @@ final class DeviceLink {
                 Thread.sleep(forTimeInterval: 3)
                 continue
             }
-            guard let device = firstDevice(adb) else {
-                state = .waiting
+            let found = listDevices(adb)
+            DispatchQueue.main.async { self.devices = found }
+            let chosen = Settings.deviceSerial
+            let target: Device?
+            if let chosen {
+                target = found.first { $0.id == chosen }
+            } else {
+                target = found.count == 1 ? found.first : nil
+            }
+            guard let device = target, let serial = device.usbSerial ?? device.wirelessSerial else {
+                if found.count > 1, chosen == nil || !found.contains(where: { $0.id == chosen }) {
+                    state = chosen == nil ? .chooseDevice : .waiting
+                } else {
+                    state = .waiting
+                }
                 Thread.sleep(forTimeInterval: 2)
                 continue
             }
+            transport = device.usbSerial != nil ? .usb : .wifi
             do {
-                try connect(adb: adb, serial: device)
+                try connect(adb: adb, serial: serial)
                 readUntilClosed()
             } catch {
                 log.error("connect failed: \(error.localizedDescription, privacy: .public)")
                 state = .failed(error.localizedDescription)
             }
             disconnect()
-            Thread.sleep(forTimeInterval: 2)
+            Thread.sleep(forTimeInterval: 1)
+        }
+    }
+
+    /// Picks a device from the menu; reconnects to it.
+    func select(_ id: String) {
+        Settings.deviceSerial = id
+        queue.async { [weak self] in self?.dropConnection() }
+    }
+
+    /// Ends the current connection; the loop picks the device again.
+    private func dropConnection() {
+        writeQueue.sync {
+            if socket >= 0 { shutdown(socket, SHUT_RDWR) }
         }
     }
 
+    // MARK: - Devices
+
+    private func listDevices(_ adb: String) -> [Device] {
+        guard let out = try? run(adb, ["devices", "-l"], timeout: 5) else { return [] }
+        var byID: [String: Device] = [:]
+        var order: [String] = []
+        for e in AdbParsing.devices(out) where e.state == "device" {
+            let id: String
+            if e.wireless {
+                if let cached = hardwareSerials[e.serial] {
+                    id = cached
+                } else if let s = try? run(adb, ["-s", e.serial, "shell", "getprop", "ro.serialno"], timeout: 5)
+                    .trimmingCharacters(in: .whitespacesAndNewlines), !s.isEmpty {
+                    hardwareSerials[e.serial] = s
+                    id = s
+                } else {
+                    id = e.serial
+                }
+            } else {
+                id = e.serial
+            }
+            var d = byID[id] ?? Device(id: id, model: e.model ?? id, usbSerial: nil, wirelessSerial: nil)
+            if e.wireless { d.wirelessSerial = e.serial } else { d.usbSerial = e.serial }
+            if byID[id] == nil { order.append(id) }
+            byID[id] = d
+        }
+        return order.compactMap { byID[$0] }
+    }
+
     private func connect(adb: String, serial: String) throws {
         self.serial = serial
         state = .connecting(serial)
@@ -161,26 +234,30 @@ final class DeviceLink {
 
     // MARK: - Helpers
 
-    private func firstDevice(_ adb: String) -> String? {
-        guard let out = try? run(adb, ["devices"]) else { return nil }
-        for line in out.split(separator: "\n").dropFirst() {
-            let parts = line.split(whereSeparator: { $0 == "\t" || $0 == " " })
-            if parts.count >= 2, parts[1] == "device" { return String(parts[0]) }
-        }
-        return nil
-    }
-
     @discardableResult
-    private func run(_ tool: String, _ args: [String]) throws -> String {
+    private func run(_ tool: String, _ args: [String], timeout: TimeInterval = 60) throws -> String {
         let p = Process()
         p.executableURL = URL(fileURLWithPath: tool)
         p.arguments = args
         let out = Pipe()
         p.standardOutput = out
         p.standardError = out
+        let done = DispatchSemaphore(value: 0)
+        p.terminationHandler = { _ in done.signal() }
         try p.run()
-        let data = out.fileHandleForReading.readDataToEndOfFile()
-        p.waitUntilExit()
+        var data = Data()
+        let reader = DispatchQueue.global()
+        let readDone = DispatchSemaphore(value: 0)
+        reader.async {
+            data = out.fileHandleForReading.readDataToEndOfFile()
+            readDone.signal()
+        }
+        if done.wait(timeout: .now() + timeout) == .timedOut {
+            p.terminate()
+            _ = done.wait(timeout: .now() + 1)
+            throw LinkError("adb \(args.joined(separator: " ")): timed out")
+        }
+        _ = readDone.wait(timeout: .now() + 2)
         let text = String(decoding: data, as: UTF8.self)
         guard p.terminationStatus == 0 else { throw LinkError("adb \(args.joined(separator: " ")): \(text)") }
         return text

mac/Sources/DroidBridge/Settings.swift

@@ -22,6 +22,12 @@ struct Settings {
         }
     }
 
+    /// The device to use (hardware serial); nil = the only one connected.
+    static var deviceSerial: String? {
+        get { d.string(forKey: "deviceSerial") }
+        set { d.set(newValue, forKey: "deviceSerial") }
+    }
+
     static var speed: Double {
         get { d.object(forKey: "speed") as? Double ?? 3.0 }
         set { d.set(newValue, forKey: "speed") }

mac/Sources/DroidBridgeCore/AdbParsing.swift

@@ -0,0 +1,22 @@
+/// Parsers for adb output.
+public enum AdbParsing {
+    public struct Entry: Equatable {
+        public let serial: String
+        public let state: String
+        public let model: String?
+        public let usb: Bool
+
+        /// Reached over the network: "ip:port" or an mDNS name.
+        public var wireless: Bool { serial.contains(":") || serial.contains("._adb-tls-connect.") }
+    }
+
+    /// `adb devices -l`
+    public static func devices(_ output: String) -> [Entry] {
+        output.split(separator: "\n").dropFirst().compactMap { line in
+            let parts = line.split(whereSeparator: { $0 == " " || $0 == "\t" }).map(String.init)
+            guard parts.count >= 2 else { return nil }
+            let model = parts.first { $0.hasPrefix("model:") }.map { String($0.dropFirst(6)).replacingOccurrences(of: "_", with: " ") }
+            return Entry(serial: parts[0], state: parts[1], model: model, usb: parts.contains { $0.hasPrefix("usb:") })
+        }
+    }
+}

mac/Sources/DroidBridgeCore/EdgeGeometry.swift

@@ -254,6 +254,30 @@ public enum EdgeGeometry {
         }
     }
 
+    /// When the device screen changes shape (it turns, or a foldable switches between its screens),
+    /// the rectangle takes the new aspect, keeping its longer side and the sides that touch displays.
+    /// Nil when nothing changes or the new rectangle would overlap a display or touch none.
+    public static func reshaped(device r: CGRect, aspect: CGFloat, displays: [CGRect]) -> CGRect? {
+        guard aspect > 0, r.height > 0, abs(r.width / r.height - aspect) / aspect > 0.02 else { return nil }
+        let long = max(r.width, r.height)
+        let w = aspect >= 1 ? long : long * aspect
+        let h = aspect >= 1 ? long / aspect : long
+        var x = r.midX - w / 2
+        var y = r.midY - h / 2
+        for p in passages(device: r, displays: displays) {
+            switch p.edge {
+            case .right: x = r.minX
+            case .left: x = r.maxX - w
+            case .top: y = r.maxY - h
+            case .bottom: y = r.minY
+            }
+        }
+        let next = CGRect(x: x, y: y, width: w, height: h)
+        let overlaps = displays.contains { $0.intersection(next).width > 0.5 && $0.intersection(next).height > 0.5 }
+        guard !overlaps, !passages(device: next, displays: displays).isEmpty else { return nil }
+        return next
+    }
+
     /// The default device rectangle: against the outermost display's free right edge, centered,
     /// `size` times that display's height.
     public static func defaultDevice(displays: [CGRect], aspect: CGFloat, size: CGFloat = 0.6) -> CGRect? {