From 9a936e6286e523f6c3b1edb82701a38c4196cb0d Mon Sep 17 00:00:00 2001 From: wavezhang <832911+wavezhang@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:38:47 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat(virtual-screen):=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=8C=96=E4=BB=BB=E5=8A=A1=E6=95=B4=E4=BD=93?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=9C=A8=E5=90=8E=E5=8F=B0=E8=99=9A=E6=8B=9F?= =?UTF-8?q?=E5=B1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 任务级启动选项,现有 JS 脚本零改动即可在虚拟屏运行:ScriptRuntime 通过 DisplayRoute 统一路由截图、点击/滑动/按键、控件查找、应用启动与 applink。 - 底座:VirtualDisplay 由 Shizuku shell-uid 常驻用户服务持有(参考 Mika), IUserService.aidl 新增事务 51-61,VirtualDisplayFlags 按 SDK 组合隐藏标志位 - 启用入口:设置页全局开关、脚本菜单单次运行、定时任务三态配置、project.json 的 useFeatures: ["virtualScreen"],以及对应的 Intent extras - 控件查找:getWindowsOnAllDisplays() + DisplayWindowScope 按 displayId 限定, 与 auto.setWindowFilter 取交集 - 查看器:侧边栏「工具」入口,实时预览、手势/按键/文本/启动应用/保存 PNG、 新建与释放屏幕,Activity 不导出 - 兼容性:虚拟屏字段不进入 ExecutionConfig 主构造,保持 com.stardust Rhino 别名承诺的 copy() 签名;同步 KSP/API surface baseline 与静态引用白名单 - 文档与示例:docs/virtual-screen/README.md、两份 JS 示例、46 个定向单测用例 --- app/src/main/AndroidManifest.xml | 6 + .../autojs/core/shizuku/IUserService.aidl | 18 + ...\217\347\244\272\344\276\213 [v6.8.0+].js" | 51 ++ ...\254\347\244\272\344\276\213 [v6.8.0+].js" | 47 ++ .../core/accessibility/AccessibilityBridge.kt | 127 ++++- .../accessibility/AccessibilityService.kt | 45 ++ .../accessibility/SimpleActionAutomator.kt | 130 ++++- .../window/DisplayWindowScope.kt | 78 +++ .../window/WindowRootProvider.kt | 17 +- .../autojs/autojs/core/shizuku/UserService.kt | 43 ++ .../virtualscreen/DisplayCoordinateMapper.kt | 68 +++ .../autojs/core/virtualscreen/DisplayRoute.kt | 149 +++++ .../core/virtualscreen/ShellCommandRunner.kt | 30 ++ .../core/virtualscreen/ShellVirtualDisplay.kt | 507 ++++++++++++++++++ .../core/virtualscreen/VirtualDisplayFlags.kt | 87 +++ .../virtualscreen/VirtualScreenFramePump.kt | 211 ++++++++ .../virtualscreen/VirtualScreenManager.kt | 109 ++++ .../virtualscreen/VirtualScreenSession.kt | 320 +++++++++++ .../autojs/execution/ExecutionConfig.kt | 63 ++- .../autojs/autojs/external/ScriptIntents.kt | 20 + .../org/autojs/autojs/model/script/Scripts.kt | 26 + .../autojs/autojs/runtime/ScriptRuntime.kt | 25 + .../org/autojs/autojs/runtime/api/Images.java | 13 +- .../autojs/runtime/api/WrappedShizuku.kt | 5 +- .../runtime/api/modules/ScriptModuleLayer.kt | 2 +- .../autojs/runtime/api/modules/app/App.kt | 46 +- .../autojs/runtime/api/modules/shell/Shell.kt | 53 +- .../virtualscreen/VirtualScreenModule.kt | 90 ++++ .../storage/database/TimedTaskDatabase.java | 11 +- .../org/autojs/autojs/timing/TimedTask.java | 16 +- .../explorer/ExplorerBuiltInActionHandler.kt | 5 + .../ui/explorer/ExplorerItemViewHolder.kt | 1 + .../autojs/ui/main/drawer/DrawerFragment.kt | 11 + .../ui/timing/TimedTaskSettingActivity.kt | 23 + .../ui/virtualscreen/VirtualScreenView.kt | 170 ++++++ .../VirtualScreenViewerActivity.kt | 424 +++++++++++++++ .../layout/activity_timed_task_setting.xml | 62 +++ .../layout/activity_virtual_screen_viewer.xml | 209 ++++++++ app/src/main/res/menu/menu_script_options.xml | 3 + app/src/main/res/values-zh/strings.xml | 53 ++ app/src/main/res/values/strings.xml | 53 ++ app/src/main/res/xml/fragment_preferences.xml | 7 + .../window/DisplayWindowScopeTest.kt | 110 ++++ .../DisplayCoordinateMapperTest.kt | 97 ++++ .../virtualscreen/VirtualDisplayFlagsTest.kt | 76 +++ .../api/modules/ScriptModuleOrderTest.kt | 9 +- .../memory/static-reference-whitelist.txt | 1 + docs/virtual-screen/README.md | 89 +++ sample/virtual-screen/virtual-screen-demo.js | 113 ++++ sample/virtual-screen/zero-change-demo.js | 66 +++ tools/augment/baseline/registrations.json | 12 + .../baseline/script-modules-surface.txt | 7 + tools/augment/baseline/surface-jvm.txt | 27 + 53 files changed, 3963 insertions(+), 78 deletions(-) create mode 100644 "app/src/main/assets-app/sample/\350\231\232\346\213\237\345\261\217\345\271\225/\345\220\216\345\217\260\350\231\232\346\213\237\345\261\217\347\244\272\344\276\213 [v6.8.0+].js" create mode 100644 "app/src/main/assets-app/sample/\350\231\232\346\213\237\345\261\217\345\271\225/\351\233\266\346\224\271\345\212\250\350\204\232\346\234\254\347\244\272\344\276\213 [v6.8.0+].js" create mode 100644 app/src/main/java/org/autojs/autojs/core/accessibility/window/DisplayWindowScope.kt create mode 100644 app/src/main/java/org/autojs/autojs/core/virtualscreen/DisplayCoordinateMapper.kt create mode 100644 app/src/main/java/org/autojs/autojs/core/virtualscreen/DisplayRoute.kt create mode 100644 app/src/main/java/org/autojs/autojs/core/virtualscreen/ShellCommandRunner.kt create mode 100644 app/src/main/java/org/autojs/autojs/core/virtualscreen/ShellVirtualDisplay.kt create mode 100644 app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualDisplayFlags.kt create mode 100644 app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenFramePump.kt create mode 100644 app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenManager.kt create mode 100644 app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenSession.kt create mode 100644 app/src/main/java/org/autojs/autojs/runtime/api/modules/virtualscreen/VirtualScreenModule.kt create mode 100644 app/src/main/java/org/autojs/autojs/ui/virtualscreen/VirtualScreenView.kt create mode 100644 app/src/main/java/org/autojs/autojs/ui/virtualscreen/VirtualScreenViewerActivity.kt create mode 100644 app/src/main/res/layout/activity_virtual_screen_viewer.xml create mode 100644 app/src/test/java/org/autojs/autojs/core/accessibility/window/DisplayWindowScopeTest.kt create mode 100644 app/src/test/java/org/autojs/autojs/core/virtualscreen/DisplayCoordinateMapperTest.kt create mode 100644 app/src/test/java/org/autojs/autojs/core/virtualscreen/VirtualDisplayFlagsTest.kt create mode 100644 docs/virtual-screen/README.md create mode 100644 sample/virtual-screen/virtual-screen-demo.js create mode 100644 sample/virtual-screen/zero-change-demo.js diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index d364ea73d0..9f96a87ee3 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -607,6 +607,12 @@ + + + { window -> A11yWindowFacts.displayId(window) } + + private fun activeProvider(): WindowRootProvider = + if (displayScope.isScoped) allDisplayRootProvider else rootProvider + + @Volatile + private var lastEmptyScopeDiagnosticMillis = 0L + /** - * The current query settings as one immutable policy; rebuilt by [setMode], [setWindowFilter] - * and [flags]. - * zh-CN: 当前查询设置构成的单个不可变策略; 由 [setMode], [setWindowFilter] 与 [flags] 重建. + * The current query settings as one immutable policy; rebuilt by [setMode], [setWindowFilter], + * [setDisplayScope] and [flags]. + * zh-CN: 当前查询设置构成的单个不可变策略; 由 [setMode], [setWindowFilter], [setDisplayScope] 与 [flags] 重建. */ @Volatile var windowRootPolicy: A11yWindowRootPolicy = WindowRootPolicy() @@ -172,30 +195,106 @@ abstract class AccessibilityBridge( * zh-CN: 查询所针对的根节点, 不含 `null`: 通过窗口过滤器的窗口的根节点; 否则当前模式下的活动窗口根节点 * (系统不提供时, 由 [FLAG_APP_WINDOWS_FALLBACK] 决定的代替根节点); 否则为空. */ - fun windowRoots(): List = rootProvider.roots(windowRootPolicy) + fun windowRoots(): List { + val roots = activeProvider().roots(windowRootPolicy) + // When a virtual-screen task finds nothing, make the distinction visible: is the system + // reporting windows for that display at all, or did the selector simply not match? + // zh-CN: 虚拟屏任务查不到节点时, 让日志能区分「系统根本没上报该屏窗口」与「选择器条件不匹配」. + if (roots.isEmpty() && displayScope.isScoped) logEmptyScopeDiagnostic() + return roots + } /** - * The service's windows, unfiltered, in the framework's order; empty without a service. - * zh-CN: 服务的全部窗口, 未过滤, 按框架的顺序; 无服务时为空. + * The service's windows in the framework's order, unfiltered by the script's window filter; + * while a display scope is bound, only the windows of that display are returned. + * zh-CN: 按框架顺序排列的服务窗口, 不受脚本窗口过滤器影响; 绑定 display 作用域时只返回该屏的窗口. */ - fun windows(): List = rootProvider.windows() + fun windows(): List { + val windows = activeProvider().windows() + val scope = displayScope.scopedDisplayId + if (scope == DisplayWindowScope.NO_SCOPE) return windows + return windows.filter { window -> + runCatching { A11yWindowFacts.displayId(window) == scope }.getOrDefault(false) + } + } /** * The first of [windowRoots], resolved lazily (with a window filter, only the windows up to the * first one that has a root are queried). * zh-CN: [windowRoots] 的首个根节点, 惰性解析 (有窗口过滤器时只查询到首个有根节点的窗口为止). */ - fun getRootInCurrentWindow(): AccessibilityNodeInfo? = rootProvider.currentRoot(windowRootPolicy) + fun getRootInCurrentWindow(): AccessibilityNodeInfo? = activeProvider().currentRoot(windowRootPolicy) /** * The active window root of the current mode, ignoring the window filter (unlike - * [getRootInCurrentWindow]); [FLAG_APP_WINDOWS_FALLBACK] applies. + * [getRootInCurrentWindow]); [FLAG_APP_WINDOWS_FALLBACK] applies. While a display scope is + * bound, the framework's active root is the *main* screen's, so the top root of the bound + * display is returned instead. * zh-CN: 当前模式下的活动窗口根节点, 忽略窗口过滤器 (与 [getRootInCurrentWindow] 不同); [FLAG_APP_WINDOWS_FALLBACK] 生效. + * 绑定 display 作用域时, 框架的活动窗口根节点属于**主屏**, 因此改返回绑定屏最上层窗口的根节点. */ - fun getRootInActiveWindow(): AccessibilityNodeInfo? = rootProvider.activeRoot(windowRootPolicy) + fun getRootInActiveWindow(): AccessibilityNodeInfo? = + // `rootInActiveWindow` is the *default display's* active window and cannot be scoped. + // While a task is bound to a virtual display, "the active window" must be the top window + // of that display, i.e. the first root passing the scoped filter. + // zh-CN: `rootInActiveWindow` 是**默认屏**的活动窗口, 无法限定 display; 任务绑定虚拟屏时, + // 「活动窗口」应是该屏的最上层窗口, 即通过作用域过滤器的首个根节点. + if (displayScope.isScoped) getRootInCurrentWindow() + else activeProvider().activeRoot(windowRootPolicy) + /** + * Installs/replaces the script's own window filter. When a display scope is bound, the filter + * is combined with (not substituted for) that scope, so a zero-change script cannot read the + * physical screen by accident. zh-CN: 安装/替换脚本自己的窗口过滤器; 已绑定 display 作用域时是叠加而非替换, + * 避免零改动脚本意外读到物理主屏. + */ fun setWindowFilter(windowFilter: WindowFilter?) { - windowRootPolicy = windowRootPolicy.copy(filter = windowFilter?.let { { info -> it.filter(info) } }) + displayScope.setScriptFilter(windowFilter?.let { filter -> { info -> filter.filter(info) } }) + rebuildWindowRootFilter() + } + + /** + * Binds every accessibility query of this runtime to one display and switches to the + * all-displays window source. Called from `ScriptRuntime.initPrologue()` before modules are + * mounted, so existing JavaScript needs no changes. Pass [DisplayWindowScope.NO_SCOPE] to clear. + * + * zh-CN: 把本运行时的所有无障碍查询绑定到某个显示屏, 并切换到全屏窗口来源; 由 + * `ScriptRuntime.initPrologue()` 在模块挂载前调用, 已有 JS 无需修改. 传 [DisplayWindowScope.NO_SCOPE] 可清除. + */ + fun setDisplayScope(id: Int) { + displayScope.setDisplayScope(id) + rebuildWindowRootFilter() + if (id == DisplayWindowScope.NO_SCOPE) { + Log.i(TAG, "Accessibility display scope cleared") + return + } + Log.i(TAG, "Accessibility display scope = $id; windows by display: ${windowsByDisplaySummary()}") + } + + private fun rebuildWindowRootFilter() { + windowRootPolicy = windowRootPolicy.copy(filter = displayScope.effectiveFilter()) + } + + private fun windowsByDisplaySummary(): String { + if (service == null) return "service not bound" + return runCatching { + allDisplayRootProvider.windows() + .groupingBy { window -> runCatching { A11yWindowFacts.displayId(window) }.getOrDefault(-1) } + .eachCount() + .toString() + }.getOrDefault("unavailable") + } + + /** Throttled: selector polling asks for roots many times a second. zh-CN: 节流: 选择器轮询每秒会多次查询根节点. */ + private fun logEmptyScopeDiagnostic() { + val now = SystemClock.elapsedRealtime() + if (now - lastEmptyScopeDiagnosticMillis < 5_000L) return + lastEmptyScopeDiagnosticMillis = now + Log.w( + TAG, + "No accessibility roots on display ${displayScope.scopedDisplayId}; " + + "windows by display: ${windowsByDisplaySummary()}", + ) } abstract val infoProvider: ActivityInfoProvider @@ -218,6 +317,8 @@ abstract class AccessibilityBridge( companion object { + private const val TAG = "AccessibilityBridge" + const val MODE_NORMAL = 0 const val MODE_FAST = 1 diff --git a/app/src/main/java/org/autojs/autojs/core/accessibility/AccessibilityService.kt b/app/src/main/java/org/autojs/autojs/core/accessibility/AccessibilityService.kt index 7b93dfc883..3e9154f8ad 100644 --- a/app/src/main/java/org/autojs/autojs/core/accessibility/AccessibilityService.kt +++ b/app/src/main/java/org/autojs/autojs/core/accessibility/AccessibilityService.kt @@ -1,6 +1,7 @@ package org.autojs.autojs.core.accessibility import android.content.Intent +import android.os.Build import android.os.Handler import android.os.Looper import android.os.SystemClock @@ -10,6 +11,7 @@ import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityEvent.TYPE_VIEW_FOCUSED import android.view.accessibility.AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED import android.view.accessibility.AccessibilityNodeInfo +import android.view.accessibility.AccessibilityWindowInfo import org.autojs.autojs.core.accessibility.AccessibilityTool.Companion.DEFAULT_A11Y_SERVICE_START_TIMEOUT import org.autojs.autojs.core.accessibility.AccessibilityTool.Companion.DEFAULT_A11Y_SERVICE_STOP_TIMEOUT import org.autojs.autojs.core.accessibility.SimpleActionAutomator.Companion.AccessibilityEventCallback @@ -167,6 +169,49 @@ open class AccessibilityService : android.accessibilityservice.AccessibilityServ return runCatching { super.getRootInActiveWindow() }.getOrNull() } + /** + * Windows of **every** display, flattened from `getWindowsOnAllDisplays()` (API 30+). + * + * `AccessibilityService.getWindows()` only reports the windows of `Display.DEFAULT_DISPLAY` + * (AOSP `AccessibilityInteractionClient#getWindows` takes just the default display's entry of + * the all-displays map). A task running on a virtual display therefore sees an empty window + * list, and every `selector()` / `UiObject` / `auto.windows()` lookup finds nothing. This + * property is the multi-display source such tasks are rooted in. It requires the service to + * opt into `FLAG_RETRIEVE_INTERACTIVE_WINDOWS`, which the manifest already does. + * + * Falls back to the default-display list on API < 30 or when the framework call fails, so the + * classic behaviour is never lost. + * + * zh-CN: **所有**显示屏的窗口, 由 `getWindowsOnAllDisplays()` (API 30+) 展开而来. + * `AccessibilityService.getWindows()` 只报告默认屏的窗口 (AOSP 的 `AccessibilityInteractionClient#getWindows` + * 只取全屏窗口映射里默认屏那一项), 因此在虚拟屏上运行的任务看到的窗口列表为空, 所有 `selector()` / + * `UiObject` / `auto.windows()` 查找都找不到元素. 本属性即这类任务的根节点来源, 需服务声明 + * `FLAG_RETRIEVE_INTERACTIVE_WINDOWS` (Manifest 已声明). API < 30 或框架调用失败时回退到默认屏列表, + * 不破坏原有行为. + */ + val windowsOnAllDisplays: List + get() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return defaultDisplayWindows() + return try { + val byDisplay = getWindowsOnAllDisplays() + buildList { + for (i in 0 until byDisplay.size()) { + val perDisplay = byDisplay.valueAt(i) ?: continue + addAll(perDisplay) + } + } + } catch (error: Throwable) { + Log.w(TAG, "getWindowsOnAllDisplays failed; using the default display only", error) + defaultDisplayWindows() + } + } + + private fun defaultDisplayWindows(): List = try { + windows + } catch (_: Throwable) { + emptyList() + } + override fun onDestroy() { disposeService("onDestroy") super.onDestroy() diff --git a/app/src/main/java/org/autojs/autojs/core/accessibility/SimpleActionAutomator.kt b/app/src/main/java/org/autojs/autojs/core/accessibility/SimpleActionAutomator.kt index 2ac52ea655..4cf874181f 100644 --- a/app/src/main/java/org/autojs/autojs/core/accessibility/SimpleActionAutomator.kt +++ b/app/src/main/java/org/autojs/autojs/core/accessibility/SimpleActionAutomator.kt @@ -5,10 +5,13 @@ package org.autojs.autojs.core.accessibility import android.accessibilityservice.AccessibilityService.GestureResultCallback import android.accessibilityservice.GestureDescription import android.content.Context +import android.graphics.Path +import android.graphics.PathMeasure import android.graphics.Rect import android.os.Build import android.os.Handler import android.util.Log +import android.view.KeyEvent import android.view.accessibility.AccessibilityNodeInfo import android.view.inputmethod.InputMethodManager import androidx.annotation.RequiresApi @@ -59,7 +62,12 @@ class SimpleActionAutomator(private val accessibilityBridge: AccessibilityBridge } init { - accessibilityBridge.tap = AccessibilityBridge.Tap { x, y -> mGlobalActionAutomatorForDevicePixels.click(x, y) } + // UiObject.clickBounds() falls back to a device-pixel tap; on a virtual display it must go + // to that display instead of the accessibility gesture service (main display only). + accessibilityBridge.tap = AccessibilityBridge.Tap { x, y -> + if (scriptRuntime.displayRoute.isVirtual) scriptRuntime.displayRoute.tap(x, y) + else mGlobalActionAutomatorForDevicePixels.click(x, y) + } } private val isRunningPackageSelf @@ -125,13 +133,13 @@ class SimpleActionAutomator(private val accessibilityBridge: AccessibilityBridge fun appendText(target: ActionTarget, text: String) = performAction(target.createAction(UiObject.ACTION_APPEND_TEXT, text)) @ScriptInterface - fun back() = mGlobalActionAutomator.back() + fun back() = virtualKey(KeyEvent.KEYCODE_BACK) ?: mGlobalActionAutomator.back() @ScriptInterface - fun home() = mGlobalActionAutomator.home() + fun home() = virtualKey(KeyEvent.KEYCODE_HOME) ?: mGlobalActionAutomator.home() @ScriptInterface - fun recents() = mGlobalActionAutomator.recents() + fun recents() = virtualKey(KeyEvent.KEYCODE_APP_SWITCH) ?: mGlobalActionAutomator.recents() @ScriptInterface fun notifications() = mGlobalActionAutomator.notifications() @@ -152,7 +160,7 @@ class SimpleActionAutomator(private val accessibilityBridge: AccessibilityBridge fun takeScreenshot() = mGlobalActionAutomator.takeScreenshot() @ScriptInterface - fun headsethook() = mGlobalActionAutomator.headsethook() + fun headsethook() = virtualKey(KeyEvent.KEYCODE_HEADSETHOOK) ?: mGlobalActionAutomator.headsethook() @ScriptInterface fun switchToInputMethod(packageName: String): Boolean { @@ -195,30 +203,56 @@ class SimpleActionAutomator(private val accessibilityBridge: AccessibilityBridge fun dismissNotificationShade() = mGlobalActionAutomator.dismissNotificationShade() @ScriptInterface - fun gesture(start: Long, duration: Long, points: Array) = mGlobalActionAutomatorForGesture.gesture(start, duration, *points) + fun gesture(start: Long, duration: Long, points: Array): Boolean = + if (scriptRuntime.displayRoute.isVirtual) dispatchPolyline(duration, points) + else mGlobalActionAutomatorForGesture.gesture(start, duration, *points) @ScriptInterface @JvmOverloads - fun gestureAsync(start: Long, duration: Long, points: Array, callback: GestureResultCallback? = null) = mGlobalActionAutomatorForGesture.gestureAsync(start, duration, points, callback) + fun gestureAsync(start: Long, duration: Long, points: Array, callback: GestureResultCallback? = null) { + if (scriptRuntime.displayRoute.isVirtual) { + val ok = dispatchPolyline(duration, points) + completeGesture(callback, ok, points) + return + } + mGlobalActionAutomatorForGesture.gestureAsync(start, duration, points, callback) + } @ScriptInterface - fun gestures(strokes: Array) = mGlobalActionAutomatorForGesture.gestures(*strokes) + fun gestures(strokes: Array): Boolean = + if (scriptRuntime.displayRoute.isVirtual) strokes.all { dispatchStroke(it) } + else mGlobalActionAutomatorForGesture.gestures(*strokes) @ScriptInterface @JvmOverloads - fun gesturesAsync(strokes: Array, callback: GestureResultCallback? = null) = mGlobalActionAutomatorForGesture.gesturesAsync(strokes, callback) + fun gesturesAsync(strokes: Array, callback: GestureResultCallback? = null) { + if (scriptRuntime.displayRoute.isVirtual) { + val ok = strokes.all { dispatchStroke(it) } + completeGesture(callback, ok) + return + } + mGlobalActionAutomatorForGesture.gesturesAsync(strokes, callback) + } @ScriptInterface - fun click(x: Int, y: Int) = mGlobalActionAutomatorForGesture.click(x, y) + fun click(x: Int, y: Int): Boolean = + if (scriptRuntime.displayRoute.isVirtual) scriptRuntime.displayRoute.tap(x, y) + else mGlobalActionAutomatorForGesture.click(x, y) @ScriptInterface - fun press(x: Int, y: Int, duration: Int) = mGlobalActionAutomatorForGesture.press(x, y, duration) + fun press(x: Int, y: Int, duration: Int): Boolean = + if (scriptRuntime.displayRoute.isVirtual) scriptRuntime.displayRoute.longPress(x, y, duration.coerceAtLeast(1)) + else mGlobalActionAutomatorForGesture.press(x, y, duration) @ScriptInterface - fun longClick(x: Int, y: Int) = mGlobalActionAutomatorForGesture.longClick(x, y) + fun longClick(x: Int, y: Int): Boolean = + if (scriptRuntime.displayRoute.isVirtual) scriptRuntime.displayRoute.longPress(x, y, 600) + else mGlobalActionAutomatorForGesture.longClick(x, y) @ScriptInterface - fun swipe(x1: Int, y1: Int, x2: Int, y2: Int, duration: Int) = mGlobalActionAutomatorForGesture.swipe(x1, y1, x2, y2, duration.toLong()) + fun swipe(x1: Int, y1: Int, x2: Int, y2: Int, duration: Int): Boolean = + if (scriptRuntime.displayRoute.isVirtual) scriptRuntime.displayRoute.swipe(x1, y1, x2, y2, duration.coerceAtLeast(1)) + else mGlobalActionAutomatorForGesture.swipe(x1, y1, x2, y2, duration.toLong()) @ScriptInterface fun paste(target: ActionTarget) = performAction(target.createAction(AccessibilityNodeInfo.ACTION_PASTE)) @@ -269,14 +303,69 @@ class SimpleActionAutomator(private val accessibilityBridge: AccessibilityBridge AccessibilityService.removeAllAccessibilityEventCallbacks(mA11yEventOwnerId) } + /** null = not a virtual-screen run, so the caller keeps using the accessibility global action. */ + private fun virtualKey(keyCode: Int): Boolean? = + if (scriptRuntime.displayRoute.isVirtual) scriptRuntime.displayRoute.key(keyCode) else null + + /** Splits a polyline gesture into straight swipes on the virtual display. */ + private fun dispatchPolyline(duration: Long, points: Array): Boolean { + val route = scriptRuntime.displayRoute + if (points.isEmpty()) return false + if (points.size == 1) { + return route.tap(points[0].getOrElse(0) { 0 }, points[0].getOrElse(1) { 0 }) + } + val segment = (duration / (points.size - 1)).coerceAtLeast(1L).toInt() + return points.asList().zipWithNext().all { (from, to) -> + route.swipe( + from.getOrElse(0) { 0 }, from.getOrElse(1) { 0 }, + to.getOrElse(0) { 0 }, to.getOrElse(1) { 0 }, + segment, + ) + } + } + + /** Approximates one accessibility stroke by its start/end points (PathMeasure samples the path). */ + private fun dispatchStroke(stroke: GestureDescription.StrokeDescription): Boolean { + val measure = PathMeasure(stroke.path, false) + val start = FloatArray(2) + val end = FloatArray(2) + measure.getPosTan(0f, start, null) + measure.getPosTan(measure.length, end, null) + return scriptRuntime.displayRoute.swipe( + start[0].toInt(), start[1].toInt(), end[0].toInt(), end[1].toInt(), + stroke.duration.coerceIn(1L, Int.MAX_VALUE.toLong()).toInt(), + ) + } + + /** Async gesture APIs still need their callback, even when the injection was synchronous. */ + private fun completeGesture(callback: GestureResultCallback?, ok: Boolean, points: Array? = null) { + if (callback == null) return + val first = points?.firstOrNull() + val x = first?.getOrElse(0) { 0 }?.toFloat() ?: 0f + val y = first?.getOrElse(1) { 0 }?.toFloat() ?: 0f + val path = Path().apply { moveTo(x, y); lineTo(x, y) } + val description = GestureDescription.Builder() + .addStroke(GestureDescription.StrokeDescription(path, 0L, 1L)) + .build() + if (ok) callback.onCompleted(description) else callback.onCancelled(description) + } + private fun performAction(simpleAction: SimpleAction): Boolean { ensureService() - if (AccessibilityConfig.isUnintendedGuardEnabled() && isRunningPackageSelf) { - return false - } - return accessibilityBridge.windowRoots().let { roots -> - roots.isNotEmpty() && roots.map { root -> simpleAction.perform(UiObject.createRoot(root)) }.all { it /* == true */ } + val roots = accessibilityBridge.windowRoots() + if (AccessibilityConfig.isUnintendedGuardEnabled()) { + // The usage-stats / shell "foreground package" describes the physical screen. On a + // virtual display the guard must look at the windows the scoped query actually sees. + // zh-CN: 使用情况/ shell 得到的「前台包名」描述的是物理主屏; 虚拟屏上保护模式应看作用域内实际窗口. + val selfInFront = if (scriptRuntime.displayRoute.isVirtual) { + roots.any { root -> DeveloperUtils.isSelfPackage("${root.packageName}") } + } else { + isRunningPackageSelf + } + if (selfInFront) return false } + return roots.isNotEmpty() && + roots.map { root -> simpleAction.perform(UiObject.createRoot(root)) }.all { it /* == true */ } } fun setScreenMetrics(metrics: ScreenMetrics) { @@ -285,6 +374,11 @@ class SimpleActionAutomator(private val accessibilityBridge: AccessibilityBridge @RequiresApi(Build.VERSION_CODES.R) fun captureScreen(): ScriptPromiseAdapter { + // A virtual display owns its ImageReader surface, so MediaProjection (default display only) + // must not be involved. Return the same ImageWrapper type as the accessibility screenshot. + if (scriptRuntime.displayRoute.isVirtual) { + return ScriptPromiseAdapter().also { it.resolve(scriptRuntime.displayRoute.capture()) } + } ScriptRuntime.requiresApi(Build.VERSION_CODES.R) ensureService() diff --git a/app/src/main/java/org/autojs/autojs/core/accessibility/window/DisplayWindowScope.kt b/app/src/main/java/org/autojs/autojs/core/accessibility/window/DisplayWindowScope.kt new file mode 100644 index 0000000000..955ab94f12 --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/core/accessibility/window/DisplayWindowScope.kt @@ -0,0 +1,78 @@ +package org.autojs.autojs.core.accessibility.window + +/** + * A task-level window scope: which display a script's accessibility queries may see, combined + * with the window filter the script itself installed (`auto.setWindowFilter`). + * + * Both parts are kept separate on purpose. A script may freely set, replace or clear its own + * filter, but it must not be able to escape the display its task was configured to run on; + * otherwise a zero-change script running on a virtual screen would silently start reading the + * windows of the physical screen. The effective filter is the **intersection** of the two. + * + * Android-free: [W] is opaque and the display of a window is read through [displayIdOf], so the + * composition rules are unit tested on a plain JVM. + * + * zh-CN: 任务级窗口作用域: 脚本的无障碍查询被限定在哪个显示屏上, 并与脚本自己安装的窗口过滤器 + * (`auto.setWindowFilter`) 组合. 两者刻意分开保存: 脚本可以自由设置/替换/清除自己的过滤器, 但不能 + * 逃出任务被配置的显示屏, 否则零改动脚本在虚拟屏运行时会悄悄读到物理主屏的窗口. 生效的过滤器是两者的**交集**. + * 本类不依赖 Android: [W] 为不透明类型, 窗口所属 display 经 [displayIdOf] 读取, 组合规则因此可在纯 JVM 上单测. + */ +class DisplayWindowScope(private val displayIdOf: (W) -> Int) { + + companion object { + + /** No display bound: the default-display behaviour of the framework. zh-CN: 未绑定 display, 沿用框架默认屏行为. */ + const val NO_SCOPE = -1 + + } + + @Volatile + private var displayId: Int = NO_SCOPE + + @Volatile + private var scriptFilter: ((W) -> Boolean)? = null + + /** Whether this scope is bound to a particular (normally virtual) display. */ + val isScoped: Boolean get() = displayId != NO_SCOPE + + /** The bound display id, or [NO_SCOPE]. */ + val scopedDisplayId: Int get() = displayId + + /** Binds (or clears, with [NO_SCOPE]) the display of every query of this bridge. */ + fun setDisplayScope(id: Int) { + displayId = id + } + + /** Installs the script's own window filter; [null] removes it but keeps the display scope. */ + fun setScriptFilter(filter: ((W) -> Boolean)?) { + scriptFilter = filter + } + + /** + * The filter `WindowRootPolicy` should use, or `null` when neither part applies (the framework's + * original single-active-root behaviour). A throwing accessor or script filter rejects just that + * window instead of failing the whole query. + * + * zh-CN: `WindowRootPolicy` 应使用的过滤器; 两部分都不存在时为 `null` (框架原本的单一活动窗口根节点行为). + * 读取 display 或执行脚本过滤器抛异常时只拒绝该窗口, 不让整次查询失败. + */ + fun effectiveFilter(): ((W) -> Boolean)? { + val scope = displayId + val script = scriptFilter + if (scope == NO_SCOPE && script == null) return null + return { window -> + val inScope = if (scope == NO_SCOPE) true else try { + displayIdOf(window) == scope + } catch (_: Throwable) { + false + } + val passesScript = if (script == null) true else try { + script(window) + } catch (_: Throwable) { + false + } + inScope && passesScript + } + } + +} diff --git a/app/src/main/java/org/autojs/autojs/core/accessibility/window/WindowRootProvider.kt b/app/src/main/java/org/autojs/autojs/core/accessibility/window/WindowRootProvider.kt index 6618d47093..18d922b27b 100644 --- a/app/src/main/java/org/autojs/autojs/core/accessibility/window/WindowRootProvider.kt +++ b/app/src/main/java/org/autojs/autojs/core/accessibility/window/WindowRootProvider.kt @@ -18,7 +18,18 @@ typealias A11yWindowRootPolicy = WindowRootPolicy * * Created by SuperMonster003 on Sep 5, 2026. */ -class WindowRootProvider(private val service: () -> AccessibilityService?) { +class WindowRootProvider( + /** + * When true the window list comes from `service.windowsOnAllDisplays` (all displays) instead + * of `service.windows` (default display only). Used by runtimes bound to a virtual display. + * zh-CN: 为 true 时窗口列表取自 `service.windowsOnAllDisplays` (所有屏幕), 而非只含默认屏的 + * `service.windows`; 供绑定虚拟屏的运行时使用. + */ + private val allDisplays: Boolean = false, + // Last on purpose: the existing `WindowRootProvider { service }` call sites keep working. + // zh-CN: 刻意放在最后, 使原有 `WindowRootProvider { service }` 调用形式保持可用. + private val service: () -> AccessibilityService?, +) { private val resolver = WindowRootResolver(ServiceSource()) { message, error -> Log.w(TAG, message, error) } @@ -36,7 +47,9 @@ class WindowRootProvider(private val service: () -> AccessibilityService?) { private inner class ServiceSource : WindowRootResolver.Source { - override fun windows(): List? = service()?.windows + override fun windows(): List? = service()?.let { + if (allDisplays) it.windowsOnAllDisplays else it.windows + } override fun rootOf(window: AccessibilityWindowInfo): AccessibilityNodeInfo? = window.root diff --git a/app/src/main/java/org/autojs/autojs/core/shizuku/UserService.kt b/app/src/main/java/org/autojs/autojs/core/shizuku/UserService.kt index 355c6cfcef..bb7270f46e 100644 --- a/app/src/main/java/org/autojs/autojs/core/shizuku/UserService.kt +++ b/app/src/main/java/org/autojs/autojs/core/shizuku/UserService.kt @@ -10,7 +10,9 @@ import android.os.Process import android.os.ParcelFileDescriptor import android.os.RemoteException import android.util.Log +import android.view.Surface import androidx.annotation.Keep +import org.autojs.autojs.core.virtualscreen.ShellVirtualDisplay import org.autojs.autojs.model.explorer.PrivilegedExplorerDirectoryService import org.autojs.autojs.model.explorer.PrivilegedFileEngine import org.autojs.autojs.model.explorer.PrivilegedFileNative @@ -34,6 +36,16 @@ class UserService : IUserService.Stub { private val explorerDirectories = PrivilegedExplorerDirectoryService() private val lifecycle = UserServiceLifecycle(MainThreadScheduler, ::exitProcessFor) + /** + * Virtual displays live in this shell-uid process for their whole lifetime. A one-shot command + * cannot own a VirtualDisplay, so the capability is exposed through this service's AIDL. + * zh-CN: 虚拟显示屏在其整个生命周期内都由本 shell 身份进程持有, 一次性命令无法持有 VirtualDisplay. + */ + private var virtualDisplays: ShellVirtualDisplay? = null + + private fun displays(): ShellVirtualDisplay = + virtualDisplays ?: ShellVirtualDisplay(mContext).also { virtualDisplays = it } + init { lifecycle.start() } @@ -74,6 +86,8 @@ class UserService : IUserService.Stub { */ override fun destroy() { Log.i(TAG, "destroy") + // DESTROY_CONTENT_ON_REMOVAL ends the activities left on every display before the process dies. + virtualDisplays?.releaseAll() lifecycle.shutdown(UserServiceLifecycle.REASON_DESTROY) } @@ -83,6 +97,35 @@ class UserService : IUserService.Stub { lifecycle.shutdown(UserServiceLifecycle.REASON_EXIT) } + // ---- Virtual screen: shell-side implementation (docs/virtual-screen/README.md) ---- + + override fun createVirtualDisplay(surface: Surface?, width: Int, height: Int, dpi: Int, flags: Int): Int = + displays().create(surface, width, height, dpi, flags) + + override fun releaseVirtualDisplay(displayId: Int): Boolean = displays().release(displayId) + + override fun listVirtualDisplays(): String = displays().list() + + override fun virtualDisplayInfo(displayId: Int): String = displays().info(displayId) + + override fun launchOnDisplay(displayId: Int, packageName: String?, component: String?, clearTask: Boolean): Int = + displays().launch(displayId, packageName ?: "", component, clearTask) + + override fun tapOnDisplay(displayId: Int, x: Int, y: Int): Boolean = displays().tap(displayId, x, y) + + override fun swipeOnDisplay(displayId: Int, x1: Int, y1: Int, x2: Int, y2: Int, durationMs: Int): Boolean = + displays().swipe(displayId, x1, y1, x2, y2, durationMs) + + override fun keyOnDisplay(displayId: Int, keyCode: Int): Boolean = displays().key(displayId, keyCode) + + override fun textOnDisplay(displayId: Int, text: String?): Boolean = displays().text(displayId, text ?: "") + + override fun forceStopOnDisplay(displayId: Int, packageName: String?): Boolean = + displays().forceStop(displayId, packageName ?: "") + + override fun launchUriOnDisplay(displayId: Int, uri: String?): Int = + displays().launchUri(displayId, uri ?: "") + override fun attachHost(host: IBinder?) { if (host == null || !lifecycle.attachHost(host)) return try { diff --git a/app/src/main/java/org/autojs/autojs/core/virtualscreen/DisplayCoordinateMapper.kt b/app/src/main/java/org/autojs/autojs/core/virtualscreen/DisplayCoordinateMapper.kt new file mode 100644 index 0000000000..999c12537d --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/core/virtualscreen/DisplayCoordinateMapper.kt @@ -0,0 +1,68 @@ +package org.autojs.autojs.core.virtualscreen + +import kotlin.math.min + +/** + * Pure-JVM mapping between a letterboxed preview view and the logical pixels of a virtual display. + * Kept free of Android types so the geometry can be unit tested on the JVM. + * + * zh-CN: 预览控件 (等比留边) 与虚拟屏逻辑像素之间的坐标换算. 不依赖 Android 类型, 便于在 JVM 单测中覆盖. + * + * Created for the virtual screen viewer on Nov 8, 2026. + */ +object DisplayCoordinateMapper { + + data class Geometry( + val scale: Float, + val offsetX: Float, + val offsetY: Float, + val screenWidth: Int, + val screenHeight: Int, + ) { + /** Width actually used by the picture inside the view. */ + val drawnWidth: Float get() = screenWidth * scale + + /** Height actually used by the picture inside the view. */ + val drawnHeight: Float get() = screenHeight * scale + } + + /** Fits [screenWidth]x[screenHeight] into [viewWidth]x[viewHeight], centered, never cropped. */ + @JvmStatic + fun fit(viewWidth: Int, viewHeight: Int, screenWidth: Int, screenHeight: Int): Geometry { + require(viewWidth > 0 && viewHeight > 0 && screenWidth > 0 && screenHeight > 0) { + "View and screen sizes must be positive: ${viewWidth}x${viewHeight} / ${screenWidth}x${screenHeight}" + } + val scale = min(viewWidth.toFloat() / screenWidth, viewHeight.toFloat() / screenHeight) + return Geometry( + scale = scale, + offsetX = (viewWidth - screenWidth * scale) / 2f, + offsetY = (viewHeight - screenHeight * scale) / 2f, + screenWidth = screenWidth, + screenHeight = screenHeight, + ) + } + + /** + * Converts a touch point in view coordinates to a logical display pixel, clamped to the + * display bounds (touches in the letterbox area land on the nearest edge). + */ + @JvmStatic + fun toDisplay(geometry: Geometry, viewX: Float, viewY: Float): Point { + val x = ((viewX - geometry.offsetX) / geometry.scale) + .coerceIn(0f, (geometry.screenWidth - 1).toFloat()) + val y = ((viewY - geometry.offsetY) / geometry.scale) + .coerceIn(0f, (geometry.screenHeight - 1).toFloat()) + return Point(x.toInt(), y.toInt()) + } + + /** Whether a touch started outside the picture (letterbox): such gestures are ignored. */ + @JvmStatic + fun isInside(geometry: Geometry, viewX: Float, viewY: Float): Boolean { + val x = (viewX - geometry.offsetX) / geometry.scale + val y = (viewY - geometry.offsetY) / geometry.scale + return x >= 0f && y >= 0f && x < geometry.screenWidth && y < geometry.screenHeight + } + + data class Point(val x: Int, val y: Int) + +} diff --git a/app/src/main/java/org/autojs/autojs/core/virtualscreen/DisplayRoute.kt b/app/src/main/java/org/autojs/autojs/core/virtualscreen/DisplayRoute.kt new file mode 100644 index 0000000000..581f5e1265 --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/core/virtualscreen/DisplayRoute.kt @@ -0,0 +1,149 @@ +package org.autojs.autojs.core.virtualscreen + +import android.view.Display +import org.autojs.autojs.core.image.ImageWrapper +import org.autojs.autojs.core.pref.Pref +import org.autojs.autojs.execution.ExecutionConfig +import org.autojs.autojs.runtime.ScriptRuntime +import org.autojs.autojs.runtime.api.ScreenMetrics + +/** + * The screen scope of one script execution. Every screen-facing API of the runtime reads this object + * instead of assuming the default display, so a task configured to run on a virtual screen needs no + * changes to its JavaScript source (docs/virtual-screen/README.md). + * + * The display itself is created in the Shizuku user service; this object only holds its id and the + * app-side [VirtualScreenSession] (which owns the ImageReader surface). A session is registered with + * the runtime's closeable manager, so the screen disappears when the task ends. + * + * zh-CN: 单次脚本执行的屏幕作用域. 运行时所有面向屏幕的 API 都从这里取当前 display, 而不再假定默认显示屏, + * 因此被配置为「在虚拟屏运行」的任务无需修改任何 JavaScript 源码. 显示屏由 Shizuku 用户服务创建, 本对象只保存 + * id 与 app 侧的 VirtualScreenSession (持有 ImageReader surface); 会话登记到 runtimes 的 closeable 管理器, + * 任务结束即自动释放. + */ +class DisplayRoute(private val runtime: ScriptRuntime) { + + @Volatile + var displayId: Int = Display.DEFAULT_DISPLAY + private set + + @Volatile + var session: VirtualScreenSession? = null + private set + + /** True while this execution is bound to a live virtual display rather than the physical screen. */ + val isVirtual: Boolean + get() = session?.let { !it.isReleased() } == true && displayId != Display.DEFAULT_DISPLAY + + val width: Int get() = session?.getWidth() ?: 0 + val height: Int get() = session?.getHeight() ?: 0 + val density: Int get() = session?.getDpi() ?: 0 + + /** + * Resolves the launch configuration into a concrete display. Called from + * `ScriptRuntime.initPrologue()` (after the engine carries `ExecutionConfig` as a tag, before + * modules are mounted), so globals such as `device` and `images` already see the right scope. + */ + fun apply(config: ExecutionConfig?) { + val enabled = isEnabled(config) + if (!enabled) { + displayId = Display.DEFAULT_DISPLAY + session = null + return + } + + val reuseId = config?.virtualScreenDisplayId ?: 0 + val reused = if (reuseId > 0) VirtualScreenManager.get(reuseId) else null + if (reuseId > 0 && reused == null) { + throw IllegalStateException( + "Virtual display $reuseId does not exist (it may have been released with its owning task). " + + "Set virtualScreenDisplayId=0 to create a new one.", + ) + } + + val bound = reused ?: try { + VirtualScreenManager.create( + runtime = runtime, + width = positiveOr(config?.virtualScreenWidth, ScreenMetrics.deviceScreenWidth), + height = positiveOr(config?.virtualScreenHeight, ScreenMetrics.deviceScreenHeight), + dpi = positiveOr(config?.virtualScreenDensity, ScreenMetrics.deviceScreenDensity), + flags = 0, + ) + } catch (t: Throwable) { + throw IllegalStateException( + "Cannot start this task on a virtual screen: ${t.message}. " + + "Shizuku must be running with AutoJs6 granted; Android 12+ is required.", + t, + ) + } + + displayId = bound.getDisplayId() + session = bound + runtime.putProperty(PROPERTY_DISPLAY_ID, displayId) + } + + /** Called when a script explicitly releases its screen; avoids a dangling scope. */ + fun onSessionReleased(id: Int) { + if (displayId == id) { + session = null + displayId = Display.DEFAULT_DISPLAY + } + } + + fun tap(x: Int, y: Int): Boolean = session?.tap(x, y) ?: false + + fun swipe(x1: Int, y1: Int, x2: Int, y2: Int, durationMs: Int): Boolean = + session?.swipe(x1, y1, x2, y2, durationMs) ?: false + + /** A long press is a zero-distance swipe on the shell side (down, hold, up). */ + fun longPress(x: Int, y: Int, durationMs: Int): Boolean = swipe(x, y, x, y, durationMs) + + fun key(keyCode: Int): Boolean = session?.key(keyCode) ?: false + + fun text(value: String): Boolean = session?.text(value) ?: false + + fun capture(): ImageWrapper? = runCatching { session?.capture() }.getOrNull() + + fun captureToFile(path: String): Boolean = + runCatching { session?.captureToFile(path) == true }.getOrDefault(false) + + /** 0 means success, see VirtualScreenSession.launch. */ + fun launch(packageName: String): Boolean = session?.launch(packageName) == 0 + + fun launchComponent(component: String): Boolean = session?.launchComponent(component) == 0 + + /** ACTION_VIEW app links (example://, https://, ...) resolved on this display. */ + fun launchUri(uri: String): Boolean = session?.launchUri(uri) ?: false + + fun forceStop(packageName: String): Boolean = session?.forceStop(packageName) ?: false + + private fun isEnabled(config: ExecutionConfig?): Boolean = when (config?.virtualScreenMode) { + ExecutionConfig.VIRTUAL_SCREEN_MODE_ON -> true + ExecutionConfig.VIRTUAL_SCREEN_MODE_OFF -> false + else -> config?.scriptConfig?.hasFeature(ExecutionConfig.FEATURE_VIRTUAL_SCREEN) == true || + runCatching { Pref.getBoolean(PREF_RUN_ON_VIRTUAL_SCREEN, false) }.getOrDefault(false) + } + + private fun positiveOr(configured: Int?, fallback: Int): Int = + if ((configured ?: 0) > 0) configured!! else fallback + + companion object { + + /** Global "run scripts on a virtual screen" switch, stored in the default preferences. */ + const val PREF_RUN_ON_VIRTUAL_SCREEN = "virtual_screen_run_mode" + + /** Intent extra: 1 = force on, 2 = force off, 0/absent = follow the setting. */ + const val EXTRA_VIRTUAL_SCREEN_MODE = "virtual_screen_mode" + + /** Intent extra: optional width / height / density / existing display id. */ + const val EXTRA_VIRTUAL_SCREEN_WIDTH = "virtual_screen_width" + const val EXTRA_VIRTUAL_SCREEN_HEIGHT = "virtual_screen_height" + const val EXTRA_VIRTUAL_SCREEN_DENSITY = "virtual_screen_density" + const val EXTRA_VIRTUAL_SCREEN_DISPLAY_ID = "virtual_screen_display_id" + + /** Script-visible runtime property holding the current display id. */ + const val PROPERTY_DISPLAY_ID = "virtualScreen.displayId" + + } + +} diff --git a/app/src/main/java/org/autojs/autojs/core/virtualscreen/ShellCommandRunner.kt b/app/src/main/java/org/autojs/autojs/core/virtualscreen/ShellCommandRunner.kt new file mode 100644 index 0000000000..c356542bdc --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/core/virtualscreen/ShellCommandRunner.kt @@ -0,0 +1,30 @@ +package org.autojs.autojs.core.virtualscreen + +/** + * A short `sh -c` command run in the Shizuku user service process, i.e. with the shell (uid 2000) + * or root identity. Used as the fallback when a hidden framework API is unavailable on a ROM. + * + * zh-CN: 在 Shizuku 用户服务进程 (shell uid 2000 或 root 身份) 中执行一条短命令, 用作系统隐藏接口不可用时的兜底. + */ +object ShellCommandRunner { + + data class Result(val code: Int, val output: String) { + val ok: Boolean get() = code == 0 + } + + fun run(command: String): Result = try { + val process = ProcessBuilder("/system/bin/sh", "-c", command) + .redirectErrorStream(true) + .start() + val output = process.inputStream.bufferedReader().use { it.readText() }.trim() + Result(process.waitFor(), output) + } catch (t: Throwable) { + Result(-1, t.message ?: t.javaClass.simpleName) + } + + /** Package names and components are the only values interpolated into shell commands. */ + private val SAFE_TOKEN = Regex("[A-Za-z0-9_.:/]+") + + fun isSafeToken(token: String?) = token != null && SAFE_TOKEN.matches(token) + +} diff --git a/app/src/main/java/org/autojs/autojs/core/virtualscreen/ShellVirtualDisplay.kt b/app/src/main/java/org/autojs/autojs/core/virtualscreen/ShellVirtualDisplay.kt new file mode 100644 index 0000000000..b62542dd7e --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/core/virtualscreen/ShellVirtualDisplay.kt @@ -0,0 +1,507 @@ +package org.autojs.autojs.core.virtualscreen + +import android.annotation.SuppressLint +import android.app.ActivityOptions +import android.content.AttributionSource +import android.content.ComponentName +import android.content.Context +import android.content.ContextWrapper +import android.content.Intent +import android.hardware.display.DisplayManager +import android.hardware.display.VirtualDisplay +import android.net.Uri +import android.os.Build +import android.os.Handler +import android.os.IBinder +import android.os.Looper +import android.os.SystemClock +import android.util.Log +import android.view.InputDevice +import android.view.InputEvent +import android.view.KeyEvent +import android.view.MotionEvent +import android.view.Surface +import androidx.annotation.RequiresApi +import org.json.JSONArray +import org.json.JSONObject +import org.lsposed.hiddenapibypass.HiddenApiBypass +import java.util.concurrent.ConcurrentHashMap + +/** + * The shell-side half of the virtual screen capability. It lives inside the Shizuku user service + * process (uid 2000), because a [VirtualDisplay] belongs to the process that created it: creating + * one from a one-shot `execCommand` would destroy the display as soon as that command returns. + * + * Responsibilities (the minimum mechanism set of Mika, minus its self-implemented wireless ADB + * pairing, which Shizuku already provides): + * - create a public, trusted virtual display with independent display group / focus; + * - force fullscreen windowing mode and a local IME policy on that display; + * - launch third-party activities onto a given displayId; + * - inject taps, strokes and keys onto that displayId; + * - keep an own-display-group screen awake and release everything on destroy. + * + * Every hidden framework call is best effort and falls back to an equivalent shell command when + * the ROM does not expose it. + * + * zh-CN: 虚拟屏能力的 shell 侧实现, 运行在 Shizuku 用户服务进程 (uid 2000) 内. VirtualDisplay 归属于创建它的进程, + * 用一次性 execCommand 建屏会随命令结束立即销毁, 所以必须由常驻的用户服务持有. 职责: 创建公开可信虚拟屏 (独立显示组/焦点)、 + * 强制全屏窗口模式与本地输入法策略、在指定 displayId 启动第三方 Activity、向该 displayId 注入点击/滑动/按键、 + * 为独立显示组保活, 并在服务销毁时全部释放. 所有隐藏接口调用均为尽力而为, 失败时退回等效 shell 命令. + * + * Created for the virtual-screen capability on Nov 8, 2026. + */ +class ShellVirtualDisplay(context: Context?) { + + private data class Entry( + val display: VirtualDisplay, + val flags: Int, + val width: Int, + val height: Int, + val dpi: Int, + ) + + /** Reports com.android.shell: system services cross-check the package name with the caller uid. */ + private open class ShellContext(base: Context) : ContextWrapper(base) { + override fun getPackageName(): String = SHELL_PACKAGE + override fun getOpPackageName(): String = SHELL_PACKAGE + + companion object { + const val SHELL_PACKAGE = "com.android.shell" + const val SHELL_UID = 2000 + } + } + + /** Loaded only on Android 12+, where AttributionSource exists. */ + @RequiresApi(31) + private class AttributedShellContext(base: Context) : ShellContext(base) { + override fun getAttributionSource(): AttributionSource = + AttributionSource.Builder(ShellContext.SHELL_UID) + .setPackageName(ShellContext.SHELL_PACKAGE) + .build() + } + + private val shellContext: Context? = context?.let { base -> + if (Build.VERSION.SDK_INT >= 31) AttributedShellContext(base) else ShellContext(base) + } + + private val entries = ConcurrentHashMap() + + /** App links are interpolated into a shell command: allow only URI-safe characters. */ + private val safeUri = Regex("[A-Za-z0-9._~:/?&=%+#@,\\-]+") + + private var cachedDisplayManager: DisplayManager? = null + + /** Only displays created here may be released, launched onto or injected into. */ + private fun owns(displayId: Int) = entries.containsKey(displayId) + + private val awakeHandler: Handler? by lazy { + (Looper.getMainLooper() ?: Looper.myLooper())?.let { Handler(it) } + } + private val awakeIds: MutableSet = ConcurrentHashMap.newKeySet() + private val awakeTask = object : Runnable { + override fun run() { + awakeIds.forEach(::pokeUserActivity) + awakeHandler?.postDelayed(this, AWAKE_PERIOD_MILLIS) + } + } + + /** + * Creates a display on [surface] (an ImageReader surface owned by the app process) and returns + * its displayId, or [INVALID_DISPLAY]. A [requestedFlags] of 0 means automatic flags; any other + * value is passed through unchanged for advanced callers. + * + * A [SecurityException] means the shell lacks `ADD_TRUSTED_DISPLAY`; the trusted-only bits are + * then dropped and the display is created again (sharing the main screen's focus). + * + * zh-CN: 在 [surface] (app 进程持有的 ImageReader surface) 上建屏并返回 displayId, 失败返回 -1. + * [requestedFlags] 为 0 时自动计算标志位, 非 0 时原样透传. 抛出 SecurityException 表示 shell 缺少 + * ADD_TRUSTED_DISPLAY, 此时去掉可信标志重建 (与主屏共享焦点). + */ + fun create(surface: Surface?, width: Int, height: Int, dpi: Int, requestedFlags: Int): Int { + val context = shellContext ?: return INVALID_DISPLAY + if (surface == null || width <= 0 || height <= 0 || dpi <= 0) return INVALID_DISPLAY + unlockHiddenApis() + + val sdk = Build.VERSION.SDK_INT + val flags = if (requestedFlags != 0) requestedFlags else VirtualDisplayFlags.forSdk(sdk, true) + + fun attempt(attemptFlags: Int): Int { + val vd = displayManager(context).createVirtualDisplay( + DISPLAY_NAME, width, height, dpi, surface, attemptFlags, + ) ?: return INVALID_DISPLAY + val id = vd.display?.displayId ?: INVALID_DISPLAY + if (id < 0) { + runCatching { vd.release() } + return INVALID_DISPLAY + } + entries[id] = Entry(vd, attemptFlags, width, height, dpi) + afterCreate(id, attemptFlags) + Log.i(TAG, "created display $id: ${width}x$height@$dpi flags=0x${Integer.toHexString(attemptFlags)}") + return id + } + + return try { + attempt(flags) + } catch (e: SecurityException) { + Log.w(TAG, "trusted display rejected, retrying without trusted flags", e) + if (requestedFlags == 0 && sdk >= VirtualDisplayFlags.TRUSTED_SDK) { + try { + attempt(VirtualDisplayFlags.forSdk(sdk, false)) + } catch (t: Throwable) { + Log.e(TAG, "createVirtualDisplay failed without trusted flags", t) + INVALID_DISPLAY + } + } else { + INVALID_DISPLAY + } + } catch (t: Throwable) { + Log.e(TAG, "createVirtualDisplay failed", t) + INVALID_DISPLAY + } + } + + /** Releases the display; DESTROY_CONTENT_ON_REMOVAL ends the activities left on it. */ + fun release(displayId: Int): Boolean { + val entry = entries.remove(displayId) ?: return false + stopAwake(displayId) + return runCatching { entry.display.release() }.onFailure { + Log.w(TAG, "release display $displayId failed", it) + }.isSuccess + } + + fun releaseAll() { + entries.keys.toList().forEach(::release) + awakeHandler?.removeCallbacks(awakeTask) + } + + /** JSON array of the displays owned by this service. */ + fun list(): String { + val array = JSONArray() + entries.forEach { (id, entry) -> array.put(describe(id, entry)) } + return array.toString() + } + + /** JSON object for one display, or an empty string for an unknown displayId. */ + fun info(displayId: Int): String = entries[displayId]?.let { describe(displayId, it).toString() } ?: "" + + /** + * Launches [packageName] (or an explicit `package/class` [component]) on [displayId]. + * Returns one of the LAUNCH_* codes. + */ + fun launch(displayId: Int, packageName: String, component: String?, clearTask: Boolean): Int { + if (!owns(displayId)) return LAUNCH_INVALID_DISPLAY + val context = shellContext ?: return LAUNCH_ERROR + if (!ShellCommandRunner.isSafeToken(packageName)) return LAUNCH_NO_ENTRY + if (!component.isNullOrEmpty() && !ShellCommandRunner.isSafeToken(component)) return LAUNCH_NO_ENTRY + + val intent = when { + !component.isNullOrEmpty() -> Intent().setComponent(ComponentName.unflattenFromString(component)) + packageName.isNotEmpty() -> runCatching { + context.packageManager.getLaunchIntentForPackage(packageName) + }.getOrNull() + else -> null + } ?: return LAUNCH_NO_ENTRY + + intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + if (clearTask) intent.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK) + + if (startOnDisplay(intent, displayId)) return LAUNCH_OK + + val target = intent.component?.flattenToShortString() ?: resolveLauncherComponent(packageName) + ?: return LAUNCH_NO_ENTRY + val clear = if (clearTask) "--activity-clear-task" else "" + val result = ShellCommandRunner.run("am start --display $displayId $clear -n $target") + return if (result.ok) LAUNCH_OK else LAUNCH_ERROR + } + + /** + * Starts an ACTION_VIEW app link (e.g. `example://...`) on [displayId]. `app.openUrl()` resolves + * such links on the default display unless the runtime routes them here. + * Returns one of the LAUNCH_* codes. + * + * zh-CN: 在 [displayId] 上打开 ACTION_VIEW applink. `app.openUrl()` 默认会落在主屏, 必须由运行时作用域路由到这里. + */ + fun launchUri(displayId: Int, uri: String): Int { + if (!owns(displayId)) return LAUNCH_INVALID_DISPLAY + if (!safeUri.matches(uri)) return LAUNCH_NO_ENTRY + val intent = Intent(Intent.ACTION_VIEW, Uri.parse(uri)) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + if (startOnDisplay(intent, displayId)) return LAUNCH_OK + // Single quotes keep `?` and `&` intact; safeUri already excludes quotes and shell metacharacters. + val quoted = "'" + uri.replace("'", "'\\''") + "'" + val result = ShellCommandRunner.run( + "am start --display $displayId -a android.intent.action.VIEW -d $quoted -f 0x10000000", + ) + return if (result.ok) LAUNCH_OK else LAUNCH_ERROR + } + + /** Shared hidden-API launch path for both launcher intents and app links. */ + private fun startOnDisplay(intent: Intent, displayId: Int): Boolean = runCatching { + val options = ActivityOptions.makeBasic().setLaunchDisplayId(displayId).toBundle() + val taskManager = Class.forName("android.app.ActivityTaskManager") + .getDeclaredMethod("getService").apply { isAccessible = true }.invoke(null) + val method = taskManager.javaClass.methods.first { + it.name == "startActivityAsUser" && it.parameterTypes.size == 12 + } + // caller, callingPackage, callingFeatureId, intent, resolvedType, resultTo, resultWho, + // requestCode, flags, profilerInfo, options, userId + (method.invoke( + taskManager, null, ShellContext.SHELL_PACKAGE, null, intent, null, null, null, + 0, 0, null, options, USER_CURRENT, + ) as Int) >= 0 + }.onFailure { + Log.w(TAG, "startActivityAsUser unavailable, falling back to am start", it) + }.getOrDefault(false) + + /** Stops an app that was launched on this display. */ + fun forceStop(displayId: Int, packageName: String): Boolean { + if (!owns(displayId) || !ShellCommandRunner.isSafeToken(packageName)) return false + return ShellCommandRunner.run("am force-stop $packageName").ok + } + + fun tap(displayId: Int, x: Int, y: Int): Boolean { + if (!owns(displayId)) return false + val now = SystemClock.uptimeMillis() + val down = MotionEvent.obtain(now, now, MotionEvent.ACTION_DOWN, x.toFloat(), y.toFloat(), 0) + val up = MotionEvent.obtain(now, now + TAP_MILLIS, MotionEvent.ACTION_UP, x.toFloat(), y.toFloat(), 0) + return try { + down.source = InputDevice.SOURCE_TOUCHSCREEN + up.source = InputDevice.SOURCE_TOUCHSCREEN + if (!inject(down, displayId, INJECT_ASYNC)) { + return ShellCommandRunner.run("input -d $displayId tap $x $y").ok + } + // A failed UP must not trigger a second tap through the command fallback. + inject(up, displayId, INJECT_ASYNC) + true + } finally { + down.recycle() + up.recycle() + } + } + + fun swipe(displayId: Int, x1: Int, y1: Int, x2: Int, y2: Int, durationMs: Int): Boolean { + if (!owns(displayId) || durationMs <= 0) return false + val steps = SWIPE_STEPS + val start = SystemClock.uptimeMillis() + fun event(index: Int, action: Int): MotionEvent { + val ratio = index.toFloat() / steps + return MotionEvent.obtain( + start, start + (durationMs * ratio).toLong(), action, + x1 + (x2 - x1) * ratio, y1 + (y2 - y1) * ratio, 0, + ).apply { source = InputDevice.SOURCE_TOUCHSCREEN } + } + + val down = event(0, MotionEvent.ACTION_DOWN) + val downOk = try { + inject(down, displayId, INJECT_ASYNC) + } finally { + down.recycle() + } + if (!downOk) { + return ShellCommandRunner.run("input -d $displayId swipe $x1 $y1 $x2 $y2 $durationMs").ok + } + val stepDelay = (durationMs / steps).coerceAtLeast(1).toLong() + for (index in 1..steps) { + val action = if (index == steps) MotionEvent.ACTION_UP else MotionEvent.ACTION_MOVE + val e = event(index, action) + try { + inject(e, displayId, INJECT_ASYNC) + } finally { + e.recycle() + } + SystemClock.sleep(stepDelay) + } + return true + } + + fun key(displayId: Int, keyCode: Int): Boolean { + if (!owns(displayId)) return false + val now = SystemClock.uptimeMillis() + val down = KeyEvent(now, now, KeyEvent.ACTION_DOWN, keyCode, 0) + val up = KeyEvent(now, now + TAP_MILLIS, KeyEvent.ACTION_UP, keyCode, 0) + // Only a failed DOWN falls back to the command; a failed UP must not repeat the key. + if (!inject(down, displayId, INJECT_WAIT_FOR_RESULT)) { + return ShellCommandRunner.run("input -d $displayId keyevent $keyCode").ok + } + inject(up, displayId, INJECT_WAIT_FOR_RESULT) + return true + } + + /** + * ASCII text through `input text`. Non-ASCII input should use the accessibility service's + * ACTION_SET_TEXT on a node of this display instead (see docs/virtual-screen/README.md). + */ + fun text(displayId: Int, value: String): Boolean { + if (!owns(displayId) || value.isEmpty()) return false + // `input text` goes through sh -c; reject shell metacharacters instead of escaping them. + if (value.any { it in "\$`|;&<>\n\r" }) { + Log.w(TAG, "text contains shell metacharacters, rejected") + return false + } + val escaped = value + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("'", "\\'") + .replace(" ", "%s") + return ShellCommandRunner.run("input -d $displayId text \"$escaped\"").ok + } + + // region internals + + @SuppressLint("PrivateApi") + private fun displayManager(context: Context): DisplayManager { + cachedDisplayManager?.let { return it } + // getSystemService() returns a DisplayManager bound to the system context, whose package + // name does not match the shell uid; the hidden constructor takes our ShellContext. + val manager = DisplayManager::class.java + .getDeclaredConstructor(Context::class.java) + .apply { isAccessible = true } + .newInstance(context) as DisplayManager + cachedDisplayManager = manager + return manager + } + + private fun unlockHiddenApis() { + runCatching { + HiddenApiBypass.addHiddenApiExemptions( + "Landroid/hardware/display/DisplayManager;", + "Landroid/app/ActivityTaskManager;", + "Landroid/app/IActivityTaskManager;", + "Landroid/app/ActivityOptions;", + "Landroid/hardware/input/InputManager;", + "Landroid/hardware/input/InputManagerGlobal;", + "Landroid/view/InputEvent;", + "Landroid/view/WindowManagerGlobal;", + "Landroid/os/ServiceManager;", + "Landroid/os/IPowerManager;", + ) + } + } + + private fun afterCreate(displayId: Int, flags: Int) { + setFullscreenWindowingMode(displayId) + setLocalImePolicy(displayId) + if (flags and VirtualDisplayFlags.OWN_DISPLAY_GROUP != 0) startAwake(displayId) + } + + /** Some ROMs default a new display to freeform, which shrinks apps and skews a11y coordinates. */ + private fun setFullscreenWindowingMode(displayId: Int) { + val ok = runCatching { + val wm = windowManagerService() + wm.javaClass.getMethod("setWindowingMode", Int::class.javaPrimitiveType, Int::class.javaPrimitiveType) + .invoke(wm, displayId, WINDOWING_MODE_FULLSCREEN) + true + }.getOrDefault(false) + if (!ok) { + ShellCommandRunner.run("wm set-display-windowing-mode -d $displayId $WINDOWING_MODE_FULLSCREEN") + } + } + + /** IME_POLICY_LOCAL: the keyboard appears on the virtual display, not on the main screen. */ + private fun setLocalImePolicy(displayId: Int) { + runCatching { + val wm = windowManagerService() + wm.javaClass.getMethod("setDisplayImePolicy", Int::class.javaPrimitiveType, Int::class.javaPrimitiveType) + .invoke(wm, displayId, DISPLAY_IME_POLICY_LOCAL) + }.onFailure { Log.w(TAG, "setDisplayImePolicy unavailable on display $displayId", it) } + } + + private fun windowManagerService(): Any = Class.forName("android.view.WindowManagerGlobal") + .getDeclaredMethod("getWindowManagerService") + .apply { isAccessible = true } + .invoke(null)!! + + /** An own-display-group screen has its own sleep timer, and injected input is not user activity. */ + private fun startAwake(displayId: Int) { + val handler = awakeHandler ?: return + val wasEmpty = awakeIds.isEmpty() + awakeIds.add(displayId) + if (wasEmpty) { + handler.removeCallbacks(awakeTask) + handler.post(awakeTask) + } + } + + private fun stopAwake(displayId: Int) { + awakeIds.remove(displayId) + if (awakeIds.isEmpty()) awakeHandler?.removeCallbacks(awakeTask) + } + + private fun pokeUserActivity(displayId: Int) { + runCatching { + val binder = Class.forName("android.os.ServiceManager") + .getMethod("getService", String::class.java).invoke(null, "power") as IBinder + val power = Class.forName("android.os.IPowerManager\$Stub") + .getMethod("asInterface", IBinder::class.java).invoke(null, binder) + val method = power.javaClass.methods.firstOrNull { + it.name == "userActivity" && it.parameterTypes.size == 4 + } + method?.invoke(power, displayId, SystemClock.uptimeMillis(), USER_ACTIVITY_EVENT_TOUCH, 0) + } + } + + private fun inject(event: InputEvent, displayId: Int, mode: Int): Boolean = runCatching { + event.javaClass.getMethod("setDisplayId", Int::class.javaPrimitiveType).invoke(event, displayId) + val manager = inputManager() + manager.javaClass.getMethod("injectInputEvent", InputEvent::class.java, Int::class.javaPrimitiveType) + .invoke(manager, event, mode) as? Boolean == true + }.onFailure { + Log.w(TAG, "injectInputEvent failed on display $displayId", it) + }.getOrDefault(false) + + @SuppressLint("PrivateApi") + private fun inputManager(): Any = runCatching { + // Android 14 moved the entry point to InputManagerGlobal. + Class.forName("android.hardware.input.InputManagerGlobal").getMethod("getInstance").invoke(null)!! + }.getOrElse { + Class.forName("android.hardware.input.InputManager").getMethod("getInstance").invoke(null)!! + } + + private fun resolveLauncherComponent(packageName: String): String? { + val result = ShellCommandRunner.run( + "cmd package resolve-activity --brief -c android.intent.category.LAUNCHER $packageName", + ) + return result.output.lines().map { it.trim() }.lastOrNull { it.contains('/') } + ?.takeIf { ShellCommandRunner.isSafeToken(it) } + } + + private fun describe(displayId: Int, entry: Entry): JSONObject = JSONObject() + .put("displayId", displayId) + .put("width", entry.width) + .put("height", entry.height) + .put("dpi", entry.dpi) + .put("flags", entry.flags) + .put("flagsHex", "0x" + Integer.toHexString(entry.flags)) + .put("flagNames", JSONArray(VirtualDisplayFlags.describe(entry.flags))) + .put("trusted", VirtualDisplayFlags.isTrusted(entry.flags)) + .put("ownFocus", entry.flags and VirtualDisplayFlags.OWN_FOCUS != 0) + .put("ownDisplayGroup", entry.flags and VirtualDisplayFlags.OWN_DISPLAY_GROUP != 0) + .put("sdkInt", Build.VERSION.SDK_INT) + + // endregion + + companion object { + + const val TAG = "ShellVirtualDisplay" + + const val INVALID_DISPLAY = -1 + + const val LAUNCH_OK = 0 + const val LAUNCH_NO_ENTRY = 1 + const val LAUNCH_ERROR = 2 + const val LAUNCH_INVALID_DISPLAY = 3 + + private const val DISPLAY_NAME = "autojs6-virtual-screen" + private const val WINDOWING_MODE_FULLSCREEN = 1 + private const val DISPLAY_IME_POLICY_LOCAL = 0 + private const val USER_CURRENT = -2 + private const val USER_ACTIVITY_EVENT_TOUCH = 2 + private const val INJECT_ASYNC = 0 + private const val INJECT_WAIT_FOR_RESULT = 1 + private const val TAP_MILLIS = 60L + private const val SWIPE_STEPS = 10 + private const val AWAKE_PERIOD_MILLIS = 4_000L + + } + +} diff --git a/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualDisplayFlags.kt b/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualDisplayFlags.kt new file mode 100644 index 0000000000..220ce8ac96 --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualDisplayFlags.kt @@ -0,0 +1,87 @@ +package org.autojs.autojs.core.virtualscreen + +/** + * Hidden flags of DisplayManager.createVirtualDisplay used by a background automation display. + * + * The values are the AOSP bit positions (kept as literals because several of them are @hide). + * Ported from the mechanism of Mika (wangzan101/Mika, Apache-2.0): a trusted virtual display + * owns its display group and, on Android 14+, its focus without stealing the main screen focus. + * This object is deliberately free of any `android.*` reference so that it can be unit tested on + * a plain JVM. + * + * zh-CN: 后台自动化虚拟屏使用的 DisplayManager 隐藏标志位. 数值取自 AOSP 位定义 (部分为 @hide 常量, 故以字面量保存), + * 机制参考 Mika (wangzan101/Mika, Apache-2.0): 可信虚拟屏自成一个显示组, Android 14+ 起还能拥有独立焦点且不抢主屏焦点. + * 本对象刻意不引用任何 android.* 类, 以便纯 JVM 单元测试. + */ +object VirtualDisplayFlags { + + /** The first Android version whose shell may be allowed to add a trusted display. */ + const val TRUSTED_SDK = 33 // TIRAMISU + + /** The first Android version on which a virtual display can own its focus. */ + const val OWN_FOCUS_SDK = 34 // UPSIDE_DOWN_CAKE + + const val PUBLIC = 1 shl 0 + const val OWN_CONTENT_ONLY = 1 shl 3 + const val SUPPORTS_TOUCH = 1 shl 6 + const val ROTATES_WITH_CONTENT = 1 shl 7 + const val DESTROY_CONTENT_ON_REMOVAL = 1 shl 8 + const val TRUSTED = 1 shl 10 + const val OWN_DISPLAY_GROUP = 1 shl 11 + const val ALWAYS_UNLOCKED = 1 shl 12 + const val TOUCH_FEEDBACK_DISABLED = 1 shl 13 + const val OWN_FOCUS = 1 shl 14 + const val STEAL_TOP_FOCUS_DISABLED = 1 shl 16 + + /** The flags a plain app is allowed to use on every supported Android version. */ + const val BASE = PUBLIC or OWN_CONTENT_ONLY or SUPPORTS_TOUCH or + ROTATES_WITH_CONTENT or DESTROY_CONTENT_ON_REMOVAL + + /** + * Flags for a display created as the shell uid. + * + * [trusted] must be false when the shell lacks `ADD_TRUSTED_DISPLAY`: the trusted-only bits + * are then dropped, and the created display shares the main screen's focus. On Android 14+ + * `OWN_FOCUS` alone would make the virtual display the top focus display and clear the main + * screen's focus window, so it is always paired with `STEAL_TOP_FOCUS_DISABLED`. + * + * zh-CN: 以 shell 身份建屏时使用的标志位. 当 shell 缺少 ADD_TRUSTED_DISPLAY 时 [trusted] 传 false, 去掉仅可信屏可用的位, + * 该屏会与主屏共享焦点. Android 14+ 上单独的 OWN_FOCUS 会让虚拟屏成为最上层焦点屏并清空主屏焦点窗口, + * 因此必须与 STEAL_TOP_FOCUS_DISABLED 成对出现. + */ + fun forSdk(sdk: Int, trusted: Boolean): Int { + var flags = BASE + if (sdk >= TRUSTED_SDK) { + flags = flags or TOUCH_FEEDBACK_DISABLED + if (trusted) { + flags = flags or TRUSTED or OWN_DISPLAY_GROUP or ALWAYS_UNLOCKED + } + } + if (sdk >= OWN_FOCUS_SDK && trusted) { + flags = flags or OWN_FOCUS or STEAL_TOP_FOCUS_DISABLED + } + return flags + } + + /** Whether a [SecurityException] from createVirtualDisplay is worth a retry without trusted flags. */ + fun isTrusted(flags: Int) = flags and TRUSTED != 0 + + /** Human-readable names of the set bits, for `virtualScreen.state()` and diagnostics. */ + fun describe(flags: Int): List = buildList { + fun bit(value: Int, name: String) { + if (flags and value != 0) add(name) + } + bit(PUBLIC, "public") + bit(OWN_CONTENT_ONLY, "ownContentOnly") + bit(SUPPORTS_TOUCH, "supportsTouch") + bit(ROTATES_WITH_CONTENT, "rotatesWithContent") + bit(DESTROY_CONTENT_ON_REMOVAL, "destroyContentOnRemoval") + bit(TRUSTED, "trusted") + bit(OWN_DISPLAY_GROUP, "ownDisplayGroup") + bit(ALWAYS_UNLOCKED, "alwaysUnlocked") + bit(TOUCH_FEEDBACK_DISABLED, "touchFeedbackDisabled") + bit(OWN_FOCUS, "ownFocus") + bit(STEAL_TOP_FOCUS_DISABLED, "stealTopFocusDisabled") + } + +} diff --git a/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenFramePump.kt b/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenFramePump.kt new file mode 100644 index 0000000000..b2ee078b20 --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenFramePump.kt @@ -0,0 +1,211 @@ +package org.autojs.autojs.core.virtualscreen + +import android.graphics.Bitmap +import android.media.Image +import android.media.ImageReader +import android.os.Handler +import android.os.HandlerThread +import android.os.SystemClock +import android.util.Log +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.atomic.AtomicBoolean + +/** + * A low-allocation live preview feed for one virtual display. + * + * The frame source is an [ImageReader] with two buffers. **It is driven by active polling, not by + * `OnImageAvailableListener`**: by the time a viewer attaches to a display that was created a + * moment earlier, frames may already have filled both buffers. A listener registered at that point + * is never called for the queued frames, while the producer blocks waiting for a free buffer — a + * deadlock that shows up as a permanently black preview "waiting for the first frame". Calling + * `acquireLatestImage()` on a dedicated thread drains those buffers and unblocks the producer + * immediately. The same call also covers displays that only produce frames occasionally. + * + * While running, this pump is the *only* consumer of the reader: `VirtualScreenSession` reads a + * copy of [bitmap] instead of acquiring images itself. Frames are written into one reusable + * [Bitmap] (fast path `copyPixelsFromBuffer`; GPU-aligned rowStride is repacked through a reusable + * direct buffer), so there is no per-frame bitmap allocation and no recycling race with the UI. + * If the producer reports a different geometry (rotation, resized display), the reusable buffers + * are resized once and the UI picks up the new bitmap on its next draw. + * + * zh-CN: 虚拟屏低分配实时预览帧源. **主动轮询取帧, 不依赖 OnImageAvailableListener**: 查看器附着到 + * 稍早已建好的屏时, 两个缓冲可能已被帧占满; 此时才注册的 listener 不会补发已排队帧, 生产者又在等待 + * 空闲缓冲, 形成死锁, 表现为预览一直「等待第一帧」. 用专用线程主动 acquireLatestImage() 排空缓冲即可解锁. + * 运行期间本泵是 ImageReader 的唯一消费者, 脚本截图改读复用 Bitmap 的副本; 尺寸/旋转变化时一次性重建复用缓冲. + * + * Created for the virtual screen viewer on Nov 8, 2026. Fixed the black-preview deadlock on Nov 9, 2026. + */ +class VirtualScreenFramePump( + private val reader: ImageReader, + private val width: Int, + private val height: Int, + private val onFrame: () -> Unit, +) { + + /** The reusable frame. May be replaced once if the producer changes geometry; never recycled. */ + @Volatile + var bitmap: Bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + private set + + private var compact = ByteBuffer.allocateDirect(width * height * 4).order(ByteOrder.nativeOrder()) + private var rowBytes = ByteArray(width * 4) + private val running = AtomicBoolean(false) + + @Volatile + private var thread: HandlerThread? = null + + @Volatile + private var handler: Handler? = null + + /** Nanosecond timestamp of the last frame actually written, 0 before the first one. */ + @Volatile + var lastFrameNanos: Long = 0L + private set + + /** Uptime millis of the last successful write, for the "no frames for N ms" diagnostic. */ + @Volatile + var lastWriteMillis: Long = 0L + private set + + /** Monotonic count of frames written, for the viewer's FPS readout. */ + @Volatile + var frameCount: Long = 0L + private set + + /** Last conversion/acquisition error, cleared on the next successful frame. Null when healthy. */ + @Volatile + var lastError: String? = null + private set + + /** True once at least one frame has landed in [bitmap]. */ + val hasFrame: Boolean get() = lastFrameNanos != 0L + + private val pollTask = object : Runnable { + override fun run() { + if (!running.get()) return + consumeOnce() + if (running.get()) handler?.postDelayed(this, FRAME_INTERVAL_MILLIS) + } + } + + /** Starts consuming frames. Calling it twice is a no-op. */ + fun start(): Boolean { + if (!running.compareAndSet(false, true)) return true + return try { + val t = HandlerThread("vs-frames-$width-$height").also { it.start() } + thread = t + handler = Handler(t.looper) + // Drain any frame that was queued before we attached; this is what breaks the deadlock. + handler?.post(pollTask) + true + } catch (error: Throwable) { + Log.w(TAG, "cannot start frame pump", error) + lastError = "start: ${error.message}" + running.set(false) + thread?.quitSafely() + thread = null + handler = null + false + } + } + + /** Requests an immediate out-of-band drain, e.g. right after launching an app on the display. */ + fun drainNow() { + handler?.post { + if (running.get()) consumeOnce() + } + } + + fun stop() { + if (!running.compareAndSet(true, false)) return + handler?.removeCallbacksAndMessages(null) + thread?.quitSafely() + thread = null + handler = null + } + + private fun consumeOnce() { + val image = runCatching { reader.acquireLatestImage() } + .onFailure { + lastError = "acquireLatestImage: ${it.message}" + Log.w(TAG, "acquireLatestImage failed", it) + } + .getOrNull() ?: return + val written = try { + write(image) + } catch (error: Throwable) { + lastError = "write: ${error.message}" + Log.w(TAG, "frame conversion failed", error) + false + } finally { + image.close() + } + if (written) { + lastError = null + lastFrameNanos = SystemClock.elapsedRealtimeNanos() + lastWriteMillis = SystemClock.uptimeMillis() + frameCount++ + runCatching { onFrame() } + } + } + + /** Copies a frame's pixels into the reusable [bitmap]; returns false on an unusable plane. */ + private fun write(image: Image): Boolean { + val sourceWidth = image.width + val sourceHeight = image.height + if (sourceWidth <= 0 || sourceHeight <= 0) { + lastError = "empty image ${sourceWidth}x${sourceHeight}" + return false + } + val plane = image.planes.firstOrNull() ?: run { + lastError = "image has no planes" + return false + } + val rowStride = plane.rowStride + val pixelStride = plane.pixelStride + if (pixelStride != 4 || rowStride < sourceWidth * 4) { + lastError = "unsupported plane: pixelStride=$pixelStride rowStride=$rowStride srcW=$sourceWidth" + return false + } + ensureGeometry(sourceWidth, sourceHeight) + + val source = plane.buffer.duplicate().order(ByteOrder.nativeOrder()) + compact.clear() + if (rowStride == sourceWidth * 4) { + // ImageReader RGBA_8888 is stored as little-endian RGBA, which is exactly what + // ARGB_8888's copyPixelsFromBuffer expects on a little-endian device. + source.rewind() + val limit = source.capacity().coerceAtMost(sourceWidth * sourceHeight * 4) + source.limit(limit) + compact.put(source) + } else { + // Some GPUs align rowStride to 64 bytes. Repack the visible part of every row. + for (y in 0 until sourceHeight) { + source.position(y * rowStride) + source.get(rowBytes, 0, sourceWidth * 4) + compact.put(rowBytes) + } + } + compact.flip() + bitmap.copyPixelsFromBuffer(compact) + return true + } + + /** Rebuilds the reusable buffers once when the producer reports a different geometry. */ + private fun ensureGeometry(sourceWidth: Int, sourceHeight: Int) { + if (bitmap.width == sourceWidth && bitmap.height == sourceHeight) return + Log.i(TAG, "frame geometry changed ${bitmap.width}x${bitmap.height} -> ${sourceWidth}x${sourceHeight}") + // The old bitmap is intentionally not recycled: the UI may still be drawing it. It is + // replaced at most once, so letting the GC collect it is safe and simple. + bitmap = Bitmap.createBitmap(sourceWidth, sourceHeight, Bitmap.Config.ARGB_8888) + compact = ByteBuffer.allocateDirect(sourceWidth * sourceHeight * 4).order(ByteOrder.nativeOrder()) + rowBytes = ByteArray(sourceWidth * 4) + } + + private companion object { + const val TAG = "VirtualScreenFramePump" + const val FRAME_INTERVAL_MILLIS = 66L // ~15 fps preview, and ~15 fps polling of the reader + } + +} diff --git a/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenManager.kt b/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenManager.kt new file mode 100644 index 0000000000..5b1ce82178 --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenManager.kt @@ -0,0 +1,109 @@ +package org.autojs.autojs.core.virtualscreen + +import android.graphics.PixelFormat +import android.media.ImageReader +import android.os.Build +import org.autojs.autojs.runtime.ScriptRuntime +import org.autojs.autojs.runtime.api.WrappedShizuku +import org.json.JSONObject +import java.util.concurrent.ConcurrentHashMap + +/** + * App-side registry of virtual screens. Creating a screen asks the Shizuku user service to create + * the display on an [ImageReader] surface owned here; the display itself is never created in the + * app process (a plain app may only create a PRIVATE display of its own content). + * + * zh-CN: app 侧的虚拟屏登记处. 建屏时请 Shizuku 用户服务在本进程持有的 ImageReader surface 上创建显示屏, + * 显示屏绝不在 app 进程创建 (普通 App 只能创建仅供自己渲染内容的 PRIVATE 屏). + */ +object VirtualScreenManager { + + private val sessions = ConcurrentHashMap() + + /** + * Creates a virtual display for [runtime]. [flags] 0 means automatic (SDK-dependent) flags. + * A null [runtime] creates a *detached* screen that outlives scripts (used by the viewer) and + * must be released explicitly. Throws when Shizuku is not operational or the shell could not + * create the display. + */ + @JvmStatic + fun create(runtime: ScriptRuntime?, width: Int, height: Int, dpi: Int, flags: Int): VirtualScreenSession { + require(width in 1..MAX_SIDE && height in 1..MAX_SIDE && dpi in 1..MAX_DPI) { + "Invalid virtual display size: ${width}x$height@$dpi" + } + val service = WrappedShizuku.getServiceOrNull() + ?: throw IllegalStateException( + "Virtual screens need Shizuku (or a root Shizuku) with AutoJs6 granted. " + + "Call shizuku.requestPermission() or enable Shizuku first.", + ) + val reader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, 2) + val displayId = try { + service.createVirtualDisplay(reader.surface, width, height, dpi, flags) + } catch (t: Throwable) { + reader.close() + throw IllegalStateException("Unable to create a virtual display: ${t.message}", t) + } + if (displayId < 0) { + reader.close() + throw IllegalStateException( + "The shell could not create a virtual display. Check Shizuku, Android ${Build.VERSION.RELEASE} " + + "and whether the ROM allows ADD_TRUSTED_DISPLAY.", + ) + } + val info = try { + JSONObject(service.virtualDisplayInfo(displayId)) + } catch (t: Throwable) { + JSONObject() + } + val session = VirtualScreenSession(runtime, service, reader, displayId, width, height, dpi, info) + sessions[displayId] = session + // Released in Stage.CLOSEABLES when the owning script exits. Detached screens deliberately + // skip this: they stay alive across activities until the viewer releases them. + runtime?.closeableManager?.add(session) + return session + } + + /** Creates a screen that is not owned by any script; the viewer releases it explicitly. */ + @JvmStatic + fun createDetached(width: Int, height: Int, dpi: Int, flags: Int = 0): VirtualScreenSession = + create(null, width, height, dpi, flags) + + @JvmStatic + fun get(displayId: Int): VirtualScreenSession? = sessions[displayId]?.takeIf { !it.isReleased() } + + /** Snapshot of every live session, sorted by display id (for the viewer's screen list). */ + @JvmStatic + fun liveSessions(): List = ids().toList().mapNotNull(::get) + + @JvmStatic + fun ids(): IntArray = sessions.keys.filter { get(it) != null }.sorted().toIntArray() + + @JvmStatic + fun release(displayId: Int): Boolean = get(displayId)?.release() ?: false + + @JvmStatic + fun releaseAll(): Int = ids().count { release(it) } + + @JvmStatic + fun unregister(displayId: Int, session: VirtualScreenSession) { + sessions.remove(displayId, session) + } + + /** Best effort: also safe when called on a plain JVM (unit tests). */ + @JvmStatic + fun isAvailable(): Boolean = runCatching { WrappedShizuku.isOperational() }.getOrDefault(false) + + @JvmStatic + fun isOwnFocusSupported(): Boolean = sdkInt() >= VirtualDisplayFlags.OWN_FOCUS_SDK + + /** 0 on a plain JVM (unit tests), where android.jar is a stub. */ + @JvmStatic + fun sdkInt(): Int = runCatching { Build.VERSION.SDK_INT }.getOrDefault(0) + + @JvmStatic + fun screenCount(): Int = ids().size + + private const val MAX_SIDE = 4096 + private const val MAX_DPI = 1200 + +} diff --git a/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenSession.kt b/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenSession.kt new file mode 100644 index 0000000000..f482b33361 --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/core/virtualscreen/VirtualScreenSession.kt @@ -0,0 +1,320 @@ +package org.autojs.autojs.core.virtualscreen + +import android.graphics.Bitmap +import android.graphics.Matrix +import android.hardware.display.DisplayManager +import android.media.Image +import android.media.ImageReader +import android.os.SystemClock +import android.util.Log +import android.view.Surface +import org.autojs.autojs.app.GlobalAppContext +import org.autojs.autojs.core.image.ImageWrapper +import org.autojs.autojs.core.shizuku.IUserService +import org.autojs.autojs.runtime.ScriptRuntime +import org.json.JSONObject +import java.io.Closeable +import java.io.FileOutputStream +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong + +/** + * One virtual screen. Instances are returned to JavaScript as Java objects, so their public methods + * are the script API (`screen.tap(...)`, `screen.capture()`). A session may also be *detached* + * (created with a null [runtime] by the viewer), in which case it outlives any script and has to be + * released explicitly from the viewer. + * + * The app process only owns the [ImageReader] surface; the display itself is held by the Shizuku + * user service process. Script-bound sessions are registered with `ScriptRuntime.closeableManager`, + * so a screen created by a script is released automatically when that script exits. + * + * While the viewer is open, [startPreview] makes a [VirtualScreenFramePump] the sole consumer of + * the ImageReader, and [snapshotBitmap] reads that pump's reusable bitmap instead of competing for + * the two image buffers. Preview and script screenshots therefore coexist without starving either. + * + * zh-CN: 一块虚拟屏. 实例作为 Java 对象返回给 JavaScript, 公开方法即脚本 API. 由查看器创建、 + * [runtime] 为空的「独立屏」不随脚本退出而销毁, 需在查看器中显式释放. app 进程只持有 ImageReader surface, + * 显示屏本身由 Shizuku 用户服务进程持有; 脚本持有的会话登记到 closeableManager, 脚本退出自动释放. + * 查看器打开期间由帧泵独占 ImageReader, 脚本截图改读帧泵的复用 Bitmap, 两边不再互相抢缓冲. + */ +class VirtualScreenSession internal constructor( + private val runtime: ScriptRuntime?, + private val service: IUserService, + private val reader: ImageReader, + private val displayId: Int, + private val width: Int, + private val height: Int, + private val dpi: Int, + private val creationInfo: JSONObject, +) : Closeable { + + private val alive = AtomicBoolean(true) + private val frameTimeNanos = AtomicLong(0L) + private val previewLock = Any() + + @Volatile + private var pump: VirtualScreenFramePump? = null + + fun getDisplayId(): Int = displayId + fun getWidth(): Int = width + fun getHeight(): Int = height + fun getDpi(): Int = dpi + fun getFlags(): Int = creationInfo.optInt("flags") + fun isTrusted(): Boolean = creationInfo.optBoolean("trusted") + fun hasOwnFocus(): Boolean = creationInfo.optBoolean("ownFocus") + fun isReleased(): Boolean = !alive.get() + fun getFrameTime(): Long = pump?.lastFrameNanos?.takeIf { it != 0L } ?: frameTimeNanos.get() + + /** True for a screen created by the viewer: it is not tied to any script's lifecycle. */ + fun isDetached(): Boolean = runtime == null + + /** JSON state of this screen: geometry, flags, preview status and whether the last frame is fresh. */ + fun state(): String = JSONObject(creationInfo.toString()) + .put("displayId", displayId) + .put("width", width) + .put("height", height) + .put("dpi", dpi) + .put("released", isReleased()) + .put("detached", isDetached()) + .put("previewing", isPreviewing()) + .put("frameTimeNanos", getFrameTime()) + .put("frameCount", previewFrameCount()) + .put("previewError", previewError().orEmpty()) + .put("rotation", currentRotation()) + .toString() + + /** Launches an app on this display. 0=ok, 1=no launcher entry, 2=error, 3=invalid display. */ + fun launch(packageName: String): Int = call { + service.launchOnDisplay(displayId, packageName, "", false) + }.getOrElse { errorCodeFor(it) } + + /** Launches an explicit `package/activity` component on this display. */ + fun launchComponent(component: String): Int = call { + service.launchOnDisplay(displayId, "", component, false) + }.getOrElse { errorCodeFor(it) } + + /** Opens an ACTION_VIEW app link on this display instead of the physical screen. */ + fun launchUri(uri: String): Boolean = call { + service.launchUriOnDisplay(displayId, uri) == 0 + }.getOrDefault(false) + + /** Like [launch] but clears the existing task first (a cold start on this display). */ + fun launchNew(packageName: String): Int = call { + service.launchOnDisplay(displayId, packageName, "", true) + }.getOrElse { errorCodeFor(it) } + + /** Stops an app that was launched on this display. */ + fun forceStop(packageName: String): Boolean = call { + service.forceStopOnDisplay(displayId, packageName) + }.getOrDefault(false) + + fun tap(x: Int, y: Int): Boolean = call { service.tapOnDisplay(displayId, x, y) }.getOrDefault(false) + + fun swipe(x1: Int, y1: Int, x2: Int, y2: Int, durationMs: Int): Boolean = call { + service.swipeOnDisplay(displayId, x1, y1, x2, y2, durationMs) + }.getOrDefault(false) + + fun key(keyCode: Int): Boolean = call { service.keyOnDisplay(displayId, keyCode) }.getOrDefault(false) + + /** ASCII text only; for CJK text use accessibility ACTION_SET_TEXT on a node of this display. */ + fun text(value: String): Boolean = call { service.textOnDisplay(displayId, value) }.getOrDefault(false) + + /** A long press is a zero-distance swipe on the shell side (down, hold, up). */ + fun longPress(x: Int, y: Int, durationMs: Int): Boolean = swipe(x, y, x, y, durationMs) + + // ---- Live preview (viewer) ---- + + /** + * Starts the frame feed. [onFrame] is called on a background HandlerThread, so the caller must + * hop to the main thread before invalidating a view. Returns false when the listener cannot be + * attached (the viewer then falls back to polling [snapshotBitmap]). + */ + fun startPreview(onFrame: () -> Unit): Boolean { + ensureAlive() + return synchronized(previewLock) { + pump?.let { return@synchronized true } + val created = VirtualScreenFramePump(reader, width, height, onFrame) + if (created.start()) { + pump = created + true + } else { + false + } + } + } + + fun stopPreview() { + synchronized(previewLock) { + pump?.stop() + pump = null + } + } + + fun isPreviewing(): Boolean = pump != null + + /** The reusable preview bitmap, held by the UI thread for drawing. Never recycle it here. */ + fun previewBitmap(): Bitmap? = pump?.bitmap?.takeIf { pump?.hasFrame == true } + + fun previewFrameCount(): Long = pump?.frameCount ?: 0L + + /** Why the preview has no frame (unsupported plane, closed reader, ...), or null when healthy. */ + fun previewError(): String? = pump?.lastError + + /** Uptime millis of the last successful frame; 0 when no frame has ever arrived. */ + fun previewLastWriteMillis(): Long = pump?.lastWriteMillis ?: 0L + + /** Forces an immediate read, e.g. right after an app was launched on this display. */ + fun drainPreview() { + pump?.drainNow() + } + + // ---- Screenshots ---- + + /** + * Latest frame of the virtual display as an [ImageWrapper], so all `images.*` helpers (find + * color, template match, save) work on it. Coordinates are the logical pixels of this display. + * Works while the viewer is previewing because it copies the pump's bitmap instead of consuming + * an ImageReader buffer. + * + * zh-CN: 取虚拟屏最新一帧并返回 ImageWrapper, 可直接用 images.* 的找色/找图/保存能力. + * 查看器预览期间从帧泵的复用 Bitmap 复制, 不占用 ImageReader 缓冲. + */ + fun capture(): ImageWrapper { + ensureAlive() + val owner = runtime + ?: throw IllegalStateException("A detached virtual screen has no script runtime; use the viewer to save a frame") + val bitmap = snapshotBitmap() ?: throw IllegalStateException("No frame on virtual display $displayId") + return ImageWrapper.ofBitmap(owner, bitmap) + } + + /** Writes the latest frame as PNG, without allocating an ImageWrapper. */ + fun captureToFile(path: String): Boolean { + ensureAlive() + val bitmap = snapshotBitmap() ?: return false + return try { + FileOutputStream(path).use { bitmap.compress(Bitmap.CompressFormat.PNG, 100, it) } + } catch (t: Throwable) { + Log.w(TAG, "captureToFile failed: $path", t) + false + } finally { + bitmap.recycle() + } + } + + /** + * A standalone copy of the latest frame, safe for the caller to recycle. Shared by the script + * screenshot path and the viewer's "save PNG" action. + */ + fun snapshotBitmap(): Bitmap? { + ensureAlive() + val active = pump + if (active != null) { + // While previewing, the pump is the ONLY ImageReader consumer. Acquiring here would + // starve both consumers and reintroduce the black-preview deadlock. + if (!active.hasFrame) return null + val copy = active.bitmap.copy(active.bitmap.config ?: Bitmap.Config.ARGB_8888, false) + return upright(copy) + } + val image = latestImage() ?: return null + return try { + toBitmap(image) + } finally { + image.close() + } + } + + /** Releases the display. Safe to call more than once; also unregisters from the script runtime. */ + fun release(): Boolean { + if (!alive.compareAndSet(true, false)) return false + stopPreview() + // Must not go through call(): it asserts alive, and we have already flipped the flag. + val ok = runCatching { service.releaseVirtualDisplay(displayId) } + .onFailure { Log.w(TAG, "releaseVirtualDisplay($displayId) failed", it) } + .getOrDefault(false) + runCatching { reader.close() } + runCatching { runtime?.closeableManager?.remove(this) } + VirtualScreenManager.unregister(displayId, this) + return ok + } + + /** [Closeable] so the script runtime releases this screen on exit. */ + override fun close() { + release() + } + + override fun toString(): String = "VirtualScreen(displayId=$displayId, ${width}x$height@$dpi, released=${isReleased()})" + + // region internals + + private fun call(block: () -> T): Result { + ensureAlive() + return runCatching { block() }.onFailure { Log.w(TAG, "virtual display $displayId call failed", it) } + } + + private fun ensureAlive() { + check(alive.get()) { "Virtual display $displayId has been released" } + } + + private fun errorCodeFor(error: Throwable): Int { + Log.w(TAG, "launch failed on display $displayId", error) + return ShellVirtualDisplay.LAUNCH_ERROR + } + + private fun latestImage(): Image? { + val deadline = SystemClock.uptimeMillis() + CAPTURE_TIMEOUT_MILLIS + do { + val image = runCatching { reader.acquireLatestImage() }.getOrNull() + if (image != null) { + // Producer timestamp, not the time the buffer was copied (Mika's FrameFreshness). + frameTimeNanos.set(image.timestamp) + return image + } + SystemClock.sleep(FRAME_POLL_MILLIS) + } while (SystemClock.uptimeMillis() < deadline) + return null + } + + private fun toBitmap(image: Image): Bitmap { + val plane = image.planes[0] + val stride = plane.rowStride / plane.pixelStride + val buffer = plane.buffer + buffer.rewind() + val raw = Bitmap.createBitmap(stride, height, Bitmap.Config.ARGB_8888) + raw.copyPixelsFromBuffer(buffer) + val cropped = if (stride == width) { + raw + } else { + Bitmap.createBitmap(raw, 0, 0, width.coerceAtMost(stride), height).also { raw.recycle() } + } + return upright(cropped) + } + + private fun upright(bitmap: Bitmap): Bitmap { + val degrees = when (currentRotation()) { + Surface.ROTATION_90 -> 90f + Surface.ROTATION_180 -> 180f + Surface.ROTATION_270 -> 270f + else -> 0f + } + if (degrees == 0f) return bitmap + val matrix = Matrix().apply { postRotate(degrees) } + val rotated = Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, false) + if (rotated !== bitmap) bitmap.recycle() + return rotated + } + + private fun currentRotation(): Int = runCatching { + GlobalAppContext.get() + .getSystemService(DisplayManager::class.java) + ?.getDisplay(displayId)?.rotation + }.getOrNull() ?: Surface.ROTATION_0 + + // endregion + + private companion object { + const val TAG = "VirtualScreenSession" + const val CAPTURE_TIMEOUT_MILLIS = 2_000L + const val FRAME_POLL_MILLIS = 20L + } + +} diff --git a/app/src/main/java/org/autojs/autojs/execution/ExecutionConfig.kt b/app/src/main/java/org/autojs/autojs/execution/ExecutionConfig.kt index 0b19c43a1f..0f5ff4a6f0 100644 --- a/app/src/main/java/org/autojs/autojs/execution/ExecutionConfig.kt +++ b/app/src/main/java/org/autojs/autojs/execution/ExecutionConfig.kt @@ -19,6 +19,27 @@ data class ExecutionConfig( var scriptConfig: ScriptConfig = ScriptConfig(), ) : Parcelable { + /** + * Task-level virtual screen mode. FOLLOW falls back to the global setting / project feature; + * ON and OFF override both. This is a launch option, not a script API: an existing script runs + * unchanged on the virtual display (docs/virtual-screen/README.md). + * + * These properties deliberately stay outside the primary constructor: adding them there would + * change the signature of the generated data-class `copy()`, which the `com.stardust` Rhino + * alias still exposes to existing scripts (see stardust-alias-snapshot.json). Callers that use + * `copy()` must carry these options over explicitly. + * + * zh-CN: 任务级虚拟屏模式. 这些属性刻意不放进主构造函数, 否则 data class 生成的 copy() 签名会变化, + * 破坏 com.stardust Rhino 别名对存量脚本的兼容承诺; 使用 copy() 的调用方需显式带上这些选项. + */ + var virtualScreenMode: Int = VIRTUAL_SCREEN_MODE_FOLLOW + /** 0 = follow the physical screen. zh-CN: 0 表示跟随物理屏幕尺寸. */ + var virtualScreenWidth: Int = 0 + var virtualScreenHeight: Int = 0 + var virtualScreenDensity: Int = 0 + /** 0 = create a new display; a positive id reuses an existing virtual display. */ + var virtualScreenDisplayId: Int = 0 + private val mArguments = HashMap() /** Monotonic host launch measurements; neither script arguments nor parcel input. */ @@ -41,13 +62,19 @@ data class ExecutionConfig( get() = mArguments constructor(parcel: Parcel) : this( - parcel.readString().orEmpty(), - parcel.createStringArray().orEmpty(), - parcel.readInt(), - parcel.readLong(), - parcel.readLong(), - parcel.readInt() - ) + workingDirectory = parcel.readString().orEmpty(), + envPath = parcel.createStringArray().orEmpty(), + intentFlags = parcel.readInt(), + delay = parcel.readLong(), + interval = parcel.readLong(), + loopTimes = parcel.readInt(), + ) { + virtualScreenMode = parcel.readInt() + virtualScreenWidth = parcel.readInt() + virtualScreenHeight = parcel.readInt() + virtualScreenDensity = parcel.readInt() + virtualScreenDisplayId = parcel.readInt() + } fun setArgument(key: String, `object`: Any) { mArguments[key] = `object` @@ -73,6 +100,11 @@ data class ExecutionConfig( if (delay != other.delay) return false if (interval != other.interval) return false if (loopTimes != other.loopTimes) return false + if (virtualScreenMode != other.virtualScreenMode) return false + if (virtualScreenWidth != other.virtualScreenWidth) return false + if (virtualScreenHeight != other.virtualScreenHeight) return false + if (virtualScreenDensity != other.virtualScreenDensity) return false + if (virtualScreenDisplayId != other.virtualScreenDisplayId) return false if (mArguments != other.mArguments) return false return true @@ -85,6 +117,11 @@ data class ExecutionConfig( delay, interval, loopTimes, + virtualScreenMode, + virtualScreenWidth, + virtualScreenHeight, + virtualScreenDensity, + virtualScreenDisplayId, mArguments, ).toHashCode() @@ -96,6 +133,11 @@ data class ExecutionConfig( writeLong(delay) writeLong(interval) writeInt(loopTimes) + writeInt(virtualScreenMode) + writeInt(virtualScreenWidth) + writeInt(virtualScreenHeight) + writeInt(virtualScreenDensity) + writeInt(virtualScreenDisplayId) } } @@ -120,6 +162,13 @@ data class ExecutionConfig( @Suppress("ConstPropertyName") const val tag = "execution.config" + const val VIRTUAL_SCREEN_MODE_FOLLOW = 0 + const val VIRTUAL_SCREEN_MODE_ON = 1 + const val VIRTUAL_SCREEN_MODE_OFF = 2 + + /** project.json feature that turns the run mode on for a whole project. */ + const val FEATURE_VIRTUAL_SCREEN = "virtualScreen" + @JvmStatic fun getTag() = tag diff --git a/app/src/main/java/org/autojs/autojs/external/ScriptIntents.kt b/app/src/main/java/org/autojs/autojs/external/ScriptIntents.kt index 6c9e057d4f..dbf79a5909 100644 --- a/app/src/main/java/org/autojs/autojs/external/ScriptIntents.kt +++ b/app/src/main/java/org/autojs/autojs/external/ScriptIntents.kt @@ -37,6 +37,14 @@ object ScriptIntents { const val EXTRA_KEY_SOURCE_KIND = "source_kind" const val EXTRA_KEY_PROJECT_ROOT = "project_root" + // Task-level virtual screen launch options (docs/virtual-screen/README.md). + // 0 = follow the global setting, 1 = force on, 2 = force off. + const val EXTRA_KEY_VIRTUAL_SCREEN_MODE = "virtual_screen_mode" + const val EXTRA_KEY_VIRTUAL_SCREEN_WIDTH = "virtual_screen_width" + const val EXTRA_KEY_VIRTUAL_SCREEN_HEIGHT = "virtual_screen_height" + const val EXTRA_KEY_VIRTUAL_SCREEN_DENSITY = "virtual_screen_density" + const val EXTRA_KEY_VIRTUAL_SCREEN_DISPLAY_ID = "virtual_screen_display_id" + data class PreparedLaunch( val launch: ScriptLaunchSourceFactory.Launch, val config: ExecutionConfig, @@ -289,6 +297,18 @@ object ScriptIntents { delay = intent.getLongExtra(EXTRA_KEY_DELAY, 0) loopTimes = intent.getIntExtra(EXTRA_KEY_LOOP_TIMES, 1) interval = intent.getLongExtra(EXTRA_KEY_LOOP_INTERVAL, 0) + virtualScreenMode = intent.getIntExtra(EXTRA_KEY_VIRTUAL_SCREEN_MODE, ExecutionConfig.VIRTUAL_SCREEN_MODE_FOLLOW) + virtualScreenWidth = intent.getIntExtra(EXTRA_KEY_VIRTUAL_SCREEN_WIDTH, 0) + virtualScreenHeight = intent.getIntExtra(EXTRA_KEY_VIRTUAL_SCREEN_HEIGHT, 0) + virtualScreenDensity = intent.getIntExtra(EXTRA_KEY_VIRTUAL_SCREEN_DENSITY, 0) + virtualScreenDisplayId = intent.getIntExtra(EXTRA_KEY_VIRTUAL_SCREEN_DISPLAY_ID, 0) + intent.readLaunchJsonOrNull()?.let { json -> + virtualScreenMode = json.optInt(EXTRA_KEY_VIRTUAL_SCREEN_MODE, virtualScreenMode) + virtualScreenWidth = json.optInt(EXTRA_KEY_VIRTUAL_SCREEN_WIDTH, virtualScreenWidth) + virtualScreenHeight = json.optInt(EXTRA_KEY_VIRTUAL_SCREEN_HEIGHT, virtualScreenHeight) + virtualScreenDensity = json.optInt(EXTRA_KEY_VIRTUAL_SCREEN_DENSITY, virtualScreenDensity) + virtualScreenDisplayId = json.optInt(EXTRA_KEY_VIRTUAL_SCREEN_DISPLAY_ID, virtualScreenDisplayId) + } if (shouldExposeAndroidIntentArgument(engineName)) { setArgument("intent", intent) } diff --git a/app/src/main/java/org/autojs/autojs/model/script/Scripts.kt b/app/src/main/java/org/autojs/autojs/model/script/Scripts.kt index 2488fd04c5..d8d78a298b 100644 --- a/app/src/main/java/org/autojs/autojs/model/script/Scripts.kt +++ b/app/src/main/java/org/autojs/autojs/model/script/Scripts.kt @@ -144,6 +144,32 @@ object Scripts { null } + /** + * Runs [file] on a fresh background virtual display without touching its JavaScript source. + * The display is created by the runtime when the script starts and released when it exits. + * + * zh-CN: 让 [file] 在新的后台虚拟屏上运行, 无需修改脚本源码. 虚拟屏在脚本启动时创建, 退出时释放. + */ + @JvmStatic + fun runOnVirtualScreen(context: Context, file: ScriptFile): ScriptExecution? = try { + val launchStartedAt = android.os.SystemClock.elapsedRealtime() + val persisted = ScriptLaunchSourceFactory.capturePersistedLaunchContext(file.path) + val launch = ScriptLaunchSourceFactory.createPersistedPathLaunch( + path = file.path, + sourceKind = persisted.sourceKind, + projectRoot = persisted.projectRoot, + ) + val config = launch.createExecutionConfig().apply { + nodeLaunchStartedAtMs = launchStartedAt + virtualScreenMode = ExecutionConfig.VIRTUAL_SCREEN_MODE_ON + } + executeLaunch(context, launch, config) + } catch (e: Exception) { + e.printStackTrace() + ViewUtils.showToast(context, e.message, true) + null + } + /** Runs a user-selected file and may show the foreground Python plugin availability prompt. */ @JvmStatic fun runInteractive(context: Context, file: ScriptFile): ScriptExecution? = try { diff --git a/app/src/main/java/org/autojs/autojs/runtime/ScriptRuntime.kt b/app/src/main/java/org/autojs/autojs/runtime/ScriptRuntime.kt index 30adee720d..b053801949 100644 --- a/app/src/main/java/org/autojs/autojs/runtime/ScriptRuntime.kt +++ b/app/src/main/java/org/autojs/autojs/runtime/ScriptRuntime.kt @@ -23,7 +23,9 @@ import org.autojs.autojs.core.plugin.r8.AndroidR8CompilerHost import org.autojs.autojs.core.plugin.r8.R8CompilerExplicitRuntimeRoute import org.autojs.autojs.core.plugin.r8.R8RetraceExplicitRuntimeRoute import org.autojs.autojs.core.pref.keys.ExtensibilityKeys +import org.autojs.autojs.core.virtualscreen.DisplayRoute import org.autojs.autojs.engine.RhinoJavaScriptEngine +import org.autojs.autojs.execution.ExecutionConfig import org.autojs.autojs.lang.ThreadCompat import org.autojs.autojs.pio.PFiles import org.autojs.autojs.pio.UncheckedIOException @@ -133,6 +135,14 @@ class ScriptRuntime private constructor(builder: Builder) { val ownerId = "runtime@${System.identityHashCode(this)}" + /** + * Screen scope of this execution (docs/virtual-screen/README.md). All screen-facing APIs read + * it, so a task configured to run on a virtual display needs no changes to its JavaScript. + * zh-CN: 本次执行的屏幕作用域. 所有面向屏幕的 API 都读取它, 因此配置为虚拟屏运行的任务无需修改 JS 源码. + */ + @JvmField + val displayRoute = DisplayRoute(this) + private var mUiHandlerAppContext: Context private val mProperties = ConcurrentHashMap() @@ -515,6 +525,21 @@ class ScriptRuntime private constructor(builder: Builder) { } fun initPrologue() { + // Resolve the task's display before any screen-facing module is mounted. The engine carries + // ExecutionConfig as a tag (RunnableScriptExecution#execute), so no launch path needs changing. + // zh-CN: 在挂载任何屏幕相关模块前先解析本次任务的 display. ExecutionConfig 已作为 engine tag 传递, 无需改动任何启动链路. + val config = runCatching { engines.myEngine().getTag(ExecutionConfig.tag) as? ExecutionConfig }.getOrNull() + displayRoute.apply(config) + if (displayRoute.isVirtual) { + // selector(), UiObject, auto.windows() and node actions are rooted in service.windows, + // which only reports the DEFAULT display. Pin the bridge to this display and switch it + // to getWindowsOnAllDisplays(); a plain window filter cannot help when the virtual + // display's windows are not in the list at all. + // zh-CN: 控件查找的根节点来自 service.windows, 而它只报告默认屏. 必须把 bridge 钉在本 display + // 并切换到 getWindowsOnAllDisplays(); 窗口根本不在列表里时, 普通窗口过滤器无济于事. + accessibilityBridge.setDisplayScope(displayRoute.displayId) + } + LooperHelper.prepare() mThread = Thread.currentThread() threads = ApiThreads(this) diff --git a/app/src/main/java/org/autojs/autojs/runtime/api/Images.java b/app/src/main/java/org/autojs/autojs/runtime/api/Images.java index ea43d14d72..aa2b4f6907 100644 --- a/app/src/main/java/org/autojs/autojs/runtime/api/Images.java +++ b/app/src/main/java/org/autojs/autojs/runtime/api/Images.java @@ -302,6 +302,12 @@ protected static void setImageCaptureCallback(ScriptRuntime scriptRuntime, OnScr } public ScriptPromiseAdapter requestScreenCapture(int orientation, int width, int height, boolean isAsync) { + // A virtual display is captured through its own ImageReader surface, so no MediaProjection + // permission dialog is needed (and MediaProjection can only mirror the default display). + // zh-CN: 虚拟屏由自身的 ImageReader surface 截图, 无需 MediaProjection 授权. + if (mScriptRuntime.displayRoute.isVirtual()) { + return alreadyGrantedScreenCaptureRequest(); + } final ScriptPromiseAdapter waiter = new ScriptPromiseAdapter(); final ScreenCapturer.Options options = new ScreenCapturer.Options( width, height, orientation, ScreenMetrics.getDeviceScreenDensity(), isAsync @@ -401,7 +407,7 @@ public void onRequestError(@NonNull Throwable t) { } public synchronized boolean hasActiveScreenCapturer() { - return mScreenCapturer != null && !mScreenCapturer.isReleased(); + return mScriptRuntime.displayRoute.isVirtual() || (mScreenCapturer != null && !mScreenCapturer.isReleased()); } private ScriptPromiseAdapter alreadyGrantedScreenCaptureRequest() { @@ -412,6 +418,11 @@ private ScriptPromiseAdapter alreadyGrantedScreenCaptureRequest() { @Nullable public ImageWrapper captureScreen() { + if (mScriptRuntime.displayRoute.isVirtual()) { + // Latest frame of the virtual display, as an ImageWrapper so all images.* helpers work. + // zh-CN: 取虚拟屏最新帧并包装为 ImageWrapper, images.* 的找色/找图能力可直接使用. + return mScriptRuntime.displayRoute.capture(); + } synchronized (this) { ScreenCapturer screenCapturer = mScreenCapturer; if (screenCapturer == null || screenCapturer.isReleased()) { diff --git a/app/src/main/java/org/autojs/autojs/runtime/api/WrappedShizuku.kt b/app/src/main/java/org/autojs/autojs/runtime/api/WrappedShizuku.kt index 8a1aad0094..566f226287 100644 --- a/app/src/main/java/org/autojs/autojs/runtime/api/WrappedShizuku.kt +++ b/app/src/main/java/org/autojs/autojs/runtime/api/WrappedShizuku.kt @@ -89,8 +89,9 @@ object WrappedShizuku { .processNameSuffix("shizuku-service-for-${pkg.substringAfterLast(".")}") // Recreate an older user process when its AIDL implementation changes after an update. // The extra revision also replaces a service from an earlier build with the same versionCode - // (+ 6: attachHost, docs/shizuku-oom/ROADMAP.md D5). - .version(BuildConfig.VERSION_CODE + 6) + // (+ 6: attachHost, docs/shizuku-oom/ROADMAP.md D5; + 7: virtual screen AIDL, + // docs/virtual-screen/README.md). Bump it again whenever IUserService.aidl changes. + .version(BuildConfig.VERSION_CODE + 8) .daemon(false) } diff --git a/app/src/main/java/org/autojs/autojs/runtime/api/modules/ScriptModuleLayer.kt b/app/src/main/java/org/autojs/autojs/runtime/api/modules/ScriptModuleLayer.kt index 3c5ddf8115..c068c319c2 100644 --- a/app/src/main/java/org/autojs/autojs/runtime/api/modules/ScriptModuleLayer.kt +++ b/app/src/main/java/org/autojs/autojs/runtime/api/modules/ScriptModuleLayer.kt @@ -36,7 +36,7 @@ object ScriptModuleLayer { "files", "files.batchRename", "crypto", "RootAutomator", "engines", "floaty", "storages", "device", "powerManager", "settings", "recorder", "toast", "media", "sensors", "base64", "notice", "notice.channel", "shizuku", "opencc", "mime", "sysprops", "sqlite", "zip", "nanoid", "pinyin", "pinyin4j", "mediainfo", "mail", "installer", "epub", "gallery", "terminal", - "speech", "speech.models", "speech.speaker", "compose", "memory", "imgui", + "speech", "speech.models", "speech.speaker", "compose", "memory", "imgui", "virtualScreen", ) /** The top-level entries of [ORDER], the names `ScriptRuntime.augment()` mounts in turn. */ diff --git a/app/src/main/java/org/autojs/autojs/runtime/api/modules/app/App.kt b/app/src/main/java/org/autojs/autojs/runtime/api/modules/app/App.kt index f155f9c8cb..878fa28996 100644 --- a/app/src/main/java/org/autojs/autojs/runtime/api/modules/app/App.kt +++ b/app/src/main/java/org/autojs/autojs/runtime/api/modules/app/App.kt @@ -100,8 +100,8 @@ class App(scriptRuntime: ScriptRuntime) : ArgumentGuards() { when (obj) { is String -> when { !options.isJsNullish() -> throw IllegalArgumentException("Method app.startActivity can only have one argument when argument[0] is String") - obj.contains("://") -> openUrlRhino(obj) - rexWebSiteWithoutProtocol.matches(obj) -> openUrlRhino("http://$obj") + obj.contains("://") -> openUrlOnCurrentDisplay(scriptRuntime, obj) + rexWebSiteWithoutProtocol.matches(obj) -> openUrlOnCurrentDisplay(scriptRuntime, "http://$obj") else -> runCatching { startActivityWithGlobalContext(Intent(GlobalAppContext.get(), ActivityShortForm.valueOf(obj.uppercase()).classType)) }.getOrElse { @@ -131,7 +131,7 @@ class App(scriptRuntime: ScriptRuntime) : ArgumentGuards() { } is Uri, is URI -> when { !options.isJsNullish() -> throw IllegalArgumentException("Method app.startActivity can only have one argument when argument[0] is URI") - else -> openUrlRhino(obj.toString()) + else -> openUrlOnCurrentDisplay(scriptRuntime, obj.toString()) } is NativeObject -> { val opt = when { @@ -502,7 +502,24 @@ class App(scriptRuntime: ScriptRuntime) : ArgumentGuards() { @JvmStatic @RhinoRuntimeFunctionInterface fun openUrl(scriptRuntime: ScriptRuntime, args: Array): Undefined = ensureArgumentsOnlyOne(args) { - openUrlRhino(it) + openUrlOnCurrentDisplay(scriptRuntime, it) + } + + /** + * Resolves a URL on the display this runtime is bound to. Without this, `app.openUrl()` + * (and the URL form of `app.startActivity`) always opens on the DEFAULT display and would + * pull a virtual-screen task's app link onto the physical screen. + * + * zh-CN: 把 URL 打开到本运行时绑定的屏幕. 否则 `app.openUrl()` 永远落在默认屏, + * 会把虚拟屏任务的 applink 拉到物理主屏上. + */ + private fun openUrlOnCurrentDisplay(scriptRuntime: ScriptRuntime, url: Any?): Undefined = undefined { + val target = coerceString(url) + if (scriptRuntime.displayRoute.isVirtual) { + scriptRuntime.displayRoute.launchUri(target) + } else { + openUrlInternal(target) + } } @JvmStatic @@ -632,8 +649,8 @@ class App(scriptRuntime: ScriptRuntime) : ArgumentGuards() { fun launchPackage(scriptRuntime: ScriptRuntime, args: Array): Boolean = ensureArgumentsOnlyOne(args) { o -> when { o.isJsNullish() -> false - o is PresetApp -> scriptRuntime.app.launchPackage(o.packageName) - else -> scriptRuntime.app.launchPackage(getAppByAliasRhino(o)?.packageName ?: Context.toString(o)) + o is PresetApp -> launchOnCurrentDisplay(scriptRuntime, o.packageName) + else -> launchOnCurrentDisplay(scriptRuntime, getAppByAliasRhino(o)?.packageName ?: Context.toString(o)) } } @@ -654,16 +671,27 @@ class App(scriptRuntime: ScriptRuntime) : ArgumentGuards() { fun launchApp(scriptRuntime: ScriptRuntime, args: Array): Boolean = ensureArgumentsOnlyOne(args) { o -> when { o.isJsNullish() -> false - o is PresetApp -> scriptRuntime.app.launchApp(o.getAppName()) + o is PresetApp -> launchByAppName(scriptRuntime, o.getAppName()) else -> getAppByAliasRhino(o)?.let { // @Hint by SuperMonster003 on May 23, 2024. // ! Much more efficient to launch with package name. // ! zh-CN: 使用包名启动应用将会高效得多. - scriptRuntime.app.launchPackage(it.packageName) - } ?: scriptRuntime.app.launchApp(Context.toString(o)) + launchOnCurrentDisplay(scriptRuntime, it.packageName) + } ?: launchByAppName(scriptRuntime, Context.toString(o)) } } + /** Launching an app must target the task's display, not the default one. */ + private fun launchOnCurrentDisplay(scriptRuntime: ScriptRuntime, packageName: String): Boolean = + if (scriptRuntime.displayRoute.isVirtual) scriptRuntime.displayRoute.launch(packageName) + else scriptRuntime.app.launchPackage(packageName) + + private fun launchByAppName(scriptRuntime: ScriptRuntime, appName: String): Boolean { + if (!scriptRuntime.displayRoute.isVirtual) return scriptRuntime.app.launchApp(appName) + val packageName = scriptRuntime.app.getPackageName(appName) ?: return false + return scriptRuntime.displayRoute.launch(packageName) + } + @JvmStatic @RhinoRuntimeFunctionInterface fun launchDualApp(scriptRuntime: ScriptRuntime, args: Array): Boolean = ensureArgumentsOnlyOne(args) { o -> diff --git a/app/src/main/java/org/autojs/autojs/runtime/api/modules/shell/Shell.kt b/app/src/main/java/org/autojs/autojs/runtime/api/modules/shell/Shell.kt index 73bd9b46cb..8b467c935e 100644 --- a/app/src/main/java/org/autojs/autojs/runtime/api/modules/shell/Shell.kt +++ b/app/src/main/java/org/autojs/autojs/runtime/api/modules/shell/Shell.kt @@ -345,26 +345,43 @@ class Shell(private val scriptRuntime: ScriptRuntime) : ArgumentGuards() { @JvmStatic @RhinoRuntimeFunctionInterface fun Text(scriptRuntime: ScriptRuntime, args: Array): Undefined = ensureArgumentsOnlyOne(args) { - undefined { compatShell(scriptRuntime).Text(coerceString(it)) } + undefined { + val value = coerceString(it) + if (scriptRuntime.displayRoute.isVirtual) scriptRuntime.displayRoute.text(value) + else compatShell(scriptRuntime).Text(value) + } } @JvmStatic @RhinoRuntimeFunctionInterface fun Input(scriptRuntime: ScriptRuntime, args: Array): Undefined = ensureArgumentsOnlyOne(args) { - undefined { compatShell(scriptRuntime).Text(coerceString(it)) } + undefined { + val value = coerceString(it) + if (scriptRuntime.displayRoute.isVirtual) scriptRuntime.displayRoute.text(value) + else compatShell(scriptRuntime).Text(value) + } } @JvmStatic @RhinoRuntimeFunctionInterface fun Tap(scriptRuntime: ScriptRuntime, args: Array): Undefined = ensureArgumentsLength(args, 2) { val (x, y) = it - undefined { compatShell(scriptRuntime).Tap(coerceIntNumber(x), coerceIntNumber(y)) } + undefined { + val px = coerceIntNumber(x) + val py = coerceIntNumber(y) + if (scriptRuntime.displayRoute.isVirtual) scriptRuntime.displayRoute.tap(px, py) + else compatShell(scriptRuntime).Tap(px, py) + } } @JvmStatic @RhinoRuntimeFunctionInterface fun Screencap(scriptRuntime: ScriptRuntime, args: Array): Undefined = ensureArgumentsOnlyOne(args) { - undefined { compatShell(scriptRuntime).Screencap(coerceString(it)) } + undefined { + val path = coerceString(it) + if (scriptRuntime.displayRoute.isVirtual) scriptRuntime.displayRoute.captureToFile(path) + else compatShell(scriptRuntime).Screencap(path) + } } @JvmStatic @@ -375,6 +392,10 @@ class Shell(private val scriptRuntime: ScriptRuntime) : ArgumentGuards() { require(codeNumber != KeyEvent.KEYCODE_UNKNOWN) { GlobalAppContext.get().getString(R.string.error_unknown_key_code, raw) } + if (scriptRuntime.displayRoute.isVirtual) { + scriptRuntime.displayRoute.key(codeNumber) + return@ensureArgumentsOnlyOne UNDEFINED + } val defaults = defaultsOf(scriptRuntime) when { defaults?.adb == true -> defaults.adbShell.KeyCode(codeNumber) @@ -413,20 +434,16 @@ class Shell(private val scriptRuntime: ScriptRuntime) : ArgumentGuards() { fun Swipe(scriptRuntime: ScriptRuntime, args: Array): Undefined = ensureArgumentsLengthInRange(args, 4..5) { val (x1, y1, x2, y2, duration) = it undefined { - when { - duration.isJsNullish() -> compatShell(scriptRuntime).Swipe( - coerceIntNumber(x1), - coerceIntNumber(y1), - coerceIntNumber(x2), - coerceIntNumber(y2), - ) - else -> compatShell(scriptRuntime).Swipe( - coerceIntNumber(x1), - coerceIntNumber(y1), - coerceIntNumber(x2), - coerceIntNumber(y2), - coerceIntNumber(duration), - ) + val px1 = coerceIntNumber(x1) + val py1 = coerceIntNumber(y1) + val px2 = coerceIntNumber(x2) + val py2 = coerceIntNumber(y2) + if (scriptRuntime.displayRoute.isVirtual) { + val ms = if (duration.isJsNullish()) 300 else coerceIntNumber(duration) + scriptRuntime.displayRoute.swipe(px1, py1, px2, py2, ms.coerceAtLeast(1)) + } else when { + duration.isJsNullish() -> compatShell(scriptRuntime).Swipe(px1, py1, px2, py2) + else -> compatShell(scriptRuntime).Swipe(px1, py1, px2, py2, coerceIntNumber(duration)) } } } diff --git a/app/src/main/java/org/autojs/autojs/runtime/api/modules/virtualscreen/VirtualScreenModule.kt b/app/src/main/java/org/autojs/autojs/runtime/api/modules/virtualscreen/VirtualScreenModule.kt new file mode 100644 index 0000000000..0b2bacbc5b --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/runtime/api/modules/virtualscreen/VirtualScreenModule.kt @@ -0,0 +1,90 @@ +package org.autojs.autojs.runtime.api.modules.virtualscreen + +import org.autojs.autojs.app.GlobalAppContext +import org.autojs.autojs.core.virtualscreen.VirtualScreenManager +import org.autojs.autojs.core.virtualscreen.VirtualScreenSession +import org.autojs.autojs.runtime.ScriptRuntime +import org.autojs.autojs.runtime.api.modules.ScriptFunction +import org.autojs.autojs.runtime.api.modules.ScriptModule +import org.autojs.autojs.util.RhinoUtils +import org.mozilla.javascript.NativeObject +import org.mozilla.javascript.Scriptable +import org.mozilla.javascript.ScriptableObject + +/** + * `virtualScreen`: runs automation on a hidden background display instead of the physical screen. + * + * The display is created as the shell uid by the Shizuku user service (see + * docs/virtual-screen/README.md). The script keeps a session object whose methods act on that + * display; the session is released automatically when the script exits. + * + * zh-CN: `virtualScreen` 模块: 把自动化操作放到一块隐藏的后台显示屏上执行, 不占用物理主屏. + * 显示屏由 Shizuku 用户服务以 shell 身份创建 (见 docs/virtual-screen/README.md). 脚本持有会话对象, + * 其方法都作用于该屏; 脚本退出时自动释放. + * + * Created on Nov 8, 2026. + */ +@ScriptModule(name = "virtualScreen") +class VirtualScreenModule(private val runtime: ScriptRuntime) { + + /** + * Creates a virtual screen. Options: width, height, dpi, flags (0 = automatic). The default + * geometry is the current main display's metrics. + * + * zh-CN: 创建虚拟屏. 选项: width / height / dpi / flags (0 表示自动); 默认尺寸取当前主屏参数. + */ + @ScriptFunction + fun create(options: Any? = null): VirtualScreenSession { + val metrics = GlobalAppContext.get().resources.displayMetrics + val source = options as? Scriptable + val width = intOption(source, "width", metrics.widthPixels) + val height = intOption(source, "height", metrics.heightPixels) + val dpi = intOption(source, "dpi", metrics.densityDpi) + val flags = intOption(source, "flags", 0) + return VirtualScreenManager.create(runtime, width, height, dpi, flags) + } + + /** Returns the live session of [displayId], or null. */ + @ScriptFunction + fun session(displayId: Int): VirtualScreenSession? = VirtualScreenManager.get(displayId) + + /** Display IDs of all live sessions. */ + @ScriptFunction + fun screens(): IntArray = VirtualScreenManager.ids() + + /** Releases one display; returns false when it is already gone. */ + @ScriptFunction + fun release(displayId: Int): Boolean = VirtualScreenManager.release(displayId) + + /** Releases every display created by this process; returns the number released. */ + @ScriptFunction + fun releaseAll(): Int = VirtualScreenManager.releaseAll() + + /** + * Diagnostics: Shizuku availability, Android version, whether own-focus is supported and the + * live display IDs. Call this before creating a screen to explain a failure to the user. + * + * zh-CN: 诊断信息: Shizuku 可用性、Android 版本、是否支持独立焦点及当前显示屏 ID 列表. + */ + @ScriptFunction + fun state(): NativeObject = RhinoUtils.newNativeObject(runtime.topLevelScope).also { o -> + val ids = VirtualScreenManager.ids() + // A real NativeArray, not a Java int[] that scripts would see as an opaque Java object. + val array = RhinoUtils.newNativeArray(ids.size.toLong()) + ids.forEachIndexed { index, id -> array.put(index, array, id) } + o.defineProperty("available", VirtualScreenManager.isAvailable(), ScriptableObject.EMPTY) + o.defineProperty("sdkInt", VirtualScreenManager.sdkInt(), ScriptableObject.EMPTY) + o.defineProperty("ownFocusSupported", VirtualScreenManager.isOwnFocusSupported(), ScriptableObject.EMPTY) + o.defineProperty("screenCount", VirtualScreenManager.screenCount(), ScriptableObject.EMPTY) + o.defineProperty("displayIds", array, ScriptableObject.EMPTY) + o.defineProperty("minRecommendedSdkInt", 31, ScriptableObject.EMPTY) + o.defineProperty("ownFocusSdkInt", 34, ScriptableObject.EMPTY) + } + + private fun intOption(options: Scriptable?, name: String, fallback: Int): Int { + if (options == null) return fallback + val value = runCatching { ScriptableObject.getProperty(options, name) }.getOrNull() + return (value as? Number)?.toInt() ?: fallback + } + +} diff --git a/app/src/main/java/org/autojs/autojs/storage/database/TimedTaskDatabase.java b/app/src/main/java/org/autojs/autojs/storage/database/TimedTaskDatabase.java index 70e52ddfd1..91b3532a40 100644 --- a/app/src/main/java/org/autojs/autojs/storage/database/TimedTaskDatabase.java +++ b/app/src/main/java/org/autojs/autojs/storage/database/TimedTaskDatabase.java @@ -7,6 +7,7 @@ import android.database.sqlite.SQLiteOpenHelper; import android.util.Log; +import org.autojs.autojs.execution.ExecutionConfig; import org.autojs.autojs.timing.TimedTask; /** @@ -22,7 +23,7 @@ */ public class TimedTaskDatabase extends Database { - public static final int VERSION = 5; + public static final int VERSION = 6; private static final String NAME = "TimedTaskDatabase"; public static final String FILE_NAME = NAME + ".db"; @@ -46,6 +47,7 @@ protected ContentValues asContentValues(TimedTask model) { values.put("end_mode", model.getEndMode()); values.put("end_value", model.getEndValue()); values.put("finished_times", model.getFinishedTimes()); + values.put("virtual_screen_mode", model.getVirtualScreenMode()); values.put("day_of_month", model.getDayOfMonth()); values.put("month_of_year", model.getMonthOfYear()); return values; @@ -66,6 +68,7 @@ protected TimedTask createModelFromCursor(Cursor cursor) { task.setEndMode(getInt(cursor, "end_mode", TimedTask.END_MODE_FOREVER)); task.setEndValue(getLong(cursor, "end_value", 0)); task.setFinishedTimes(getInt(cursor, "finished_times", 0)); + task.setVirtualScreenMode(getInt(cursor, "virtual_screen_mode", ExecutionConfig.VIRTUAL_SCREEN_MODE_FOLLOW)); task.setDayOfMonth(getInt(cursor, "day_of_month", 0)); task.setMonthOfYear(getInt(cursor, "month_of_year", 0)); task.setSourceKind(getString(cursor, "source_kind")); @@ -112,7 +115,8 @@ public void onCreate(SQLiteDatabase db) { "`day_of_month` INTEGER DEFAULT 0, " + "`month_of_year` INTEGER DEFAULT 0, " + "`source_kind` TEXT, " + - "`project_root` TEXT);"); + "`project_root` TEXT, " + + "`virtual_screen_mode` INTEGER DEFAULT 0);"); } @Override @@ -129,6 +133,9 @@ public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { addColumn(db, "source_kind", "TEXT"); addColumn(db, "project_root", "TEXT"); } + if (oldVersion < 6) { + addColumn(db, "virtual_screen_mode", "INTEGER DEFAULT 0"); + } } /** diff --git a/app/src/main/java/org/autojs/autojs/timing/TimedTask.java b/app/src/main/java/org/autojs/autojs/timing/TimedTask.java index 76c8a16098..8e3aea4001 100644 --- a/app/src/main/java/org/autojs/autojs/timing/TimedTask.java +++ b/app/src/main/java/org/autojs/autojs/timing/TimedTask.java @@ -75,6 +75,9 @@ public class TimedTask extends BaseModel { int mFinishedTimes = 0; + /** Task-level virtual screen mode; FOLLOW uses the global switch / project feature. */ + int mVirtualScreenMode = ExecutionConfig.VIRTUAL_SCREEN_MODE_FOLLOW; + int mDayOfMonth = 0; int mMonthOfYear = 0; @@ -92,6 +95,7 @@ public TimedTask(long millis, long timeFlag, String scriptPath, ExecutionConfig mDelay = config.getDelay(); mLoopTimes = config.getLoopTimes(); mInterval = config.getInterval(); + mVirtualScreenMode = config.getVirtualScreenMode(); mRepeatMode = inferRepeatMode(timeFlag); captureLaunchContext(); } @@ -354,6 +358,14 @@ public void setFinishedTimes(int finishedTimes) { mFinishedTimes = finishedTimes; } + public int getVirtualScreenMode() { + return mVirtualScreenMode; + } + + public void setVirtualScreenMode(int virtualScreenMode) { + mVirtualScreenMode = virtualScreenMode; + } + public int getDayOfMonth() { return mDayOfMonth; } @@ -382,7 +394,8 @@ public Intent createIntent() { .putExtra(ScriptIntents.EXTRA_KEY_PATH, mScriptPath) .putExtra(ScriptIntents.EXTRA_KEY_DELAY, mDelay) .putExtra(ScriptIntents.EXTRA_KEY_LOOP_TIMES, mLoopTimes) - .putExtra(ScriptIntents.EXTRA_KEY_LOOP_INTERVAL, mInterval); + .putExtra(ScriptIntents.EXTRA_KEY_LOOP_INTERVAL, mInterval) + .putExtra(ScriptIntents.EXTRA_KEY_VIRTUAL_SCREEN_MODE, mVirtualScreenMode); if (mSourceKind != null) { intent.putExtra(ScriptIntents.EXTRA_KEY_SOURCE_KIND, mSourceKind); } @@ -412,6 +425,7 @@ public String toString() { ", mEndMode=" + mEndMode + ", mEndValue=" + mEndValue + ", mFinishedTimes=" + mFinishedTimes + + ", mVirtualScreenMode=" + mVirtualScreenMode + ", mDayOfMonth=" + mDayOfMonth + ", mMonthOfYear=" + mMonthOfYear + ", mScriptPath='" + mScriptPath + '\'' + diff --git a/app/src/main/java/org/autojs/autojs/ui/explorer/ExplorerBuiltInActionHandler.kt b/app/src/main/java/org/autojs/autojs/ui/explorer/ExplorerBuiltInActionHandler.kt index d6dae2db54..c986caa98d 100644 --- a/app/src/main/java/org/autojs/autojs/ui/explorer/ExplorerBuiltInActionHandler.kt +++ b/app/src/main/java/org/autojs/autojs/ui/explorer/ExplorerBuiltInActionHandler.kt @@ -73,6 +73,11 @@ internal class ExplorerBuiltInActionHandler( host.notifyItemOperated() host.hideHostDialog() } + R.id.action_run_on_virtual_screen -> { + Scripts.runOnVirtualScreen(host.context, host.selectedItem!!.toScriptFile()) + host.notifyItemOperated() + host.hideHostDialog() + } R.id.action_run_repeatedly -> { ScriptLoopDialog(host.context, host.selectedItem!!.toScriptFile()).show() host.notifyItemOperated() diff --git a/app/src/main/java/org/autojs/autojs/ui/explorer/ExplorerItemViewHolder.kt b/app/src/main/java/org/autojs/autojs/ui/explorer/ExplorerItemViewHolder.kt index 9e5e076ae9..9d015d5e0a 100644 --- a/app/src/main/java/org/autojs/autojs/ui/explorer/ExplorerItemViewHolder.kt +++ b/app/src/main/java/org/autojs/autojs/ui/explorer/ExplorerItemViewHolder.kt @@ -660,6 +660,7 @@ internal class ExplorerItemViewHolder( if (!mExplorerItem.isExecutable) { menu.removeItem(R.id.action_create_shortcut) menu.removeItem(R.id.action_timed_task) + menu.removeItem(R.id.action_run_on_virtual_screen) menu.removeItem(R.id.action_run_repeatedly) menu.removeItem(R.id.action_floating_run) } diff --git a/app/src/main/java/org/autojs/autojs/ui/main/drawer/DrawerFragment.kt b/app/src/main/java/org/autojs/autojs/ui/main/drawer/DrawerFragment.kt index d8191747c6..2772b81334 100644 --- a/app/src/main/java/org/autojs/autojs/ui/main/drawer/DrawerFragment.kt +++ b/app/src/main/java/org/autojs/autojs/ui/main/drawer/DrawerFragment.kt @@ -78,6 +78,7 @@ import org.autojs.autojs.ui.settings.PreferencesActivity import org.autojs.autojs.ui.settings.TrashAndVersionHistorySettingsActivity import org.autojs.autojs.ui.storage.TrashActivity import org.autojs.autojs.ui.storage.VersionHistoryActivity +import org.autojs.autojs.ui.virtualscreen.VirtualScreenViewerActivity import org.autojs.autojs.ui.wizard.SetupWizards import org.autojs.autojs.util.ActivityLaunchOptions import org.autojs.autojs.util.IntentUtils.App.exit @@ -160,6 +161,7 @@ open class DrawerFragment : Fragment() { private lateinit var mTrashItem: DrawerMenuShortcutItem private lateinit var mVersionHistoryItem: DrawerMenuShortcutItem private lateinit var mAboutAppAndDevItem: DrawerMenuShortcutItem + private lateinit var mVirtualScreenViewerItem: DrawerMenuShortcutItem private lateinit var mA11yTool: AccessibilityTool private lateinit var mScreenColorPickerTool: ScreenColorPickerTool @@ -770,6 +772,14 @@ open class DrawerFragment : Fragment() { } TerminalPluginStateMonitor.addListener(mContext, terminalPluginStateListener) + mVirtualScreenViewerItem = DrawerMenuShortcutItem( + icon = R.drawable.ic_ali_fullscreen, + title = R.string.text_virtual_screen_viewer, + ).apply { + setAction { VirtualScreenViewerActivity.launch(mContext) } + subtitle = mContext.getString(R.string.summary_virtual_screen_viewer) + } + mThemeColorItem = DrawerMenuShortcutItem( icon = R.drawable.ic_personalize_thicker, title = R.string.text_theme_color, @@ -1044,6 +1054,7 @@ open class DrawerFragment : Fragment() { mAccessibilityServiceItem, mForegroundServiceItem, DrawerMenuGroup(R.string.text_tools), + mVirtualScreenViewerItem, mFloatingButtonItem, mPointerLocationItem, mScreenColorPickerItem, diff --git a/app/src/main/java/org/autojs/autojs/ui/timing/TimedTaskSettingActivity.kt b/app/src/main/java/org/autojs/autojs/ui/timing/TimedTaskSettingActivity.kt index 3d9188dde1..b006771476 100644 --- a/app/src/main/java/org/autojs/autojs/ui/timing/TimedTaskSettingActivity.kt +++ b/app/src/main/java/org/autojs/autojs/ui/timing/TimedTaskSettingActivity.kt @@ -107,6 +107,7 @@ class TimedTaskSettingActivity : BaseActivity() { private lateinit var mRepeatEndCount: RadioButton private lateinit var mRepeatUntilDate: TextView private lateinit var mRepeatCount: EditText + private lateinit var mVirtualScreenModeGroup: RadioGroup private val mDayOfWeekCheckBoxes: MutableList = ArrayList() @@ -227,12 +228,14 @@ class TimedTaskSettingActivity : BaseActivity() { mCountdownTaskMinutes = binding.countdownTaskMinutes.apply { setUpCountdownInput() } + mVirtualScreenModeGroup = binding.virtualScreenModeGroup loadTaskFromIntent() setToolbarAsBack(R.string.text_timed_task) mToolbar.subtitle = mScriptFile.name setUpTaskSettings() + setUpVirtualScreenMode() updateRepeatEndVisibility() updateRepeatEndInputState() @@ -371,6 +374,25 @@ class TimedTaskSettingActivity : BaseActivity() { mDailyTaskRadio.isChecked = true } + /** Restores the persisted task-level virtual screen mode (FOLLOW for a new task). */ + private fun setUpVirtualScreenMode() { + val mode = mTimedTask?.virtualScreenMode ?: ExecutionConfig.VIRTUAL_SCREEN_MODE_FOLLOW + mVirtualScreenModeGroup.check( + when (mode) { + ExecutionConfig.VIRTUAL_SCREEN_MODE_ON -> R.id.virtual_screen_mode_on + ExecutionConfig.VIRTUAL_SCREEN_MODE_OFF -> R.id.virtual_screen_mode_off + else -> R.id.virtual_screen_mode_follow + }, + ) + } + + /** The launch mode written into the task Intent as [ScriptIntents.EXTRA_KEY_VIRTUAL_SCREEN_MODE]. */ + private fun selectedVirtualScreenMode(): Int = when (mVirtualScreenModeGroup.checkedRadioButtonId) { + R.id.virtual_screen_mode_on -> ExecutionConfig.VIRTUAL_SCREEN_MODE_ON + R.id.virtual_screen_mode_off -> ExecutionConfig.VIRTUAL_SCREEN_MODE_OFF + else -> ExecutionConfig.VIRTUAL_SCREEN_MODE_FOLLOW + } + private fun setUpTime(timedTask: TimedTask) { if (timedTask.isCountdown) { mCountdownTaskRadio.isChecked = true @@ -596,6 +618,7 @@ class TimedTaskSettingActivity : BaseActivity() { task.sourceKind = preservedContext.sourceKind task.projectRoot = preservedContext.projectRoot } + task.virtualScreenMode = selectedVirtualScreenMode() return task } diff --git a/app/src/main/java/org/autojs/autojs/ui/virtualscreen/VirtualScreenView.kt b/app/src/main/java/org/autojs/autojs/ui/virtualscreen/VirtualScreenView.kt new file mode 100644 index 0000000000..3d893e9292 --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/ui/virtualscreen/VirtualScreenView.kt @@ -0,0 +1,170 @@ +package org.autojs.autojs.ui.virtualscreen + +import android.content.Context +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.RectF +import android.util.AttributeSet +import android.view.MotionEvent +import android.view.View +import org.autojs.autojs.core.virtualscreen.DisplayCoordinateMapper +import org.autojs.autojs.core.virtualscreen.DisplayCoordinateMapper.Point +import org.autojs.autojs.core.virtualscreen.VirtualScreenSession +import kotlin.math.abs + +/** + * Draws the live frame of a virtual display, letterboxed, and turns touches on the preview into + * logical taps / swipes / long presses on that display. Coordinate conversion lives in + * [DisplayCoordinateMapper] so it can be unit tested without Android. + * + * zh-CN: 等比绘制虚拟屏实时画面, 并把预览上的触摸换算为该屏的点击/滑动/长按. 坐标换算抽到 + * DisplayCoordinateMapper 以便 JVM 单测. + * + * Created for the virtual screen viewer on Nov 8, 2026. + */ +class VirtualScreenView @JvmOverloads constructor( + context: Context, + attrs: AttributeSet? = null, + defStyleAttr: Int = 0, +) : View(context, attrs, defStyleAttr) { + + interface GestureListener { + fun onTap(point: Point) + fun onSwipe(from: Point, to: Point, durationMs: Int) + fun onLongPress(point: Point, durationMs: Int) + } + + var gestureListener: GestureListener? = null + + /** When false the view only displays frames: touches are ignored (used while injecting). */ + var interactionEnabled: Boolean = true + set(value) { + field = value + if (!value) cancelPendingLongPress() + } + + private var session: VirtualScreenSession? = null + private val bitmapPaint = Paint(Paint.FILTER_BITMAP_FLAG or Paint.ANTI_ALIAS_FLAG) + private val framePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + style = Paint.Style.STROKE + color = Color.argb(110, 255, 255, 255) + strokeWidth = 2f + } + private val backgroundPaint = Paint().apply { color = Color.rgb(24, 26, 30) } + private val destination = RectF() + + private var geometry: DisplayCoordinateMapper.Geometry? = null + private var startX = 0f + private var startY = 0f + private var lastX = 0f + private var lastY = 0f + private var startTimeMillis = 0L + private var tracking = false + private var longPressFired = false + + private val longPressRunnable = Runnable { + if (!tracking || longPressFired || !interactionEnabled) return@Runnable + val g = geometry ?: return@Runnable + val point = DisplayCoordinateMapper.toDisplay(g, startX, startY) + longPressFired = true + gestureListener?.onLongPress(point, LONG_PRESS_MILLIS) + } + + fun attach(target: VirtualScreenSession?) { + session = target + geometry = null + cancelPendingLongPress() + invalidate() + } + + fun renderFrame() { + invalidate() + } + + override fun onDraw(canvas: Canvas) { + super.onDraw(canvas) + canvas.drawPaint(backgroundPaint) + val bitmap = session?.previewBitmap() ?: run { + geometry = null + return + } + val g = DisplayCoordinateMapper.fit(width, height, bitmap.width, bitmap.height) + geometry = g + destination.set(g.offsetX, g.offsetY, g.offsetX + g.drawnWidth, g.offsetY + g.drawnHeight) + canvas.drawBitmap(bitmap, null, destination, bitmapPaint) + canvas.drawRect(destination, framePaint) + } + + override fun onTouchEvent(event: MotionEvent): Boolean { + if (!interactionEnabled || session == null) return false + val g = geometry ?: return false + when (event.actionMasked) { + MotionEvent.ACTION_DOWN -> { + if (event.pointerCount != 1 || !DisplayCoordinateMapper.isInside(g, event.x, event.y)) { + tracking = false + return true + } + startX = event.x + startY = event.y + lastX = event.x + lastY = event.y + startTimeMillis = System.currentTimeMillis() + tracking = true + longPressFired = false + postDelayed(longPressRunnable, LONG_PRESS_DELAY_MILLIS) + performClick() + return true + } + MotionEvent.ACTION_MOVE -> { + if (!tracking) return true + lastX = event.x + lastY = event.y + if (abs(lastX - startX) > TOUCH_SLOP || abs(lastY - startY) > TOUCH_SLOP) { + cancelPendingLongPress() + } + return true + } + MotionEvent.ACTION_UP -> { + if (!tracking) return true + cancelPendingLongPress() + tracking = false + if (longPressFired) return true + val from = DisplayCoordinateMapper.toDisplay(g, startX, startY) + val to = DisplayCoordinateMapper.toDisplay(g, event.x, event.y) + val moved = abs(event.x - startX) > TOUCH_SLOP || abs(event.y - startY) > TOUCH_SLOP + val elapsed = (System.currentTimeMillis() - startTimeMillis).toInt() + if (moved) { + gestureListener?.onSwipe(from, to, elapsed.coerceIn(MIN_SWIPE_MILLIS, MAX_SWIPE_MILLIS)) + } else { + gestureListener?.onTap(from) + } + return true + } + MotionEvent.ACTION_CANCEL -> { + cancelPendingLongPress() + tracking = false + return true + } + } + return false + } + + override fun performClick(): Boolean { + // Gestures are forwarded to the virtual display, but keep the accessibility contract. + return super.performClick() + } + + private fun cancelPendingLongPress() { + removeCallbacks(longPressRunnable) + } + + private companion object { + const val TOUCH_SLOP = 12f + const val LONG_PRESS_DELAY_MILLIS = 500L + const val LONG_PRESS_MILLIS = 600 + const val MIN_SWIPE_MILLIS = 100 + const val MAX_SWIPE_MILLIS = 1_200 + } + +} diff --git a/app/src/main/java/org/autojs/autojs/ui/virtualscreen/VirtualScreenViewerActivity.kt b/app/src/main/java/org/autojs/autojs/ui/virtualscreen/VirtualScreenViewerActivity.kt new file mode 100644 index 0000000000..5802b7aab6 --- /dev/null +++ b/app/src/main/java/org/autojs/autojs/ui/virtualscreen/VirtualScreenViewerActivity.kt @@ -0,0 +1,424 @@ +package org.autojs.autojs.ui.virtualscreen + +import android.app.Activity +import android.content.Context +import android.content.Intent +import android.os.Bundle +import android.os.Environment +import android.os.Handler +import android.os.Looper +import android.view.KeyEvent +import android.view.View +import android.widget.EditText +import android.widget.FrameLayout +import android.widget.Toast +import androidx.appcompat.app.AlertDialog +import org.autojs.autojs.core.virtualscreen.DisplayCoordinateMapper +import org.autojs.autojs.core.virtualscreen.VirtualScreenManager +import org.autojs.autojs.core.virtualscreen.VirtualScreenSession +import org.autojs.autojs.runtime.api.WrappedShizuku +import org.autojs.autojs.ui.BaseActivity +import org.autojs.autojs.util.ViewUtils.setToolbarAsBack +import org.autojs.autojs6.R +import org.autojs.autojs6.databinding.ActivityVirtualScreenViewerBinding +import java.io.File +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * "View and operate a virtual screen": a live, letterboxed preview of a background display plus + * direct control over it. Touches become taps / swipes / long presses in that display's logical + * coordinates; the buttons cover navigation keys, text, app launch, PNG capture and screen + * lifecycle (create / switch / release). + * + * A screen created here is *detached* (no ScriptRuntime): it survives leaving this activity and + * can also be reused by a task through `virtualScreenDisplayId`. A screen created by a running + * script can be attached to as well, and the frame pump makes preview and script screenshots + * coexist without starving the ImageReader. + * + * zh-CN: 「虚拟屏幕查看和操作」: 实时等比预览后台虚拟屏, 并直接操作它. 预览上的触摸换算为该屏逻辑坐标的 + * 点击/滑动/长按; 工具栏提供导航键、文本输入、启动应用、保存 PNG 与屏幕生命周期 (新建/切换/释放). + * 本页新建的屏是独立屏 (无 ScriptRuntime), 退出页面后仍然保留, 也可被任务通过 virtualScreenDisplayId 复用; + * 脚本创建的屏同样可以附着查看, 帧泵保证预览与脚本截图互不抢帧. + * + * Created on Nov 8, 2026. See docs/virtual-screen/README.md. + */ +class VirtualScreenViewerActivity : BaseActivity(), VirtualScreenView.GestureListener { + + private lateinit var binding: ActivityVirtualScreenViewerBinding + + private val mainHandler = Handler(Looper.getMainLooper()) + private var io: ExecutorService = Executors.newSingleThreadExecutor() + + private var session: VirtualScreenSession? = null + private var requestedDisplayId = 0 + private var previewPaused = false + private var lastFrameCount = 0L + + private val statusTask = object : Runnable { + override fun run() { + refreshStatus() + mainHandler.postDelayed(this, STATUS_INTERVAL_MILLIS) + } + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + binding = ActivityVirtualScreenViewerBinding.inflate(layoutInflater) + setContentView(binding.root) + binding.toolbar.setNavigationOnClickListener { finish() } + setToolbarAsBack(this, R.string.text_virtual_screen_viewer) + + binding.preview.gestureListener = this + requestedDisplayId = intent.getIntExtra(EXTRA_DISPLAY_ID, 0) + bindButtons() + refreshHint() + refreshStatus() + } + + override fun onStart() { + super.onStart() + val target = when { + requestedDisplayId > 0 -> VirtualScreenManager.get(requestedDisplayId) + else -> VirtualScreenManager.liveSessions().firstOrNull() + } + attach(target) + mainHandler.removeCallbacks(statusTask) + mainHandler.post(statusTask) + } + + override fun onStop() { + mainHandler.removeCallbacks(statusTask) + // Only the frame feed is stopped; the display itself stays alive across activities. + session?.stopPreview() + super.onStop() + } + + override fun onDestroy() { + io.shutdownNow() + binding.preview.attach(null) + super.onDestroy() + } + + // ---- Attachment & preview ---- + + private fun attach(target: VirtualScreenSession?) { + session?.stopPreview() + session = target?.takeIf { !it.isReleased() } + // Keep the selection across background/foreground cycles. + requestedDisplayId = session?.getDisplayId() ?: 0 + binding.preview.attach(session) + lastFrameCount = 0L + previewPaused = false + binding.btnPause.text = getString(R.string.text_vs_pause) + if (session != null) startPreview() + refreshHint() + refreshStatus() + } + + private fun startPreview() { + val target = session ?: return + val ok = runCatching { + target.startPreview { + mainHandler.post { + binding.preview.renderFrame() + if (binding.tvHint.visibility != View.GONE) refreshHint() + } + } + }.onFailure { showError(it) }.getOrDefault(false) + if (!ok) toast(getString(R.string.text_vs_frame_error, target.previewError().orEmpty())) + refreshStatus() + } + + // ---- Status ---- + + private fun refreshStatus() { + val current = session + if (current != null && current.isReleased()) { + // A script exited and released the display underneath the viewer. + session = null + binding.preview.attach(null) + } + val live = session + if (live == null) { + binding.tvStatus.text = getString(R.string.text_vs_status_idle) + binding.tvMetrics.text = when { + !VirtualScreenManager.isAvailable() -> getString(R.string.text_vs_shizuku_required) + else -> getString(R.string.text_vs_no_screen) + } + refreshHint() + return + } + val frames = live.previewFrameCount() + val fps = (frames - lastFrameCount).coerceAtLeast(0).toInt() + lastFrameCount = frames + val kind = getString( + if (live.isDetached()) R.string.text_vs_detached else R.string.text_vs_script_bound, + ) + binding.tvStatus.text = getString( + R.string.text_vs_status, live.getDisplayId(), live.getWidth(), live.getHeight(), live.getDpi(), fps, kind, + ) + binding.tvMetrics.text = buildString { + append("frames=").append(frames) + append(" · preview=").append(if (previewPaused) "paused" else "live") + append(" · trusted=").append(live.isTrusted()) + append(" · ownFocus=").append(live.hasOwnFocus()) + append(" · err=").append(live.previewError() ?: "-") + if (live.previewLastWriteMillis() == 0L) append(" · noFrameYet") + } + refreshHint() + } + + private fun refreshHint() { + val live = session + val hasFrame = live?.previewBitmap() != null + binding.tvHint.visibility = if (live != null && hasFrame && !previewPaused) View.GONE else View.VISIBLE + binding.tvHint.text = when { + live == null -> getString(R.string.text_vs_no_screen) + previewPaused -> getString(R.string.text_vs_preview_paused) + !VirtualScreenManager.isAvailable() -> getString(R.string.text_vs_shizuku_required) + else -> live.previewError()?.let { getString(R.string.text_vs_frame_error, it) } + ?: getString(R.string.text_vs_waiting_frame) + } + } + + // ---- Gestures ---- + + override fun onTap(point: DisplayCoordinateMapper.Point) { + execute("tap") { it.tap(point.x, point.y) } + } + + override fun onSwipe(from: DisplayCoordinateMapper.Point, to: DisplayCoordinateMapper.Point, durationMs: Int) { + execute("swipe") { it.swipe(from.x, from.y, to.x, to.y, durationMs) } + } + + override fun onLongPress(point: DisplayCoordinateMapper.Point, durationMs: Int) { + execute("long press") { it.longPress(point.x, point.y, durationMs) } + } + + // ---- Buttons ---- + + private fun bindButtons() { + binding.btnBack.setOnClickListener { key(KeyEvent.KEYCODE_BACK) } + binding.btnHome.setOnClickListener { key(KeyEvent.KEYCODE_HOME) } + binding.btnRecents.setOnClickListener { key(KeyEvent.KEYCODE_APP_SWITCH) } + binding.btnPower.setOnClickListener { key(KeyEvent.KEYCODE_POWER) } + binding.btnInput.setOnClickListener { promptText() } + binding.btnLaunch.setOnClickListener { promptLaunch() } + binding.btnSave.setOnClickListener { savePng() } + binding.btnPause.setOnClickListener { togglePreview() } + binding.btnRefresh.setOnClickListener { retryFrame() } + binding.btnNew.setOnClickListener { createScreen() } + binding.btnSwitch.setOnClickListener { chooseScreen() } + binding.btnRelease.setOnClickListener { confirmRelease() } + binding.btnGrant.setOnClickListener { grantShizuku() } + } + + private fun key(keyCode: Int) = execute("key $keyCode") { it.key(keyCode) } + + private fun togglePreview() { + val live = session ?: return + if (previewPaused) { + previewPaused = false + binding.btnPause.text = getString(R.string.text_vs_pause) + startPreview() + } else { + previewPaused = true + binding.btnPause.text = getString(R.string.text_vs_resume) + live.stopPreview() + } + refreshHint() + refreshStatus() + } + + /** Re-drains the ImageReader, useful after putting content on a previously empty display. */ + private fun retryFrame() { + val live = session ?: return + if (previewPaused) { + togglePreview() + return + } + if (!live.isPreviewing()) { + startPreview() + } else { + live.drainPreview() + mainHandler.postDelayed({ live.drainPreview() }, 350L) + } + refreshStatus() + } + + private fun createScreen() { + if (!VirtualScreenManager.isAvailable()) { + toast(getString(R.string.text_vs_shizuku_required)) + return + } + val metrics = resources.displayMetrics + val width = metrics.widthPixels + val height = metrics.heightPixels + val dpi = metrics.densityDpi + runOnIo { + val created = runCatching { VirtualScreenManager.createDetached(width, height, dpi, 0) } + created.onSuccess { screen -> + // A freshly created display hosts no window, so some ROMs never submit a frame. + // Put a system app on it first; the polling pump then drains the pending frames. + val launched = runCatching { screen.launch(DEFAULT_LAUNCH_PACKAGE) == 0 }.getOrDefault(false) + mainHandler.post { + attach(screen) + toast( + if (launched) getString(R.string.text_vs_created_hint, screen.getDisplayId()) + else getString(R.string.text_vs_auto_launch_failed), + ) + screen.drainPreview() + mainHandler.postDelayed({ screen.drainPreview(); refreshStatus() }, 600L) + } + }.onFailure { showError(it) } + } + } + + private fun chooseScreen() { + val screens = VirtualScreenManager.liveSessions() + if (screens.isEmpty()) { + toast(getString(R.string.text_vs_no_screen)) + return + } + val labels = screens.map { + getString( + R.string.text_vs_status, it.getDisplayId(), it.getWidth(), it.getHeight(), it.getDpi(), 0, + getString(if (it.isDetached()) R.string.text_vs_detached else R.string.text_vs_script_bound), + ) + }.toTypedArray() + AlertDialog.Builder(this) + .setTitle(R.string.text_vs_screen_choices) + .setItems(labels) { _, index -> attach(screens[index]) } + .setNegativeButton(R.string.text_vs_cancel, null) + .show() + } + + private fun confirmRelease() { + val live = session ?: return + AlertDialog.Builder(this) + .setTitle(R.string.text_vs_release_title) + .setMessage(R.string.text_vs_release_confirm) + .setPositiveButton(R.string.text_vs_ok) { _, _ -> + val id = live.getDisplayId() + runOnIo { + runCatching { live.release() } + mainHandler.post { + toast(getString(R.string.text_vs_released, id)) + attach(VirtualScreenManager.liveSessions().firstOrNull()) + } + } + } + .setNegativeButton(R.string.text_vs_cancel, null) + .show() + } + + private fun promptText() { + val input = EditText(this).apply { hint = getString(R.string.text_vs_input_hint) } + AlertDialog.Builder(this) + .setTitle(R.string.text_vs_input) + .setView(input.inDialogContainer()) + .setPositiveButton(R.string.text_vs_ok) { _, _ -> + val value = input.text.toString() + if (value.isNotEmpty()) execute("text") { it.text(value) } + } + .setNegativeButton(R.string.text_vs_cancel, null) + .show() + } + + private fun promptLaunch() { + val input = EditText(this).apply { hint = getString(R.string.text_vs_launch_hint) } + AlertDialog.Builder(this) + .setTitle(R.string.text_vs_launch) + .setView(input.inDialogContainer()) + .setPositiveButton(R.string.text_vs_ok) { _, _ -> + val target = input.text.toString().trim() + if (target.isEmpty()) return@setPositiveButton + execute("launch") { + if (target.contains('/')) it.launchComponent(target) == 0 else it.launch(target) == 0 + } + } + .setNegativeButton(R.string.text_vs_cancel, null) + .show() + } + + private fun savePng() { + val live = session ?: return + val directory = getExternalFilesDir(Environment.DIRECTORY_PICTURES) ?: filesDir + val stamp = SimpleDateFormat("yyyyMMdd-HHmmss", Locale.US).format(Date()) + val file = File(directory, "virtual-screen-${live.getDisplayId()}-$stamp.png") + runOnIo { + val ok = runCatching { live.captureToFile(file.absolutePath) }.getOrDefault(false) + mainHandler.post { + toast( + if (ok) getString(R.string.text_vs_saved, file.absolutePath) + else getString(R.string.text_vs_action_failed, "save PNG"), + ) + } + } + } + + private fun grantShizuku() { + runCatching { WrappedShizuku.requestPermission() }.onFailure { showError(it) } + } + + // ---- Helpers ---- + + /** Runs a Binder call off the main thread; only a failure produces a toast. */ + private fun execute(label: String, block: (VirtualScreenSession) -> Boolean) { + val live = session ?: return + runOnIo { + val ok = runCatching { block(live) }.onFailure { showError(it) }.getOrDefault(false) + if (!ok) { + mainHandler.post { toast(getString(R.string.text_vs_action_failed, label)) } + } + } + } + + private fun runOnIo(block: () -> Unit) { + if (io.isShutdown || io.isTerminated) return + io.execute { + runCatching { block() }.onFailure { showError(it) } + } + } + + private fun showError(error: Throwable) { + mainHandler.post { toast(getString(R.string.text_vs_error, error.message.orEmpty())) } + } + + private fun toast(message: String) { + Toast.makeText(this, message, Toast.LENGTH_SHORT).show() + } + + private fun EditText.inDialogContainer(): View = FrameLayout(context).apply { + val padding = (16 * resources.displayMetrics.density).toInt() + setPadding(padding, padding / 2, padding, 0) + addView(this@inDialogContainer, FrameLayout.LayoutParams( + FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.WRAP_CONTENT, + )) + } + + companion object { + + /** Optional: attach straight to one display instead of the first live one. */ + const val EXTRA_DISPLAY_ID = "virtual_screen_display_id" + + private const val STATUS_INTERVAL_MILLIS = 1_000L + + /** Present on virtually every Android build; gives a new display visible content. */ + private const val DEFAULT_LAUNCH_PACKAGE = "com.android.settings" + + @JvmStatic + fun launch(context: Context, displayId: Int = 0) { + val intent = Intent(context, VirtualScreenViewerActivity::class.java) + .putExtra(EXTRA_DISPLAY_ID, displayId) + if (context !is Activity) intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + context.startActivity(intent) + } + + } + +} diff --git a/app/src/main/res/layout/activity_timed_task_setting.xml b/app/src/main/res/layout/activity_timed_task_setting.xml index 18631dea68..5eff9f4b91 100644 --- a/app/src/main/res/layout/activity_timed_task_setting.xml +++ b/app/src/main/res/layout/activity_timed_task_setting.xml @@ -820,6 +820,68 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/src/main/res/layout/activity_virtual_screen_viewer.xml b/app/src/main/res/layout/activity_virtual_screen_viewer.xml new file mode 100644 index 0000000000..b3c7284e7e --- /dev/null +++ b/app/src/main/res/layout/activity_virtual_screen_viewer.xml @@ -0,0 +1,209 @@ + + + + + + + + + + + + + + + + + + + + + + + +