Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions app/src/main/AndroidManifest.xml
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,12 @@

<activity android:name="org.autojs.autojs.ui.project.ProjectConfigActivity" />

<!-- Live preview and direct control of a background virtual display. Not exported. -->
<activity
android:name="org.autojs.autojs.ui.virtualscreen.VirtualScreenViewerActivity"
android:theme="@style/AppTheme.Settings"
android:exported="false" />

<activity android:name="org.autojs.autojs.ui.keystore.ManageKeyStoreActivity" />

<activity
Expand Down
21 changes: 21 additions & 0 deletions app/src/main/aidl/org/autojs/autojs/core/shizuku/IUserService.aidl
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.autojs.autojs.core.shizuku;

import android.os.ParcelFileDescriptor;
import android.view.Surface;
import org.autojs.autojs.core.shizuku.IRemoteShellProcess;

interface IUserService {
Expand Down Expand Up @@ -32,4 +33,24 @@ interface IUserService {
// and exits 30 s after it started when no host attached (docs/shizuku-oom/ROADMAP.md D5).
oneway void attachHost(IBinder host) = 41;

// ---- Virtual screen (docs/virtual-screen/README.md) ----
// The display is owned by this shell-uid process: it dies with the process, so it must not be
// created by a one-shot execCommand. `surface` is an ImageReader surface in the app process.
int createVirtualDisplay(in Surface surface, int width, int height, int dpi, int flags) = 51;
boolean releaseVirtualDisplay(int displayId) = 52;
String listVirtualDisplays() = 53;
String virtualDisplayInfo(int displayId) = 54;
int launchOnDisplay(int displayId, String packageName, String component, boolean clearTask) = 55;
boolean tapOnDisplay(int displayId, int x, int y) = 56;
boolean swipeOnDisplay(int displayId, int x1, int y1, int x2, int y2, int durationMs) = 57;
boolean keyOnDisplay(int displayId, int keyCode) = 58;
boolean textOnDisplay(int displayId, String text) = 59;
boolean forceStopOnDisplay(int displayId, String packageName) = 60;
// ACTION_VIEW app links (e.g. example://...) must also be launched on the virtual display;
// app.openUrl() alone would resolve them on the default display.
int launchUriOnDisplay(int displayId, String uri) = 61;
// VirtualDisplay shares STREAM_MUSIC with the physical display. Apps on it are muted by
// default; the shell remembers the original media volume and restores it on the last release.
boolean setMediaMuted(boolean muted) = 62;

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/**
* 后台虚拟屏示例 [v6.8.0+]
*
* 在一块隐藏的显示屏上打开设置并截图, 主屏照常可用.
*
* 注意: 这是「显式 JS 模块」的高级示例. 常规用法无需改脚本 —— 见同目录
* 「零改动脚本示例 [v6.8.0+].js」与 docs/virtual-screen/README.md.
* 需要: Shizuku 已启动并授权 AutoJs6; 推荐 Android 14+ (虚拟屏拥有独立焦点).
* 文档: docs/virtual-screen/README.md (应用内「项目文档」亦收录)
*/

let s = virtualScreen.state();
console.log('环境:', JSON.stringify({
available: s.available,
sdkInt: s.sdkInt,
ownFocusSupported: s.ownFocusSupported,
}));

if (!s.available) {
toast('请先启动 Shizuku 并授权 AutoJs6');
exit();
}

let screen = virtualScreen.create({ width: 1080, height: 1920, dpi: 480 });
console.log('虚拟屏:', screen.state());

try {
// 0 = 成功, 1 = 无启动入口, 2 = 启动失败, 3 = displayId 无效
let code = screen.launch('com.android.settings');
if (code !== 0) {
throw new Error('启动失败, 返回码 ' + code);
}
sleep(1500);

let img = screen.capture(); // ImageWrapper, 可用于 images.* 全套能力
img.saveTo('/sdcard/Download/virtual-screen.png');
console.log('帧时间戳(ns):', screen.getFrameTime());

let p = images.findColor(img, '#ffffff', { threshold: 8 });
if (p) {
screen.tap(p.x, p.y); // 坐标是虚拟屏的逻辑像素
}
screen.swipe(540, 1500, 540, 500, 600);
screen.key(4); // 返回键
img.recycle();

toastLog('完成, 主屏未受干扰');
} finally {
// 显式释放; 即使省略, 脚本退出时运行时也会自动释放.
screen.release();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/**
* 零改动脚本示例 [v6.8.0+]
*
* 本文件没有任何虚拟屏代码. 想让整个脚本运行在后台虚拟屏上, 只需:
* 文件菜单 →「在虚拟屏幕运行」, 或 设置 → 脚本运行 → 打开「在虚拟屏幕运行脚本」.
* 之后 images.captureScreen / click / swipe / selector / app.launchPackage 会自动作用于虚拟屏,
* 主屏保持可用, 脚本一行都不用改.
*
* 需要: Shizuku 已启动并授权 AutoJs6; Android 12+ (推荐 14+).
* 文档: docs/virtual-screen/README.md
*/

'auto';

auto.waitFor();
app.launchPackage('com.android.settings');
sleep(1500);

// 虚拟屏模式下直接返回已授权, 不弹 MediaProjection 对话框.
if (!images.requestScreenCapture(false)) {
toast('无法获取截图权限');
exit();
}

let img = images.captureScreen();
if (img === null) {
toast('截图失败');
exit();
}

let shot = '/sdcard/Download/virtual-screen-capture.png';
img.saveTo(shot);
console.log('截图已保存:', shot);

let point = images.findColor(img, '#ffffff', { threshold: 8 });
if (point !== null) {
click(point.x, point.y);
sleep(400);
}
img.recycle();

swipe(540, 1500, 540, 500, 600);
let node = selector().clickable(true).findOne(1000);
console.log('控件:', node === null ? '未找到' : node.text());

back();
console.log('完成: 这段代码里没有任何 virtualScreen.* 调用');
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
package org.autojs.autojs.core.accessibility

import android.content.Context
import android.os.SystemClock
import android.util.Log
import android.view.accessibility.AccessibilityNodeInfo
import android.view.accessibility.AccessibilityWindowInfo
import org.autojs.autojs.annotation.ScriptInterface
import org.autojs.autojs.core.accessibility.event.A11yEventPulse
import org.autojs.autojs.core.accessibility.window.A11yWindowFacts
import org.autojs.autojs.core.accessibility.window.A11yWindowRootPolicy
import org.autojs.autojs.core.accessibility.window.DisplayWindowScope
import org.autojs.autojs.core.accessibility.window.WindowRootPolicy
import org.autojs.autojs.core.accessibility.window.WindowRootProvider
import org.autojs.autojs.core.activity.ActivityInfoProvider
Expand All @@ -26,7 +30,10 @@ import org.autojs.autojs6.R
* - [windowRoots] and [getRootInCurrentWindow] honour the window filter; without one they resolve
* to the active window root of the current mode.
* - [getRootInActiveWindow] ignores the window filter and always resolves to the active window root
* of the current mode.
* of the current mode, except while a display scope is bound (see below).
* - A display scope bound by [setDisplayScope] (a task configured to run on a virtual screen)
* switches the window source to all displays and pins every query to that display. It cannot be
* removed by a script's `auto.setWindowFilter`; the script filter is applied on top of it.
* - Neither is affected by the guard mode blacklist ([AccessibilityConfig]); that is applied by
* `UiSelector.findImpl` (roots of blacklisted packages are skipped) and by
* `SimpleActionAutomator.performAction` (no action while AutoJs6 itself is in front).
Expand All @@ -35,7 +42,9 @@ import org.autojs.autojs6.R
* 类名与 public 签名与 Java 版一致 (脚本可能经 `Packages` 引用); 根节点逻辑本身在 [WindowRootProvider] / [WindowRootPolicy].
* 三个根节点查询各自遵循的设置不同, 且是有意为之 (作为文档化行为保留):
* [windowRoots] 与 [getRootInCurrentWindow] 遵循窗口过滤器, 无过滤器时解析为当前模式下的活动窗口根节点;
* [getRootInActiveWindow] 忽略窗口过滤器, 总是解析为当前模式下的活动窗口根节点;
* [getRootInActiveWindow] 忽略窗口过滤器, 总是解析为当前模式下的活动窗口根节点 (绑定 display 作用域时除外);
* [setDisplayScope] 绑定的 display 作用域 (被配置为在虚拟屏运行的任务) 会把窗口来源切换到所有屏幕, 并把每次查询
* 钉在该屏幕上; 脚本的 `auto.setWindowFilter` 无法解除它, 只能叠加在其之上.
* 两者都不受保护模式黑名单 ([AccessibilityConfig]) 影响, 黑名单由 `UiSelector.findImpl` (跳过黑名单包名的根节点)
* 与 `SimpleActionAutomator.performAction` (AutoJs6 自身前置时不执行动作) 应用.
*
Expand Down Expand Up @@ -118,10 +127,24 @@ abstract class AccessibilityBridge(
// zh-CN: 每次查询时查找服务, 因此 bridge 在服务重启后依然可用.
private val rootProvider = WindowRootProvider { service }

// The default-display-only window list cannot contain a virtual display's windows. A scoped
// runtime uses this provider instead, whose windows come from getWindowsOnAllDisplays().
// zh-CN: 只含默认屏的窗口列表不可能包含虚拟屏窗口; 绑定了作用域的运行时改用这个 provider.
private val allDisplayRootProvider = WindowRootProvider(allDisplays = true) { service }

/** The display this task's accessibility queries are pinned to. zh-CN: 本任务无障碍查询被钉住的显示屏. */
private val displayScope = DisplayWindowScope<AccessibilityWindowInfo> { window -> A11yWindowFacts.displayId(window) }

private fun activeProvider(): WindowRootProvider =
if (displayScope.isScoped) allDisplayRootProvider else rootProvider

@Volatile
private var lastEmptyScopeDiagnosticMillis = 0L

/**
* The current query settings as one immutable policy; rebuilt by [setMode], [setWindowFilter]
* and [flags].
* zh-CN: 当前查询设置构成的单个不可变策略; 由 [setMode], [setWindowFilter] 与 [flags] 重建.
* The current query settings as one immutable policy; rebuilt by [setMode], [setWindowFilter],
* [setDisplayScope] and [flags].
* zh-CN: 当前查询设置构成的单个不可变策略; 由 [setMode], [setWindowFilter], [setDisplayScope] 与 [flags] 重建.
*/
@Volatile
var windowRootPolicy: A11yWindowRootPolicy = WindowRootPolicy()
Expand Down Expand Up @@ -172,30 +195,106 @@ abstract class AccessibilityBridge(
* zh-CN: 查询所针对的根节点, 不含 `null`: 通过窗口过滤器的窗口的根节点; 否则当前模式下的活动窗口根节点
* (系统不提供时, 由 [FLAG_APP_WINDOWS_FALLBACK] 决定的代替根节点); 否则为空.
*/
fun windowRoots(): List<AccessibilityNodeInfo> = rootProvider.roots(windowRootPolicy)
fun windowRoots(): List<AccessibilityNodeInfo> {
val roots = activeProvider().roots(windowRootPolicy)
// When a virtual-screen task finds nothing, make the distinction visible: is the system
// reporting windows for that display at all, or did the selector simply not match?
// zh-CN: 虚拟屏任务查不到节点时, 让日志能区分「系统根本没上报该屏窗口」与「选择器条件不匹配」.
if (roots.isEmpty() && displayScope.isScoped) logEmptyScopeDiagnostic()
return roots
}

/**
* The service's windows, unfiltered, in the framework's order; empty without a service.
* zh-CN: 服务的全部窗口, 未过滤, 按框架的顺序; 无服务时为空.
* The service's windows in the framework's order, unfiltered by the script's window filter;
* while a display scope is bound, only the windows of that display are returned.
* zh-CN: 按框架顺序排列的服务窗口, 不受脚本窗口过滤器影响; 绑定 display 作用域时只返回该屏的窗口.
*/
fun windows(): List<AccessibilityWindowInfo> = rootProvider.windows()
fun windows(): List<AccessibilityWindowInfo> {
val windows = activeProvider().windows()
val scope = displayScope.scopedDisplayId
if (scope == DisplayWindowScope.NO_SCOPE) return windows
return windows.filter { window ->
runCatching { A11yWindowFacts.displayId(window) == scope }.getOrDefault(false)
}
}

/**
* The first of [windowRoots], resolved lazily (with a window filter, only the windows up to the
* first one that has a root are queried).
* zh-CN: [windowRoots] 的首个根节点, 惰性解析 (有窗口过滤器时只查询到首个有根节点的窗口为止).
*/
fun getRootInCurrentWindow(): AccessibilityNodeInfo? = rootProvider.currentRoot(windowRootPolicy)
fun getRootInCurrentWindow(): AccessibilityNodeInfo? = activeProvider().currentRoot(windowRootPolicy)

/**
* The active window root of the current mode, ignoring the window filter (unlike
* [getRootInCurrentWindow]); [FLAG_APP_WINDOWS_FALLBACK] applies.
* [getRootInCurrentWindow]); [FLAG_APP_WINDOWS_FALLBACK] applies. While a display scope is
* bound, the framework's active root is the *main* screen's, so the top root of the bound
* display is returned instead.
* zh-CN: 当前模式下的活动窗口根节点, 忽略窗口过滤器 (与 [getRootInCurrentWindow] 不同); [FLAG_APP_WINDOWS_FALLBACK] 生效.
* 绑定 display 作用域时, 框架的活动窗口根节点属于**主屏**, 因此改返回绑定屏最上层窗口的根节点.
*/
fun getRootInActiveWindow(): AccessibilityNodeInfo? = rootProvider.activeRoot(windowRootPolicy)
fun getRootInActiveWindow(): AccessibilityNodeInfo? =
// `rootInActiveWindow` is the *default display's* active window and cannot be scoped.
// While a task is bound to a virtual display, "the active window" must be the top window
// of that display, i.e. the first root passing the scoped filter.
// zh-CN: `rootInActiveWindow` 是**默认屏**的活动窗口, 无法限定 display; 任务绑定虚拟屏时,
// 「活动窗口」应是该屏的最上层窗口, 即通过作用域过滤器的首个根节点.
if (displayScope.isScoped) getRootInCurrentWindow()
else activeProvider().activeRoot(windowRootPolicy)

/**
* Installs/replaces the script's own window filter. When a display scope is bound, the filter
* is combined with (not substituted for) that scope, so a zero-change script cannot read the
* physical screen by accident. zh-CN: 安装/替换脚本自己的窗口过滤器; 已绑定 display 作用域时是叠加而非替换,
* 避免零改动脚本意外读到物理主屏.
*/
fun setWindowFilter(windowFilter: WindowFilter?) {
windowRootPolicy = windowRootPolicy.copy(filter = windowFilter?.let { { info -> it.filter(info) } })
displayScope.setScriptFilter(windowFilter?.let { filter -> { info -> filter.filter(info) } })
rebuildWindowRootFilter()
}

/**
* Binds every accessibility query of this runtime to one display and switches to the
* all-displays window source. Called from `ScriptRuntime.initPrologue()` before modules are
* mounted, so existing JavaScript needs no changes. Pass [DisplayWindowScope.NO_SCOPE] to clear.
*
* zh-CN: 把本运行时的所有无障碍查询绑定到某个显示屏, 并切换到全屏窗口来源; 由
* `ScriptRuntime.initPrologue()` 在模块挂载前调用, 已有 JS 无需修改. 传 [DisplayWindowScope.NO_SCOPE] 可清除.
*/
fun setDisplayScope(id: Int) {
displayScope.setDisplayScope(id)
rebuildWindowRootFilter()
if (id == DisplayWindowScope.NO_SCOPE) {
Log.i(TAG, "Accessibility display scope cleared")
return
}
Log.i(TAG, "Accessibility display scope = $id; windows by display: ${windowsByDisplaySummary()}")
}

private fun rebuildWindowRootFilter() {
windowRootPolicy = windowRootPolicy.copy(filter = displayScope.effectiveFilter())
}

private fun windowsByDisplaySummary(): String {
if (service == null) return "service not bound"
return runCatching {
allDisplayRootProvider.windows()
.groupingBy { window -> runCatching { A11yWindowFacts.displayId(window) }.getOrDefault(-1) }
.eachCount()
.toString()
}.getOrDefault("unavailable")
}

/** Throttled: selector polling asks for roots many times a second. zh-CN: 节流: 选择器轮询每秒会多次查询根节点. */
private fun logEmptyScopeDiagnostic() {
val now = SystemClock.elapsedRealtime()
if (now - lastEmptyScopeDiagnosticMillis < 5_000L) return
lastEmptyScopeDiagnosticMillis = now
Log.w(
TAG,
"No accessibility roots on display ${displayScope.scopedDisplayId}; " +
"windows by display: ${windowsByDisplaySummary()}",
)
}

abstract val infoProvider: ActivityInfoProvider
Expand All @@ -218,6 +317,8 @@ abstract class AccessibilityBridge(

companion object {

private const val TAG = "AccessibilityBridge"

const val MODE_NORMAL = 0
const val MODE_FAST = 1

Expand Down
Loading