From b071e925c1f969cf144d85ebaf2d9333d9f548d3 Mon Sep 17 00:00:00 2001 From: anothaDev <10740588+anothaDev@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:33:31 +0200 Subject: [PATCH] Take the device recorder out of the repository dev/mirror/ held the browser and SSH recorders that read a real device. They are a private tool for the maintainer's own network and have no part in CI, the packages or the contribution workflow, so they leave the tree and the directory is ignored. The request policy they shared with the replay (allowlist, body parsers, secret scrubbing) moves to dev/replay/policy.js, where the replay already needed it; its test follows as tests/policy.test.js. The recorder's end-to-end test leaves with it. Docs no longer describe how recordings are made. The replay's README gains a Recording format section (browser-*/rpc.jsonl, http.jsonl, static/, ssh-*/snapshots.jsonl) so anyone can produce or inspect one; README, CONTRIBUTING, AGENTS, SECURITY and SPEC point there. No change to the packages. --- .gitignore | 4 + AGENTS.md | 8 +- CONTRIBUTING.md | 13 +- README.md | 15 +- SECURITY.md | 6 +- dev/mirror/record-browser.js | 383 ------------------ dev/mirror/record-ssh.js | 82 ---- dev/replay/README.md | 77 ++-- dev/{mirror => replay}/policy.js | 3 +- dev/replay/server.js | 8 +- dev/replay/synthetic.js | 2 +- docs/SPEC.md | 11 +- ...recorder-policy.test.js => policy.test.js} | 8 +- tests/recorder-browser.test.js | 266 ------------ 14 files changed, 73 insertions(+), 813 deletions(-) delete mode 100644 dev/mirror/record-browser.js delete mode 100644 dev/mirror/record-ssh.js rename dev/{mirror => replay}/policy.js (99%) rename tests/{recorder-policy.test.js => policy.test.js} (98%) delete mode 100644 tests/recorder-browser.test.js diff --git a/.gitignore b/.gitignore index 2c29987..8d7ab7c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,10 @@ browser-[0-9][0-9][0-9][0-9]-*/ ssh-[0-9][0-9][0-9][0-9]-*/ blocked.log +# the device recorder is a private tool kept next to the recordings, not +# part of the repository +/dev/mirror/ + # local SDK build output (packages, SHA256SUMS, BUILDINFO, dist/.tools/) /dist/ # a build work directory, if TMPDIR points into the tree diff --git a/AGENTS.md b/AGENTS.md index 4215bc0..4f1d16a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,6 @@ Two OpenWrt 25.12 LuCI packages plus offline tooling. No npm, no | `luci-theme-vantage/` | Theme: ucode templates (`ucode/template/themes/vantage/*.ut`), CSS and icons (`htdocs/luci-static/vantage/`), shell script (`htdocs/luci-static/resources/menu-vantage.js`, `vantage-theme/host.js`), `root/etc/uci-defaults/30_luci-theme-vantage`, `postrm` in the `Makefile` | | `luci-app-vantage/` | Dashboard at `admin/dashboard`: views (`htdocs/.../view/vantage/overview.js`; `firewall.js`, the Network → Firewall → Overview tab), modules (`htdocs/.../vantage/*.js`), `app.css`, menu entry, rpcd ACL (`root/usr/share/rpcd/acl.d/`), rpcd ucode plugin `luci.vantage` (`root/usr/share/rpcd/ucode/luci.vantage`), default `/etc/config/vantage` | | `dev/replay/` | Offline LuCI server that replays a recording; see its README | -| `dev/mirror/` | Recorders that read a real device. Do not run them (rule 1) | | `dev/build/` | `sdk-build.sh`: reproducible build in the OpenWrt SDK image (podman) | | `dev/icons/build.js` | Generates the status icon set; edit the generator, not the SVGs | | `dev/i18n/` | `update.sh`: translation templates and catalogues (`po/`) with LuCI's i18n tools; `catalog.js`: LuCI's catalogue format (tests, replay `--lang`) | @@ -25,8 +24,8 @@ Two OpenWrt 25.12 LuCI packages plus offline tooling. No npm, no ## Hard rules -1. **No real devices.** Do not SSH to, copy to, browse, or run - `dev/mirror/` against a router or access point unless the human asks for +1. **No real devices.** Do not SSH to, copy to, browse, or run any + recorder against a router or access point unless the human asks for exactly that in the current conversation. All UI work goes through the replay. 2. **No real network data in the tree.** Recordings live outside the @@ -83,7 +82,8 @@ Needs Node.js (the maintainer uses a current release; there are no dependencies to install). The replay needs two local inputs that are not in the repository: a -recording (`--mirror`, made with `dev/mirror/`) and a LuCI 25.12 root +recording (`--mirror`, in the format described in `dev/replay/README.md` +under Recording format) and a LuCI 25.12 root filesystem for the core templates and static files (`--rootfs` or `$VANTAGE_ROOTFS`). If either is missing, say so and stick to the tests; do not fake them. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d036331..7dc7e7b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,13 +30,12 @@ The UI is developed offline against recorded device data, never against a live router. You need Node.js (there are no dependencies to install) and, to build packages, podman. -1. **Record** a device once with `dev/mirror/` (read-only: it refuses - writes, and strips secrets before writing anything). Keep the - recording outside this repository, e.g. in `../vantage-mirror`. - The browser recorder pins the device's TLS certificate: copy it over - SSH (`scp -O root@:/etc/uhttpd.crt /tmp/device.crt`) and pass - `--cert /tmp/device.crt` (or `--spki `; see the header of - `dev/mirror/record-browser.js`). +1. **Device data.** The replay reads a recording of a LuCI session in + the format described under Recording format in + [`dev/replay/README.md`](dev/replay/README.md#recording-format), or + hand-written fixtures (`--fixtures`, `dev/replay/addons.sh`). Keep a + recording outside this repository, e.g. in `../vantage-mirror`, and + never commit one: it holds your network's addresses and names. 2. **Replay** it with the theme and app from your working tree, plus a LuCI 25.12 root filesystem for LuCI's own templates and static files (`--rootfs`, see [`dev/replay/README.md`](dev/replay/README.md)): diff --git a/README.md b/README.md index ab7f0f1..0a24298 100644 --- a/README.md +++ b/README.md @@ -159,14 +159,13 @@ inputs. Optional index signing and the details are in the The UI is developed offline against recorded device data, never against a live device: -1. **Record** – `dev/mirror/` records a real LuCI session read-only - (browser and SSH snapshots). Writes, applies, scans and uploads are - refused before they leave the browser, and secrets are removed before - anything is written. Recordings stay outside the repository. -2. **Replay** – `dev/replay/` serves the complete LuCI UI from a recording on - `127.0.0.1`, with the theme and app loaded from this tree; reload to see - an edit. Save & Apply works against an in-memory overlay. -3. **Demo mode** – `--demo` pseudonymises the recording as it loads: +1. **Replay** – `dev/replay/` serves the complete LuCI UI on `127.0.0.1` + from a recording of a real LuCI session (ubus replies, HTTP responses + and static files; the format is in the replay's README) or from + hand-written fixtures, with the theme and app loaded from this tree; + reload to see an edit. Save & Apply works against an in-memory overlay. + Recordings stay outside the repository. +2. **Demo mode** – `--demo` pseudonymises the recording as it loads: documentation MACs and addresses (`00:00:5E:00:53:xx`, `192.0.2.0/24`, `2001:db8::/32`), neutral hostname, SSIDs and client names. The screenshots above were taken this way. diff --git a/SECURITY.md b/SECURITY.md index 579b845..57f419e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -42,9 +42,9 @@ In scope: - the dashboard's rpcd ACL: permissions broader than documented, or writes outside `/etc/config/vantage` - packaging: the install and removal scripts, file permissions -- the dev tooling that handles recorded device data (`dev/mirror/`, - `dev/replay/`), for example secrets or real identifiers leaking into - recordings, demo output or the repository +- the dev tooling that handles recorded device data (`dev/replay/`), + for example real identifiers leaking into demo output or the + repository Out of scope: diff --git a/dev/mirror/record-browser.js b/dev/mirror/record-browser.js deleted file mode 100644 index 5b26ab4..0000000 --- a/dev/mirror/record-browser.js +++ /dev/null @@ -1,383 +0,0 @@ -#!/usr/bin/env node -'use strict'; -/* Record a real LuCI session read-only. - - Opens a visible Chromium window with a throwaway profile on the device's - LuCI. You log in and click through pages. The recorder holds a - browser-level DevTools session (over a pipe, no TCP port) with Fetch - interception for every URL, so every request of every tab, popup, worker - and service worker in that browser is paused and decided by policy.js - before it leaves the browser: - - JSON-RPC calls on the read allowlist are forwarded (re-serialised - exactly as checked) and recorded, - - cgi-exec of the few read-only commands LuCI's status pages run is - forwarded (canonical body) and recorded, - - the login form POST (exactly the two login fields) is forwarded but - never recorded, - - GETs of LuCI pages and static files are forwarded, - - anything else (writes, applies, reboots, scans, uploads, backups, - legacy form posts, other hosts or other spellings of the device) is - refused. - Responses are sanitized (policy.js) before being written. If the - recorder dies, the browser is killed with it and the profile removed; - while the recorder is not answering, paused requests stay paused. - - This is an accident guard for clicking through stock LuCI, not a sandbox - for hostile page JavaScript (see policy.js). - - usage: node record-browser.js --spki - node record-browser.js --cert - - The device's LuCI certificate is self-signed. Instead of ignoring - certificate errors, the browser accepts only the device's own key: pass - its SPKI pin (base64 SHA-256 of the DER public key) with --spki (or - $VANTAGE_DEVICE_SPKI), or a copy of the certificate with --cert. Get - either over SSH, whose host key you have already verified, never from - the TLS connection itself: - - scp -O root@:/etc/uhttpd.crt /tmp/device.crt - node record-browser.js --cert /tmp/device.crt https:// ../vantage-mirror - - or compute the pin (uhttpd's generated certificate is DER; for a PEM one - drop "-inform der"): - - ssh root@ cat /etc/uhttpd.crt | openssl x509 -inform der -pubkey -noout \ - | openssl pkey -pubin -outform der | openssl dgst -sha256 -binary | openssl base64 - - Record over the wired management path. http:// is accepted only for a - loopback test server (127.0.0.1, localhost, [::1]). - - Close the browser window (or Ctrl+C) to finish. */ - -const { spawn } = require('child_process'); -const crypto = require('crypto'); -const fs = require('fs'); -const os = require('os'); -const path = require('path'); -const policy = require('./policy'); - -const USAGE = 'usage: record-browser.js (--spki | --cert ) '; - -function die(msg) { - console.error(msg + '\n' + USAGE); - process.exit(2); -} - -function spkiOfCert(file) { - const cert = new crypto.X509Certificate(fs.readFileSync(file)); - return crypto.createHash('sha256').update(cert.publicKey.export({ type: 'spki', format: 'der' })).digest('base64'); -} - -/* ------------------------------------------------------------ arguments */ - -const pos = []; -let pins = []; -{ - const argv = process.argv.slice(2); - for (let i = 0; i < argv.length; i++) { - const m = /^--(spki|cert)(?:=(.*))?$/.exec(argv[i]); - if (!m) { if (argv[i].startsWith('--')) die('unknown option ' + argv[i]); pos.push(argv[i]); continue; } - const v = m[2] !== undefined ? m[2] : argv[++i]; - if (v == null) die('--' + m[1] + ' needs a value'); - if (m[1] === 'spki') pins.push(...v.split(',')); - else { - try { pins.push(spkiOfCert(v)); } catch (e) { die('--cert ' + v + ': ' + e.message); } - } - } - if (!pins.length && process.env.VANTAGE_DEVICE_SPKI) pins = process.env.VANTAGE_DEVICE_SPKI.split(','); - pins = pins.map(p => p.trim()).filter(Boolean); -} -const base = (pos[0] || '').replace(/\/+$/, ''); -const outRoot = pos[1]; -if (pos.length !== 2 || !outRoot) die('need a device URL and a mirror directory'); -const LOOPBACK = /^http:\/\/(127\.0\.0\.1|localhost|\[::1\])(:\d{1,5})?$/; -if (!/^https:\/\/[^/?#@\s]+$/.test(base) && !LOOPBACK.test(base)) - die('the device URL must be https://[:port] (http:// only for a loopback test server)'); -if (base.startsWith('https:')) { - if (!pins.length) die('missing certificate pin: pass --spki or --cert (see the header of this file for getting it over SSH)'); - for (const p of pins) if (!/^[A-Za-z0-9+/]{43}=$/.test(p) || Buffer.from(p, 'base64').length !== 32) die('bad --spki pin (want base64 of a SHA-256 digest): ' + p); -} -const origin = new URL(base).origin; - -/* ------------------------------------------------------------ output */ - -const stamp = new Date().toISOString().replace(/[:.]/g, '-'); -const out = path.join(outRoot, 'browser-' + stamp); -const staticRoot = path.join(out, 'static'); -fs.mkdirSync(staticRoot, { recursive: true, mode: 0o700 }); -const PROFILE_PREFIX = path.join(os.tmpdir(), 'vantage-rec-profile-'); -const profile = fs.mkdtempSync(PROFILE_PREFIX); -/* no saved passwords in the throwaway profile */ -fs.mkdirSync(path.join(profile, 'Default'), { mode: 0o700 }); -fs.writeFileSync(path.join(profile, 'Default', 'Preferences'), - JSON.stringify({ credentials_enable_service: false, profile: { password_manager_enabled: false } }), { mode: 0o600 }); -const rpcLog = fs.openSync(path.join(out, 'rpc.jsonl'), 'a', 0o600); -const httpLog = fs.openSync(path.join(out, 'http.jsonl'), 'a', 0o600); -const blockLog = fs.openSync(path.join(out, 'blocked.log'), 'a', 0o600); -const stats = { rpc: 0, http: 0, static: 0, blocked: 0, pages: 0, workers: 0 }; - -function write(fd, obj) { fs.writeSync(fd, JSON.stringify(obj) + '\n'); } -/* request-derived text on the terminal: no control characters */ -const safe = s => String(s).replace(/[\x00-\x1f\x7f-\x9f]/g, c => '\\x' + c.charCodeAt(0).toString(16).padStart(2, '0')); - -/* ------------------------------------------------------------ cleanup */ - -let chrome = null, cleaned = false, finished = false, tick = null; - -/* Is any live (non-zombie) process left in the browser's process group? - Chromium's helpers keep writing into the profile for a moment after the - browser process dies, so the profile is removed only once they are gone. */ -function groupAlive(pgid) { - let procs; - try { procs = fs.readdirSync('/proc'); } catch (e) { try { process.kill(-pgid, 0); return true; } catch (e2) { return false; } } - for (const p of procs) { - if (!/^\d+$/.test(p)) continue; - let st; - try { st = fs.readFileSync(`/proc/${p}/stat`, 'utf8'); } catch (e) { continue; } - const f = st.slice(st.lastIndexOf(')') + 2).split(' '); - if (+f[2] === pgid && f[0] !== 'Z') return true; - } - return false; -} - -const pause = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); - -function removeProfile() { - /* the mkdtemp path above, nothing else */ - if (!profile.startsWith(PROFILE_PREFIX) || path.dirname(profile) !== os.tmpdir()) return; - /* Chromium's singleton socket directory in $TMPDIR, linked from the profile */ - let single = null; - try { - const dir = path.dirname(fs.readlinkSync(path.join(profile, 'SingletonSocket'))); - if (path.dirname(dir) === os.tmpdir() && /^org\.chromium\.Chromium\.[A-Za-z0-9]{6}$/.test(path.basename(dir))) single = dir; - } catch (e) {} - for (const dir of [ profile, single ].filter(Boolean)) { - try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } - catch (e) { console.error('could not remove ' + dir + ': ' + e.message); } - } -} - -/* idempotent and synchronous, so it also works from process 'exit' */ -function cleanup() { - if (cleaned) return; - cleaned = true; - if (tick) clearInterval(tick); - if (chrome && chrome.pid) { - try { process.kill(-chrome.pid, 'SIGKILL'); } catch (e) { try { chrome.kill('SIGKILL'); } catch (e2) {} } - for (let i = 0; i < 100 && groupAlive(chrome.pid); i++) pause(50); - } - removeProfile(); - /* a helper that was still exiting may have recreated a directory */ - pause(200); - if (fs.existsSync(profile)) removeProfile(); -} - -function finish(reason, code) { - if (!finished) { - finished = true; - try { - fs.writeFileSync(path.join(out, 'summary.json'), JSON.stringify({ base, stats, end: reason, finished: new Date().toISOString() }, null, 2), { mode: 0o600 }); - } catch (e) {} - console.log('done (' + reason + ')', JSON.stringify(stats)); - } - cleanup(); - process.exit(code); -} - -process.on('exit', cleanup); -for (const [ sig, code ] of [ [ 'SIGINT', 130 ], [ 'SIGTERM', 143 ], [ 'SIGHUP', 129 ] ]) - process.on(sig, () => finish(sig, code)); -process.on('uncaughtException', e => { console.error('recorder error:', e); finish('error', 1); }); -process.on('unhandledRejection', e => { console.error('recorder error:', e); finish('error', 1); }); - -/* ------------------------------------------------------------ browser */ - -const flags = [ - '--user-data-dir=' + profile, '--remote-debugging-pipe', '--no-first-run', '--no-default-browser-check', - '--disable-background-networking', '--disable-component-update', '--disable-sync', '--disable-default-apps', - '--disable-extensions', '--password-store=basic' -]; -if (pins.length) flags.push('--ignore-certificate-errors-spki-list=' + pins.join(',')); -flags.push('about:blank'); - -/* own process group, so the whole browser can be killed at once; it also - exits by itself when the DevTools pipe closes (the recorder died) */ -chrome = spawn('chromium', flags, { stdio: [ 'ignore', 'ignore', 'ignore', 'pipe', 'pipe' ], detached: true }); -chrome.on('error', e => { console.error('cannot start chromium: ' + e.message); finish('error', 1); }); -chrome.on('exit', () => finish('browser closed', 0)); - -/* CDP over --remote-debugging-pipe: fd 3 to the browser, fd 4 from it, - NUL-terminated JSON messages */ -const toChrome = chrome.stdio[3], fromChrome = chrome.stdio[4]; -let nextId = 0; -const pend = new Map(); -function send(method, params, sessionId) { - return new Promise((resolve, reject) => { - const id = ++nextId; - pend.set(id, { resolve, reject, method }); - const msg = { id, method, params: params || {} }; - if (sessionId) msg.sessionId = sessionId; - toChrome.write(JSON.stringify(msg) + '\0'); - }); -} -toChrome.on('error', () => {}); -let inbuf = Buffer.alloc(0); -fromChrome.on('data', chunk => { - inbuf = Buffer.concat([ inbuf, chunk ]); - let z; - while ((z = inbuf.indexOf(0)) >= 0) { - const text = inbuf.subarray(0, z).toString('utf8'); - inbuf = inbuf.subarray(z + 1); - let m; - try { m = JSON.parse(text); } catch (e) { continue; } - if (m.id && pend.has(m.id)) { - const p = pend.get(m.id); pend.delete(m.id); - if (m.error) p.reject(new Error(p.method + ': ' + m.error.message)); else p.resolve(m.result); - continue; - } - onEvent(m).catch(failClosed); - } -}); -fromChrome.on('error', () => {}); -fromChrome.on('end', () => finish('browser pipe closed', 0)); - -/* A paused request the browser has meanwhile cancelled (navigation, tab - closed) makes Fetch.* answer with an error. That is harmless: nothing was - forwarded. Any other failure of ours stops the browser. */ -function settle(promise) { - return promise.catch(e => { if (!/Invalid InterceptionId|Invalid state|not found/i.test(e.message)) console.error('devtools:', safe(e.message)); }); -} - -/* a bug here must not leave an unfiltered browser behind */ -function failClosed(e) { - console.error('recorder error, closing the browser:', e); - finish('error', 1); -} - -/* ------------------------------------------------------------ recording */ - -let ready = false; -const targets = new Map(); /* targetId -> { type, url } */ -const tracked = new Map(); /* Fetch requestId -> { kind, url, calls, page } */ - -function pageOf(frameId) { - const t = targets.get(frameId); - if (!t || !t.url.startsWith(origin)) return ''; - return t.url.slice(origin.length).replace(/[?#].*$/, ''); -} - -function block(p, why) { - stats.blocked++; - fs.writeSync(blockLog, new Date().toISOString() + ' ' + safe(why) + '\n'); - console.log('BLOCKED', safe(why)); - return settle(send('Fetch.failRequest', { requestId: p.requestId, errorReason: 'BlockedByClient' })); -} - -async function onEvent(m) { - switch (m.method) { - case 'Target.targetCreated': - case 'Target.targetInfoChanged': { - const t = m.params.targetInfo; - if (!targets.has(t.targetId) && m.method === 'Target.targetCreated') { - if (t.type === 'page') stats.pages++; - else if (/worker/.test(t.type)) stats.workers++; - if (ready && t.type === 'page') console.log('new tab or popup (filtered like the first)'); - } - const was = targets.get(t.targetId); - targets.set(t.targetId, { type: t.type, url: t.url }); - if (t.type === 'page' && t.url.startsWith(origin) && (!was || was.url !== t.url)) console.log('page', safe(t.url.slice(origin.length))); - return; - } - case 'Target.targetDestroyed': - targets.delete(m.params.targetId); - return; - case 'Fetch.requestPaused': - return onPaused(m.params); - } -} - -async function onPaused(p) { - /* response stage: record what an allowed request returned */ - if (p.responseStatusCode !== undefined || p.responseErrorReason !== undefined) { - const t = tracked.get(p.requestId); - tracked.delete(p.requestId); - if (t && p.responseStatusCode !== undefined && !(p.responseStatusCode >= 300 && p.responseStatusCode < 400)) { - let r = null; - try { r = await send('Fetch.getResponseBody', { requestId: p.requestId }); } catch (e) {} - if (r) record(t, r.base64Encoded ? Buffer.from(r.body, 'base64') : Buffer.from(r.body, 'utf8')); - } - return settle(send('Fetch.continueRequest', { requestId: p.requestId })); - } - - /* request stage: decide */ - const req = p.request; - let body = req.postData; - if (body === undefined && Array.isArray(req.postDataEntries)) - body = Buffer.concat(req.postDataEntries.map(e => Buffer.from(e.bytes || '', 'base64'))).toString('utf8'); - if (body === undefined && req.hasPostData) return block(p, `${req.method} ${req.url}: request body not available`); - const v = policy.classify({ url: req.url, method: req.method, headers: req.headers, postData: body }, base); - if (!v.ok) return block(p, v.why); - if (v.kind && v.kind !== 'login') - tracked.set(p.requestId, { kind: v.kind, url: new URL(req.url).pathname, calls: v.calls, page: pageOf(p.frameId) }); - const cont = { requestId: p.requestId }; - if (v.postData !== undefined) cont.postData = Buffer.from(v.postData, 'utf8').toString('base64'); - return settle(send('Fetch.continueRequest', cont)); -} - -function record(t, buf) { - const at = Date.now(); - if (t.kind === 'rpc') { - let replies; - try { replies = JSON.parse(buf.toString('utf8')); } catch (e) { return; } - if (!Array.isArray(replies)) replies = [ replies ]; - const byId = new Map(replies.map(x => [ x && x.id, x ])); - for (const c of t.calls) { - if (c.list) continue; - const reply = byId.get(c.id) || {}; - write(rpcLog, { at, page: t.page, object: c.object, method: c.method, - args: policy.sanitize(c.args), - result: reply.result !== undefined ? policy.sanitizeResult(c.object, c.method, c.args, reply.result) : undefined, - error: reply.error }); - stats.rpc++; - } - } - else if (t.kind === 'exec') { - write(httpLog, { at, page: t.page, url: '/cgi-bin/cgi-exec', argv: t.calls, body: policy.sanitize(buf.toString('utf8')) }); - stats.http++; - } - else if (t.kind === 'http') { - let body = buf.toString('utf8'); - try { body = policy.sanitize(JSON.parse(body)); } catch (e) { body = policy.sanitize(body); } - write(httpLog, { at, page: t.page, url: t.url, body }); - stats.http++; - } - else if (t.kind === 'static') { - let rel; - try { rel = decodeURIComponent(t.url).replace(/^\/+/, ''); } catch (e) { return; } - if (!rel || /(^|\/)\.\.?(\/|$)|[\0\\]/.test(rel) || rel.endsWith('/')) return; - const file = path.join(staticRoot, rel); - if (!file.startsWith(staticRoot + path.sep)) return; - try { - fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); - fs.writeFileSync(file, buf, { mode: 0o600 }); - stats.static++; - } - catch (e) { console.error('static not saved:', safe(rel), e.code || e.message); } - } -} - -(async () => { - await send('Target.setDiscoverTargets', { discover: true }); - /* browser-wide: every target of this browser, including ones created later */ - await send('Fetch.enable', { patterns: [ - { urlPattern: '*', requestStage: 'Request' }, - { urlPattern: origin + '/*', requestStage: 'Response' } - ] }); - /* the device page is only opened once interception is on */ - const blank = (await send('Target.getTargets')).targetInfos.filter(t => t.type === 'page'); - await send('Target.createTarget', { url: base + '/cgi-bin/luci/' }); - for (const t of blank) await send('Target.closeTarget', { targetId: t.targetId }).catch(() => {}); - ready = true; - console.log('recording to', out, '- log in and browse; close the window when done'); - tick = setInterval(() => console.log('stats', JSON.stringify(stats)), 30000); -})().catch(failClosed); diff --git a/dev/mirror/record-ssh.js b/dev/mirror/record-ssh.js deleted file mode 100644 index eccc501..0000000 --- a/dev/mirror/record-ssh.js +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env node -'use strict'; -/* Record time-series snapshots of the data the dashboard uses, over SSH. - - Runs a fixed, read-only command loop on the device (no argument from this - side reaches the remote shell except the numeric sample count/interval): - system/network/wireless/hostapd status every seconds. Output is - parsed and sanitized (policy.js) locally before anything is written. - - usage: node record-ssh.js [samples=180] [interval=5] [ssh options...] */ - -const { spawn } = require('child_process'); -const fs = require('fs'); -const path = require('path'); -const readline = require('readline'); -const policy = require('./policy'); - -const [ target, outRoot, samplesArg, intervalArg, ...sshOpts ] = process.argv.slice(2); -const samples = parseInt(samplesArg || '180', 10), interval = parseInt(intervalArg || '5', 10); -if (!target || !outRoot || !(samples > 0 && samples <= 5000) || !(interval >= 1 && interval <= 300)) { - console.error('usage: record-ssh.js [samples] [interval] [ssh options...]'); - process.exit(2); -} -const out = path.join(outRoot, 'ssh-' + new Date().toISOString().replace(/[:.]/g, '-')); -fs.mkdirSync(out, { recursive: true, mode: 0o700 }); -const log = fs.openSync(path.join(out, 'snapshots.jsonl'), 'a', 0o600); - -/* read-only loop; everything it runs is a query */ -const remote = ` -n=0 -ubus call system board >/dev/null 2>&1 && { echo "@@C system board {}"; ubus call system board; } -while [ $n -lt ${samples} ]; do - echo "@@T $(date +%s)" - for c in "system info" "network.interface dump" "network.device status" "network.wireless status" "luci-rpc getWirelessDevices" "luci-rpc getHostHints"; do - echo "@@C $c {}"; ubus call $c 2>/dev/null || echo '{"@@error":1}' - done - for d in $(ubus call luci-rpc getWirelessDevices 2>/dev/null | jsonfilter -e '@.*.interfaces[*].ifname' 2>/dev/null); do - a="{\\"device\\":\\"$d\\"}" - echo "@@C iwinfo assoclist $a"; ubus call iwinfo assoclist "$a" 2>/dev/null || echo '{"@@error":1}' - echo "@@C iwinfo info $a"; ubus call iwinfo info "$a" 2>/dev/null || echo '{"@@error":1}' - for m in get_status get_clients; do - echo "@@C hostapd.$d $m {}"; ubus call hostapd.$d $m 2>/dev/null || echo '{"@@error":1}' - done - done - echo "@@F /proc/stat"; head -n 9 /proc/stat - echo "@@E" - n=$((n+1)); [ $n -lt ${samples} ] && sleep ${interval} -done -`; - -const ssh = spawn('ssh', [ ...sshOpts, '-o', 'BatchMode=yes', target, 'sh -s' ], { stdio: [ 'pipe', 'pipe', 'inherit' ] }); -ssh.stdin.end(remote); - -let t = null, cur = null, buf = [], count = 0, calls = 0; -function flush() { - if (!cur) return; - const text = buf.join('\n'); - if (cur.kind === 'file') { - fs.writeSync(log, JSON.stringify({ at: t, object: 'file', method: 'read', args: { path: cur.path }, result: { data: policy.sanitize(text + '\n') } }) + '\n'); - } else { - let result; - try { result = JSON.parse(text); } catch (e) { result = undefined; } - const error = !result || result['@@error'] ? true : undefined; - fs.writeSync(log, JSON.stringify({ at: t, object: cur.object, method: cur.method, args: policy.sanitize(cur.args), - result: error ? undefined : policy.sanitizeResult(cur.object, cur.method, cur.args, result), error }) + '\n'); - } - calls++; cur = null; buf = []; -} - -readline.createInterface({ input: ssh.stdout }).on('line', line => { - let m; - if ((m = /^@@T (\d+)$/.exec(line))) { flush(); t = +m[1] * 1000; } - else if ((m = /^@@C (\S+) (\S+) (.*)$/.exec(line))) { flush(); let a = {}; try { a = JSON.parse(m[3]); } catch (e) {} cur = { object: m[1], method: m[2], args: a }; } - else if ((m = /^@@F (\S+)$/.exec(line))) { flush(); cur = { kind: 'file', path: m[1] }; } - else if (line === '@@E') { flush(); count++; if (count % 12 === 0) console.log('samples', count, 'calls', calls); } - else if (cur) buf.push(line); -}); -ssh.on('exit', code => { - flush(); - fs.writeFileSync(path.join(out, 'summary.json'), JSON.stringify({ target: target.replace(/^.*@/, ''), samples: count, calls, interval, exit: code, finished: new Date().toISOString() }, null, 2), { mode: 0o600 }); - console.log('done', count, 'samples,', calls, 'calls, exit', code); -}); diff --git a/dev/replay/README.md b/dev/replay/README.md index 0f49b2b..a3a4072 100644 --- a/dev/replay/README.md +++ b/dev/replay/README.md @@ -62,7 +62,7 @@ The replay listens on 127.0.0.1 only, and on top of that: Inputs: -- `--mirror`: output of `dev/mirror/record-browser.js` / `record-ssh.js`. +- `--mirror`: a recording directory, see [Recording format](#recording-format). All `browser-*` and `ssh-*` directories are merged, newer wins. - `--rootfs` (or `$VANTAGE_ROOTFS`, default: `../vantage-rootfs` next to the repository; a directory or symlink holding an unpacked OpenWrt 25.12 @@ -125,7 +125,7 @@ within the JS-compatible subset of ucode (all upstream themes do). Routing follows the device dispatcher: the recorded menu (`/admin/menu`) resolves the path (`firstchild`, `alias`, `rewrite`, -`view`, `template`). Menu nodes blanked by the recorder's secret filter +`view`, `template`). Menu nodes blanked by the recording's secret filter (e.g. `admin/system/admin/password`) are restored from `menu.d`. ## ubus / HTTP replay @@ -160,7 +160,7 @@ Routing follows the device dispatcher: the recorded menu prints the loaded plugins at start (`vantage-plugin.js`: `luci.vantage`, luci-app-vantage's plugin). - Mirrored static files that are uhttpd's 404 page are served as 404, as on - the device (the recorder keeps bodies, not statuses). + the device (recordings keep bodies, not statuses). stderr lists coverage gaps once each: `unknown call`, `approximate (args differ)`, `unrecorded exec`, `write dropped`, `static not found`. Record the @@ -191,8 +191,8 @@ calls then answer from the recording or NOT_FOUND as before). - `luci getConntrackList`: ~25-30 invented flows between documentation addresses (192.0.2.0/24 -> 198.51.100.0/24, 203.0.113.0/24, 2001:db8::/32), in rpcd's field layout. -- `iwinfo scan ` (Status -> Channel Analysis; the recorder blocks - scans on purpose): the AP's own sibling BSSIDs on that radio (read from +- `iwinfo scan ` (Status -> Channel Analysis; never recorded, a + scan disturbs clients): the AP's own sibling BSSIDs on that radio (read from the recorded `network.wireless status` at run time, never written to the repo) plus 12 (2.4 GHz) / 7 (6 GHz, channels 1-93) invented neighbours with example SSIDs (`ExampleNet`, `Office-Demo`, ...) and locally @@ -377,44 +377,29 @@ The recorded device still runs the old UniWRT theme; its menu entries (Dashboard, System → Vantage Theme) are views that only work with that theme's CSS, so they are dropped from the menu unless `--keep-uniwrt`. -## Recording (`dev/mirror/`) - -Do not run the recorders unless you mean to read your own device, over the -wired management path. - -`record-browser.js` opens Chromium with a throwaway profile and a -browser-wide DevTools interceptor (a pipe, no TCP port): every request of -every tab, popup, worker and service worker is decided by -`dev/mirror/policy.js` before it leaves the browser. Reads on the allowlist -are forwarded with the body re-serialised exactly as checked and recorded; -writes, applies, scans, uploads, other hosts and other spellings of the -device are refused and listed in `blocked.log`. The profile is removed on -every exit (window closed, Ctrl+C, SIGTERM, SIGHUP, an error); if the -recorder is killed outright, the browser exits with it (the pipe closes), -but the profile stays in `$TMPDIR/vantage-rec-profile-*`. - -It needs the device certificate's pin, obtained over SSH (whose host key -you have verified), never from the TLS connection itself: - - scp -O root@:/etc/uhttpd.crt /tmp/device.crt - node dev/mirror/record-browser.js --cert /tmp/device.crt https:// ../vantage-mirror - -or, as a pin (`--spki` or `$VANTAGE_DEVICE_SPKI`; uhttpd's generated -certificate is DER, drop `-inform der` for a PEM one): - - ssh root@ cat /etc/uhttpd.crt | openssl x509 -inform der -pubkey -noout \ - | openssl pkey -pubin -outform der | openssl dgst -sha256 -binary | openssl base64 - -Chromium then accepts only that key (`--ignore-certificate-errors-spki-list`) -instead of ignoring certificate errors. Secrets are scrubbed before -anything is written (see the header of `policy.js`); `file.read` contents -are kept only for a short list of non-secret system files (`/proc`, `/sys`, -`board.json`, `rt_tables`, `sysupgrade.conf`, ...), so `rc.local` and -crontabs replay as ``. - -The recorder is an accident guard for a person clicking through stock -LuCI, not a sandbox for hostile page JavaScript: it does not see -WebSockets, for example. `tests/recorder-policy.test.js` covers the policy; -`VANTAGE_BROWSER_TESTS=1 node --test tests/recorder-browser.test.js` runs -the recorder against a local mock device in headless Chromium (second tab, -popup, workers, profile removal, the pin). +## Recording format + +A mirror is a directory of recordings. The replay loads every +`browser-/` and `ssh-/` directory in it, oldest +first, so newer recordings win for identical keys. The recorder that makes +them is not part of this repository; what matters is the layout: + +- `browser-*/rpc.jsonl`: one JSON object per line for each ubus call the + LuCI session made: `at` (ms since the epoch), `object`, `method`, `args` + and the device's `result` array (or `error`), as `/ubus` returned them. +- `browser-*/http.jsonl`: one object per line for other responses: `url` + and `body` for a page or file, or `argv` and `body` for a `cgi-exec` + command. Bodies are kept, statuses are not. +- `browser-*/static/`: the device's static files as served (LuCI's + `luci-static/` tree), used where the rootfs dump has no copy. +- `ssh-*/snapshots.jsonl`: repeated ubus calls over SSH, one object per + line with `at`, `object`, `method`, `args` and `result`; the replay turns + rows with the same key into a time series and samples it at their median + spacing (`Live` data). + +A recording holds everything the session saw, so it never goes into a +repository. Secrets (Wi-Fi keys, password hashes, tokens, private keys) +must be removed before a recording is written; `policy.js` holds the +scrub rules and the read-only allowlist a recorder is expected to apply, +and `tests/policy.test.js` covers them. Scans (`iwinfo scan`) are never +recorded: they take radios off channel; the replay generates them. diff --git a/dev/mirror/policy.js b/dev/replay/policy.js similarity index 99% rename from dev/mirror/policy.js rename to dev/replay/policy.js index a83fa1b..b626cf9 100644 --- a/dev/mirror/policy.js +++ b/dev/replay/policy.js @@ -1,5 +1,6 @@ 'use strict'; -/* Recording policy for the LuCI data mirror. +/* Policy for the LuCI data mirror: what a recording may contain. The + replay uses the parsers; a recorder applies the whole policy. 1. Only read-only requests may reach the device while recording. Every request the recording browser makes (any tab, popup or worker) is diff --git a/dev/replay/server.js b/dev/replay/server.js index 52e59fa..1071134 100644 --- a/dev/replay/server.js +++ b/dev/replay/server.js @@ -3,7 +3,7 @@ /* Offline LuCI replay server for theme work. Serves the LuCI web UI on 127.0.0.1 entirely from a recorded mirror - (dev/mirror/record-*.js), the device rootfs dump (core templates, + (see README, Recording format), the device rootfs dump (core templates, bootstrap, menu.d) and optionally a theme under development. Nothing is forwarded anywhere; writes land in an in-memory uci overlay. @@ -55,7 +55,7 @@ const { Engine } = require('./ut'); const { Store, safe } = require('./store'); const { Fixtures } = require('./fixtures'); const i18n = require('../i18n/catalog'); -const policy = require('../mirror/policy'); +const policy = require('./policy'); const wired = require('./wired'); /* ------------------------------------------------------------ arguments */ @@ -153,7 +153,7 @@ const version = (() => { return res; })(); -/* recorded menu; nodes the recorder's secret filter blanked out (e.g. a +/* recorded menu; nodes the recording's secret filter blanked out (e.g. a node named "password") are restored from the rootfs menu.d */ const menuSpecs = {}; for (const f of fs.readdirSync(path.join(rootfs, 'usr/share/luci/menu.d')).filter(f => f.endsWith('.json'))) { @@ -506,7 +506,7 @@ const MIME = { '.html': 'text/html; charset=UTF-8', '.txt': 'text/plain; charset=UTF-8', '.map': 'application/json' }; -/* the recorder keeps response bodies, not statuses: a mirrored file that is +/* a recording keeps response bodies, not statuses: a mirrored file that is uhttpd's 404 page was a 404 on the device too */ function recorded404(file, size) { if (size > 512) return false; diff --git a/dev/replay/synthetic.js b/dev/replay/synthetic.js index dfa0ace..2727b07 100644 --- a/dev/replay/synthetic.js +++ b/dev/replay/synthetic.js @@ -5,7 +5,7 @@ luci-bwc; the recorded firmware lacks it for load/conntrack, and the rows that were recorded for interface/wireless carry old timestamps, so a replayed graph never advances. Status -> Channel Analysis reads - `iwinfo scan`, which the recorder blocks on purpose (it disturbs clients). + `iwinfo scan`, which a recording never holds (a scan disturbs clients). Everything here is generated, labelled "synthetic" once on stderr, and switched off with --no-synthetic. diff --git a/docs/SPEC.md b/docs/SPEC.md index 53de1f8..2e53516 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -374,10 +374,13 @@ text around it so it looks the same size. ## 7. Development workflow -- `dev/mirror/`: read-only recorders (browser + SSH). Private data stays in - `~/Work/vantage-mirror` (mode 0700), never in the repo. -- `dev/replay/`: offline LuCI served from the mirror; `--theme-dir` loads - the theme from the working tree, reload to see edits. +- Recordings of the device (browser session + SSH snapshots) are made + with a private read-only tool that is not part of the repository. + Private data stays in `~/Work/vantage-mirror` (mode 0700), never in the + repo. +- `dev/replay/`: offline LuCI served from the mirror (format in its + README); `--theme-dir` loads the theme from the working tree, reload to + see edits. - Tests: node unit tests for data logic, headless screenshot suite over a fixed page list (light/dark/mobile) for visual review. - Build: OpenWrt SDK 25.12 in podman, same reproducible pipeline as before. diff --git a/tests/recorder-policy.test.js b/tests/policy.test.js similarity index 98% rename from tests/recorder-policy.test.js rename to tests/policy.test.js index 9527de3..bc3668d 100644 --- a/tests/recorder-policy.test.js +++ b/tests/policy.test.js @@ -1,10 +1,10 @@ 'use strict'; -/* dev/mirror/policy.js: what the recording browser may send to the device, - how bodies are parsed (like json-c and cgi-io do on the device), and what - is scrubbed before anything is written to the mirror. */ +/* dev/replay/policy.js: which LuCI requests are read-only, how bodies are + parsed (like json-c and cgi-io do on the device), and what is scrubbed + from recorded data. */ const test = require('node:test'); const assert = require('node:assert/strict'); -const policy = require('../dev/mirror/policy'); +const policy = require('../dev/replay/policy'); const BASE = 'https://device.example'; const SID = '0123456789abcdef0123456789abcdef'; diff --git a/tests/recorder-browser.test.js b/tests/recorder-browser.test.js deleted file mode 100644 index 703a28d..0000000 --- a/tests/recorder-browser.test.js +++ /dev/null @@ -1,266 +0,0 @@ -'use strict'; -const mkTmp = require('./tmpdir'); -/* dev/mirror/record-browser.js end to end, against a local mock device in - headless Chromium: requests from a second tab, a popup, a dedicated - worker and a service worker are refused like the first tab's; the device - gets the canonical bodies; the profile is removed on close and on - SIGTERM; the certificate pin admits the right key only. - - Starts one browser at a time, so it is opt-in: - VANTAGE_BROWSER_TESTS=1 node --test tests/recorder-browser.test.js - Needs chromium on PATH (and openssl for the pin test). Uses ports - 8220-8239 on 127.0.0.1 only. */ -const test = require('node:test'); -const assert = require('node:assert/strict'); -const { spawn, execFileSync } = require('child_process'); -const fs = require('fs'); -const http = require('http'); -const https = require('https'); -const net = require('net'); -const os = require('os'); -const path = require('path'); - -const ROOT = path.join(__dirname, '..'); -const RECORDER = path.join(ROOT, 'dev/mirror/record-browser.js'); -const which = b => { try { return execFileSync('sh', [ '-c', 'command -v ' + b ]).toString().trim(); } catch (e) { return null; } }; -const CHROMIUM = which('chromium'); -const enabled = process.env.VANTAGE_BROWSER_TESTS === '1' && CHROMIUM; -const skip = !enabled && 'set VANTAGE_BROWSER_TESTS=1 (and have chromium on PATH) to run'; -const sleep = ms => new Promise(r => setTimeout(r, ms)); - -async function freePort() { - for (let p = 8220; p <= 8239; p++) { - const ok = await new Promise(res => { - const s = net.createServer().once('error', () => res(false)).listen(p, '127.0.0.1', () => s.close(() => res(true))); - }); - if (ok) return p; - } - throw new Error('no free port in 8220-8239'); -} - -const SID = 'ab'.repeat(16); -const rpc = (id, object, method, args) => ({ jsonrpc: '2.0', id, method: 'call', params: [ SID, object, method, args || {} ] }); -const READ_BODY = JSON.stringify([ rpc(1, 'session', 'list') ]); -const WRITE = name => JSON.stringify([ rpc(7, 'system', 'reboot', { from: name }) ]); - -/* mock device: logs every request that reaches it */ -function mockDevice(tls) { - const seen = []; - const handler = (req, res) => { - const chunks = []; - req.on('data', c => chunks.push(c)); - req.on('end', () => { - const body = Buffer.concat(chunks).toString('utf8'); - const p = req.url.replace(/\?.*$/, ''); - seen.push({ method: req.method, path: p, body, type: req.headers['content-type'] || '' }); - if (p === '/ubus/' || p === '/ubus') { - let msgs = []; - try { msgs = JSON.parse(body); } catch (e) {} - const reply = (Array.isArray(msgs) ? msgs : [ msgs ]).map(m => ({ jsonrpc: '2.0', id: m && m.id, result: [ 0, { mock: 'reply', password: 'hunter2' } ] })); - res.writeHead(200, { 'Content-Type': 'application/json' }); - return res.end(JSON.stringify(reply)); - } - if (p === '/cgi-bin/cgi-exec') { res.writeHead(200, { 'Content-Type': 'text/plain' }); return res.end('kernel log line\n'); } - if (req.method === 'POST' && p.startsWith('/cgi-bin/luci')) { - res.writeHead(302, { Location: '/cgi-bin/luci/', 'Set-Cookie': 'sysauth_https=' + SID + '; path=/cgi-bin/luci/' }); - return res.end(); - } - if (p === '/worker.js') { - res.writeHead(200, { 'Content-Type': 'application/javascript' }); - return res.end(`onmessage = e => fetch('/ubus/', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: e.data }) - .then(r => postMessage('status ' + r.status), () => postMessage('blocked'));`); - } - if (p === '/sw.js') { res.writeHead(200, { 'Content-Type': 'application/javascript' }); return res.end('self.addEventListener("fetch", () => {});'); } - res.writeHead(200, { 'Content-Type': 'text/html' }); - res.end('mock

mock luci ' + p + '

'); - }); - }; - const server = tls ? https.createServer(tls, handler) : http.createServer(handler); - return { server, seen }; -} - -/* the recorder, with chromium made headless and given a debugging port for - this test's own driver (the recorder itself only uses the pipe) */ -function startRecorder(tmp, args) { - const bin = path.join(tmp, 'bin'); - fs.mkdirSync(bin, { recursive: true }); - fs.writeFileSync(path.join(bin, 'chromium'), `#!/bin/sh\nexec ${CHROMIUM} --headless=new --remote-debugging-port=0 "$@"\n`, { mode: 0o755 }); - const ptmp = path.join(tmp, 'ptmp'); - fs.mkdirSync(ptmp, { recursive: true }); - const env = Object.assign({}, process.env, { PATH: bin + ':' + process.env.PATH, TMPDIR: ptmp }); - delete env.VANTAGE_DEVICE_SPKI; - const child = spawn(process.execPath, [ RECORDER, ...args ], { env, stdio: [ 'ignore', 'pipe', 'pipe' ] }); - let log = ''; - child.stdout.on('data', d => { log += d; }); - child.stderr.on('data', d => { log += d; }); - const exited = new Promise(r => child.on('exit', (code, sig) => r({ code, sig }))); - return { child, exited, ptmp, log: () => log }; -} - -function profileDirs(ptmp) { - return fs.readdirSync(ptmp).filter(d => d.startsWith('vantage-rec-profile-')).map(d => path.join(ptmp, d)); -} - -/* CDP client for the test driver */ -async function driver(ptmp) { - let port, bpath; - for (let i = 0; i < 100 && !port; i++) { - const dirs = profileDirs(ptmp); - try { [ port, bpath ] = fs.readFileSync(path.join(dirs[0], 'DevToolsActivePort'), 'utf8').split('\n'); } catch (e) { await sleep(100); } - } - assert.ok(port, 'debugging port of the test browser'); - const ws = new WebSocket(`ws://127.0.0.1:${port}${bpath}`); - await new Promise((r, j) => { ws.onopen = r; ws.onerror = j; }); - let id = 0; - const pend = new Map(); - ws.onmessage = e => { const m = JSON.parse(e.data); if (m.id && pend.has(m.id)) { pend.get(m.id)(m); pend.delete(m.id); } }; - const send = (method, params, sessionId) => new Promise(r => { const i = ++id; pend.set(i, r); ws.send(JSON.stringify({ id: i, method, params: params || {}, sessionId })); }); - const evaluate = async (sessionId, expression, userGesture) => { - const r = await send('Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true, userGesture: !!userGesture }, sessionId); - if (r.error) throw new Error(r.error.message); - return r.result.result.value; - }; - return { ws, send, evaluate }; -} - -async function waitFor(fn, ms, what) { - for (let t = 0; t < ms; t += 100) { const v = await fn(); if (v) return v; await sleep(100); } - throw new Error('timed out waiting for ' + what); -} - -const post = (url, body, type) => `fetch(${JSON.stringify(url)}, { method: 'POST', headers: { 'Content-Type': ${JSON.stringify(type || 'application/json')} }, body: ${JSON.stringify(body)} }).then(r => 'status ' + r.status, () => 'blocked')`; - -test('recorder: every tab, popup and worker is filtered; canonical bodies; profile removed', { skip, timeout: 90000 }, async () => { - const tmp = mkTmp('vantage-rec-test-'); - const port = await freePort(); - const { server, seen } = mockDevice(null); - await new Promise(r => server.listen(port, '127.0.0.1', r)); - const B = `http://127.0.0.1:${port}`; - const mirror = path.join(tmp, 'mirror'); - const rec = startRecorder(tmp, [ B, mirror ]); - try { - await waitFor(() => seen.some(s => s.path === '/cgi-bin/luci/'), 20000, 'the device page'); - const d = await driver(rec.ptmp); - const pages = (await d.send('Target.getTargets')).result.targetInfos.filter(t => t.type === 'page' && t.url.startsWith(B)); - assert.equal(pages.length, 1); - const s1 = (await d.send('Target.attachToTarget', { targetId: pages[0].targetId, flatten: true })).result.sessionId; - - /* first tab: a read goes through (re-serialised), a write does not */ - assert.equal(await d.evaluate(s1, post(B + '/ubus/', '[ ' + READ_BODY.slice(1))), 'status 200', rec.log()); - assert.equal(await d.evaluate(s1, post(B + '/ubus/', WRITE('tab1'))), 'blocked'); - /* duplicate keys, NUL-suffixed keys, REST ubus, other spellings of the device */ - assert.equal(await d.evaluate(s1, post(B + '/ubus/', '{"jsonrpc":"2.0","id":1,"method":"call","params":["x","session","list",{}],"params":["x","system","reboot",{}]}')), 'blocked'); - assert.equal(await d.evaluate(s1, post(B + '/ubus/', '{"jsonrpc":"2.0","id":1,"method":"call","params":["x","session","list",{}],"params\\u0000":["x","system","reboot",{}]}')), 'blocked'); - assert.equal(await d.evaluate(s1, post(B + '/ubus/call/system', '{"jsonrpc":"2.0","id":1,"method":"call","params":["x","system","reboot",{}]}')), 'blocked'); - assert.equal(await d.evaluate(s1, post(`http://localhost:${port}/ubus/`, WRITE('alias'))), 'blocked'); - /* login: exactly the two fields, form encoded */ - assert.equal(await d.evaluate(s1, post(B + '/cgi-bin/luci/admin/status', 'luci_username=root&luci_password=pw', 'application/x-www-form-urlencoded')), 'status 200'); - assert.equal(await d.evaluate(s1, post(B + '/cgi-bin/luci/admin/system/flash', 'luci_username=root&luci_password=pw&reset=1', 'application/x-www-form-urlencoded')), 'blocked'); - assert.equal(await d.evaluate(s1, post(B + '/cgi-bin/cgi-exec/x', 'luci_username=&luci_password=&sessionid=' + SID + '&command=/sbin/reboot', 'application/x-www-form-urlencoded')), 'blocked'); - /* cgi-exec: allowlisted argv in canonical form only */ - assert.equal(await d.evaluate(s1, post(B + '/cgi-bin/cgi-exec', `sessionid=${SID}&command=%2Fbin%2Fdmesg%20-r&stderr=0`, 'application/x-www-form-urlencoded')), 'status 200'); - assert.equal(await d.evaluate(s1, post(B + '/cgi-bin/cgi-exec', `sessionid=${SID}&command=/usr/sbin/iptables-save&command=/sbin/reboot`, 'application/x-www-form-urlencoded')), 'blocked'); - - /* popup opened by the page */ - await d.evaluate(s1, `window.__p = window.open(${JSON.stringify(B + '/cgi-bin/luci/admin/popup')}); !!window.__p`, true); - await waitFor(() => seen.some(s => s.path === '/cgi-bin/luci/admin/popup'), 10000, 'the popup'); - assert.equal(await d.evaluate(s1, `window.__p.${post(B + '/ubus/', WRITE('popup'))}`), 'blocked'); - - /* a new tab (Ctrl+T, middle click, target=_blank) */ - const nt = (await d.send('Target.createTarget', { url: B + '/cgi-bin/luci/admin/newtab' })).result.targetId; - await waitFor(() => seen.some(s => s.path === '/cgi-bin/luci/admin/newtab'), 10000, 'the new tab'); - const s2 = (await d.send('Target.attachToTarget', { targetId: nt, flatten: true })).result.sessionId; - assert.equal(await d.evaluate(s2, post(B + '/ubus/', WRITE('newtab'))), 'blocked'); - assert.equal(await d.evaluate(s2, post(B + '/cgi-bin/luci/admin/system/flash', 'reset=1', 'application/x-www-form-urlencoded')), 'blocked'); - - /* dedicated worker and service worker */ - assert.equal(await d.evaluate(s2, `new Promise(r => { const w = new Worker('/worker.js'); w.onmessage = e => r(e.data); w.onerror = () => r('worker failed'); w.postMessage(${JSON.stringify(WRITE('worker'))}); })`), 'blocked'); - assert.equal(await d.evaluate(s2, `navigator.serviceWorker.register('/sw.js').then(() => 'registered', () => 'refused')`), 'refused'); - - /* what reached the device */ - const bodies = seen.map(s => s.body).join('\n'); - assert.doesNotMatch(bodies, /reboot|reset=1/, 'no write reached the device'); - const rpcs = seen.filter(s => s.path === '/ubus/'); - assert.deepEqual(rpcs.map(s => s.body), [ READ_BODY ], 'the checked, re-serialised body'); - const exec = seen.filter(s => s.path === '/cgi-bin/cgi-exec'); - assert.deepEqual(exec.map(s => s.body), [ `sessionid=${SID}&command=%2Fbin%2Fdmesg%20-r&stderr=0` ]); - assert.ok(!seen.some(s => s.path === '/sw.js'), 'service worker script not fetched'); - - await d.send('Browser.close'); - d.ws.close(); - const ex = await rec.exited; - assert.equal(ex.code, 0, rec.log()); - assert.deepEqual(profileDirs(rec.ptmp), [], 'browser profile removed'); - const [ run ] = fs.readdirSync(mirror); - const summary = JSON.parse(fs.readFileSync(path.join(mirror, run, 'summary.json'), 'utf8')); - assert.ok(summary.stats.blocked >= 12, JSON.stringify(summary.stats)); - assert.ok(summary.stats.pages >= 3, JSON.stringify(summary.stats)); - const rows = fs.readFileSync(path.join(mirror, run, 'rpc.jsonl'), 'utf8').trim().split('\n').map(l => JSON.parse(l)); - assert.deepEqual(rows.map(r => r.object + '.' + r.method), [ 'session.list' ]); - assert.equal(rows[0].result[1].password, ''); - assert.match(fs.readFileSync(path.join(mirror, run, 'http.jsonl'), 'utf8'), /"argv":\["\/bin\/dmesg","-r"\]/); - } - finally { - rec.child.kill('SIGKILL'); - await new Promise(r => server.close(r)); - } -}); - -test('recorder: SIGTERM kills the browser and removes the profile', { skip, timeout: 60000 }, async () => { - const tmp = mkTmp('vantage-rec-test-'); - const port = await freePort(); - const { server, seen } = mockDevice(null); - await new Promise(r => server.listen(port, '127.0.0.1', r)); - const rec = startRecorder(tmp, [ `http://127.0.0.1:${port}`, path.join(tmp, 'mirror') ]); - try { - await waitFor(() => seen.some(s => s.path === '/cgi-bin/luci/'), 20000, 'the device page'); - const [ prof ] = profileDirs(rec.ptmp); - rec.child.kill('SIGTERM'); - const ex = await rec.exited; - assert.equal(ex.code, 143, rec.log()); - assert.ok(!fs.existsSync(prof), 'profile removed'); - await sleep(500); - const alive = fs.readdirSync('/proc').filter(p => /^\d+$/.test(p)).some(p => { - try { return fs.readFileSync(`/proc/${p}/cmdline`, 'utf8').includes('--user-data-dir=' + prof); } catch (e) { return false; } - }); - assert.ok(!alive, 'no browser process left'); - } - finally { - rec.child.kill('SIGKILL'); - await new Promise(r => server.close(r)); - } -}); - -test('recorder: https needs a pin, and only the pinned key is accepted', { skip: skip || (!which('openssl') && 'no openssl'), timeout: 90000 }, async () => { - const tmp = mkTmp('vantage-rec-test-'); - const key = path.join(tmp, 'k.pem'), crt = path.join(tmp, 'c.pem'); - execFileSync('openssl', [ 'req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256', '-nodes', '-keyout', key, '-out', crt, - '-days', '1', '-subj', '/CN=127.0.0.1', '-addext', 'subjectAltName=IP:127.0.0.1' ], { stdio: 'ignore' }); - const port = await freePort(); - const { server, seen } = mockDevice({ key: fs.readFileSync(key), cert: fs.readFileSync(crt) }); - await new Promise(r => server.listen(port, '127.0.0.1', r)); - const B = `https://127.0.0.1:${port}`; - try { - /* no pin: refused before a browser starts */ - const none = startRecorder(tmp, [ B, path.join(tmp, 'm0') ]); - assert.equal((await none.exited).code, 2); - assert.match(none.log(), /missing certificate pin/); - - /* wrong pin: the page never loads */ - const wrong = startRecorder(tmp, [ '--spki', Buffer.alloc(32, 7).toString('base64'), B, path.join(tmp, 'm1') ]); - await sleep(5000); - assert.equal(seen.length, 0, 'no request with a wrong pin'); - wrong.child.kill('SIGTERM'); - await wrong.exited; - - /* the device's own certificate */ - const right = startRecorder(tmp, [ '--cert', crt, B, path.join(tmp, 'm2') ]); - await waitFor(() => seen.some(s => s.path === '/cgi-bin/luci/'), 20000, 'the pinned page'); - right.child.kill('SIGTERM'); - await right.exited; - assert.deepEqual(profileDirs(right.ptmp), [], wrong.log() + right.log()); - } - finally { - await new Promise(r => server.close(r)); - } -});