feat(watch): gameplay watcher slices reference frames into tmp

README.md

@@ -87,3 +87,28 @@ Run `npm run scout -- <id> <repo> <sha>`; it clones into `.sources/`, logs to `.
 and then runs the validator on that game. Promoting a new map with a catalog row belongs
 to the review lane, not to the scout.
 Every card gets a full human or agent review before merge.
+
+## Gameplay watcher
+
+Mechanic cards say what a game does; sometimes the authoring agent needs to *see* it. The watcher
+searches YouTube for a gameplay video of a reference game, downloads one segment, and slices
+frames plus a contact sheet into a tmp folder:
+
+```sh
+npm run watch -- "super mario 64 triple jump" --duration 25 --frames 6
+# [0] 00:20:29 Super Mario 64 Speedrun Movement Guide — Derian Sera
+# look: /tmp/threenative-watch-Fo7al9/sheet.jpg
+```
+
+It needs `yt-dlp` (`pipx install yt-dlp`) and `ffmpeg` on `PATH`. Read `sheet.jpg` first: one
+image holding every frame, in order. Then aim — `--pick N` for another search result, `--from
+NN%` to jump into a long video, `--at 12,35,61` for exact frames of a known moment, `--frames N`
+for more or fewer slices. Each frame is 640px wide and named for its timestamp; `watch.json`
+records the video, the segment and every frame time, so a claim about the mechanic can name the
+clip it came from. Cards still cite source code at the pinned SHA; they never cite a video.
+
+The default window opens 10% into the video, past the title card and menu. A vertical slice of an
+open-ended session is still a guess: the first sheet is for deciding where to look next.
+
+Frames are transcription of someone else's footage. They stay in tmp for studying a mechanic,
+never in `data/`, and `check:data` cannot see them to complain.

package.json

@@ -8,7 +8,8 @@
     "test": "node --test tests/*.spec.mjs",
     "validate": "node scripts/validate.mjs",
     "release:assets": "node scripts/validate.mjs --release",
-    "scout": "sh scripts/scout/run.sh"
+    "scout": "sh scripts/scout/run.sh",
+    "watch": "node scripts/watch.mjs"
   },
   "devDependencies": {
     "ajv": "8.17.1"

scripts/watch.mjs

@@ -0,0 +1,221 @@
+// Gameplay watcher: search YouTube for a gameplay video of a mechanic reference, slice frames
+// out of one segment and drop them, plus a contact sheet, into a tmp folder for the agent to look
+// at. Needs `yt-dlp` (search and download) and `ffmpeg` (frames) on PATH.
+import { execFile } from 'node:child_process';
+import { mkdtempSync, mkdirSync, writeFileSync, existsSync, readdirSync, statSync, unlinkSync } from 'node:fs';
+import { promisify } from 'node:util';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const execute = promisify(execFile);
+const usage = `usage: node scripts/watch.mjs <game mechanic words> [options]
+
+  --pick N        use search result N (default 0, the top one)
+  --from T        segment start: seconds, mm:ss, hh:mm:ss, or NN% of the video (default 10%, past the intro)
+  --duration T    seconds of gameplay to sample (default 90)
+  --frames N      frames to slice from the segment (default 12)
+  --at T[,T...]   exact timestamps to slice instead of even sampling
+  --out DIR       output folder (default: a fresh mkdtemp under the system tmp)
+  --keep          keep the downloaded clip next to the frames
+  --help          print this text
+
+Slices at 640px wide, named frames/NN-tSECONDSs.jpg, plus sheet.jpg (one image, all frames).
+Frames are reference material for editing a mechanic: never commit them.`;
+
+export function seconds(value, label = 'time') {
+  const text = String(value ?? '').trim();
+  if (!/^\d+(:\d{1,2}){0,2}(\.\d+)?$/u.test(text)) throw new Error(`bad ${label}: ${value}`);
+  const parts = text.split(':').map(Number);
+  if (parts.slice(1).some(part => part >= 60)) throw new Error(`bad ${label}: ${value}`);
+  return parts.reduce((total, part) => total * 60 + part, 0);
+}
+
+/** Segment start: plain time or a share of the whole video, which is how you guess your way into
+ *  the middle of a long playthrough when the search result is two hours long. */
+export function offset(value, durationSeconds) {
+  if (value === undefined || value === '0') return 0;
+  if (!String(value).trim().endsWith('%')) return seconds(value, '--from');
+  const share = Number(String(value).trim().slice(0, -1));
+  if (!Number.isFinite(share) || share < 0 || share > 100) throw new Error(`bad --from percent: ${value}`);
+  if (!durationSeconds) throw new Error('--from percent needs a video duration from the search result');
+  return durationSeconds * share / 100;
+}
+
+/** Even sampling: one frame in the middle of each of N cells, so both ends of the segment show. */
+export const sampleTimes = (from, duration, frames) =>
+  Array.from({ length: frames }, (_, index) => from + (index + 0.5) * duration / frames);
+
+export const frameName = (index, at) => `${String(index).padStart(2, '0')}-t${at.toFixed(1)}s.jpg`;
+
+export function tileGrid(count) {
+  const columns = Math.min(3, Math.max(1, count));
+  return { columns, rows: Math.ceil(count / columns) };
+}
+
+const clock = value => [Math.floor(value / 3600), Math.floor(value / 60) % 60, Math.floor(value % 60)]
+  .map(part => String(part).padStart(2, '0')).join(':');
+
+export const section = (from, to) => `*${clock(from)}-${clock(to)}`;
+
+export function parseArgs(args) {
+  const options = { duration: 90, frames: 12, pick: 0, keep: false, words: [] };
+  const valued = new Set(['--pick', '--from', '--duration', '--frames', '--at', '--out']);
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index];
+    if (arg === '--help' || arg === '-h') { options.help = true; continue; }
+    if (arg === '--keep') { options.keep = true; continue; }
+    if (valued.has(arg)) {
+      const value = args[index + 1];
+      if (value === undefined || value.startsWith('--')) throw new Error(`missing value for ${arg}`);
+      options[arg.slice(2)] = value;
+      index += 1;
+      continue;
+    }
+    if (arg.startsWith('--')) throw new Error(`unknown flag ${arg}`);
+    options.words.push(arg);
+  }
+  const whole = name => {
+    const value = Number(options[name]);
+    if (!Number.isInteger(value) || value < 0) throw new Error(`bad --${name}: ${options[name]}`);
+    return value;
+  };
+  options.frames = whole('frames');
+  options.pick = whole('pick');
+  options.duration = whole('duration');
+  if (options.frames < 1) throw new Error('--frames needs at least one frame');
+  if (options.duration < 1) throw new Error('--duration needs at least one second');
+  if (options.at) {
+    const times = String(options.at).split(',').map(part => seconds(part, '--at'));
+    if (!times.length) throw new Error(`bad --at: ${options.at}`);
+    options.at = times.sort((left, right) => left - right);
+  }
+  options.query = options.words.join(' ').trim();
+  return options;
+}
+
+function tool(name, hint) {
+  const found = [process.env.PATH ?? '', path.join(os.homedir(), '.local', 'bin')]
+    .join(path.delimiter).split(path.delimiter)
+    .map(directory => path.join(directory, name))
+    .find(file => existsSync(file));
+  if (!found) throw new Error(`${name} not found on PATH${hint ? `: ${hint}` : ''}`);
+  return found;
+}
+
+async function search(bin, query) {
+  const { stdout } = await execute(bin, ['--flat-playlist', '-J', `ytsearch5:${query}`], { maxBuffer: 64 * 1024 * 1024 });
+  return (JSON.parse(stdout).entries ?? []).filter(entry => entry.id).map(entry => ({
+    id: entry.id,
+    title: entry.title ?? entry.id,
+    channel: entry.channel ?? entry.uploader ?? '',
+    durationSeconds: entry.duration ?? null,
+    url: `https://www.youtube.com/watch?v=${entry.id}`,
+  }));
+}
+
+async function download(bin, video, from, length, folder) {
+  try {
+    await execute(bin, [
+      '--no-playlist', '--no-warnings', '--no-progress', '--force-overwrites',
+      '--merge-output-format', 'mp4',
+      '--download-sections', section(from, from + length),
+      '-f', 'bv*[height<=480]+ba/b[height<=480]/b',
+      '-o', path.join(folder, 'clip.%(ext)s'),
+      video.url,
+    ], { maxBuffer: 64 * 1024 * 1024 });
+  } catch (error) {
+    const tail = String(error.stderr ?? error.message).trim().split('\n').slice(-3).join(' ');
+    throw new Error(`download failed for ${video.url}: ${tail}`);
+  }
+  const clip = readdirSync(folder).find(name => name.startsWith('clip.'));
+  if (!clip) throw new Error(`download produced no clip for ${video.url}`);
+  return path.join(folder, clip);
+}
+
+async function slice(bin, clip, times, from, folder) {
+  const written = [];
+  for (const [position, at] of times.entries()) {
+    const file = path.join(folder, frameName(position + 1, at));
+    // yt-dlp rebases a downloaded section to 0, so the seek is relative to the segment start.
+    const seek = Math.max(0, at - from);
+    try {
+      await execute(bin, ['-nostdin', '-loglevel', 'error', '-ss', seek.toFixed(2), '-i', clip,
+        '-frames:v', '1', '-vf', 'scale=640:-2', '-q:v', '3', '-y', file]);
+      if (existsSync(file) && statSync(file).size) written.push({ index: position + 1, at, file });
+    } catch {
+      // A time past the end of the downloaded clip yields no frame and is dropped.
+    }
+  }
+  return written;
+}
+
+async function contactSheet(bin, frames, folder, file) {
+  if (!frames.length) return null;
+  const { columns, rows } = tileGrid(frames.length);
+  await execute(bin, ['-nostdin', '-loglevel', 'error', '-pattern_type', 'glob',
+    '-i', path.join(folder, '*.jpg'), '-vf', `tile=${columns}x${rows}:padding=6:margin=6`,
+    '-frames:v', '1', '-q:v', '3', '-y', file]);
+  return existsSync(file) ? file : null;
+}
+
+export async function watch(args) {
+  const options = parseArgs(args);
+  if (options.help) { console.log(usage); return; }
+  if (!options.query) throw new Error(`missing search words\n\n${usage}`);
+  const ytDlp = tool('yt-dlp', 'install it with `pipx install yt-dlp`');
+  const ffmpeg = tool('ffmpeg');
+  const folder = options.out ? path.resolve(options.out) : mkdtempSync(path.join(os.tmpdir(), 'threenative-watch-'));
+  if (options.out) {
+    if (existsSync(folder) && readdirSync(folder).length) throw new Error(`output folder is not empty: ${folder}`);
+    mkdirSync(folder, { recursive: true });
+  }
+
+  const candidates = await search(ytDlp, options.query);
+  if (!candidates.length) throw new Error(`no YouTube result for "${options.query}"`);
+  candidates.forEach((video, position) => console.log(`[${position}] ${video.durationSeconds ? clock(video.durationSeconds) : '--:--'} ${video.title}${video.channel ? ` — ${video.channel}` : ''}`));
+  const video = candidates[options.pick];
+  if (!video) throw new Error(`--pick ${options.pick} but the search returned ${candidates.length}`);
+
+  const from = options.at
+    ? Math.max(0, options.at[0] - 1)
+    : offset(options.from ?? '10%', video.durationSeconds);
+  const length = options.at
+    ? Math.max(1, options.at.at(-1) + 1 - from)
+    : options.duration;
+  const times = options.at ?? sampleTimes(from, length, options.frames);
+
+  const framesFolder = path.join(folder, 'frames');
+  mkdirSync(framesFolder, { recursive: true });
+
+  const clip = await download(ytDlp, video, from, length, folder);
+  const frames = await slice(ffmpeg, clip, times, from, framesFolder);
+  if (!frames.length) throw new Error(`no frame sliced from ${video.url} between ${clock(from)} and ${clock(from + length)}`);
+  const sheetFile = await contactSheet(ffmpeg, frames, framesFolder, path.join(folder, 'sheet.jpg'));
+  if (!options.keep) unlinkSync(clip);
+  const keptClip = options.keep ? path.basename(clip) : null;
+
+  const relative = file => path.relative(folder, file);
+  writeFileSync(path.join(folder, 'watch.json'), `${JSON.stringify({
+    version: 1,
+    query: options.query,
+    video: { id: video.id, title: video.title, channel: video.channel, url: video.url, durationSeconds: video.durationSeconds },
+    segment: { from, seconds: length, sampled: Boolean(options.at) ? 'timestamps' : 'even' },
+    frames: frames.map(frame => ({ index: frame.index, at: Number(frame.at.toFixed(2)), path: relative(frame.file) })),
+    sheet: sheetFile ? relative(sheetFile) : null,
+    clip: keptClip,
+    note: 'Gameplay frames for studying a mechanic. Reference only: not redistributable, never commit.',
+  }, null, 2)}\n`);
+
+  console.log(`\n${video.title} @ ${clock(from)} +${Math.round(length)}s → ${frames.length} frames`);
+  console.log(`watch: ${folder}`);
+  if (sheetFile) console.log(`look: ${sheetFile}`);
+}
+
+export function run(args = process.argv.slice(2)) {
+  return watch(args);
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  run().catch(error => { console.error(error.message); process.exitCode = 1; });
+}

tests/watch.spec.mjs

@@ -0,0 +1,129 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { readFileSync, readdirSync, existsSync } from 'node:fs';
+import path from 'node:path';
+import { frameName, offset, parseArgs, sampleTimes, section, seconds, tileGrid } from '../scripts/watch.mjs';
+import { cleanup, root, tempDir, write, writeExecutable } from './support.mjs';
+
+test('seconds accepts plain, mm:ss and hh:mm:ss', () => {
+  assert.equal(seconds('90'), 90);
+  assert.equal(seconds('1:30'), 90);
+  assert.equal(seconds('01:02:03.5'), 3723.5);
+  assert.throws(() => seconds('1:75'), /bad/);
+  assert.throws(() => seconds('soon'), /bad/);
+});
+
+test('offset reads a whole-video share and defaults to the start', () => {
+  assert.equal(offset(undefined, 600), 0);
+  assert.equal(offset('50%', 600), 300);
+  assert.equal(offset('1:00', 600), 60);
+  assert.throws(() => offset('50%', null), /duration/);
+  assert.throws(() => offset('120%', 600), /percent/);
+});
+
+test('sampleTimes centres one frame in each cell and spans the segment', () => {
+  assert.deepEqual(sampleTimes(0, 90, 3), [15, 45, 75]);
+  assert.equal(sampleTimes(10, 90, 12).length, 12);
+  assert.ok(sampleTimes(10, 90, 12).every(at => at > 10 && at < 100));
+});
+
+test('tileGrid never leaves an empty trailing cell row', () => {
+  assert.deepEqual(tileGrid(1), { columns: 1, rows: 1 });
+  assert.deepEqual(tileGrid(5), { columns: 3, rows: 2 });
+  assert.deepEqual(tileGrid(12), { columns: 3, rows: 4 });
+});
+
+test('section quotes a download window and frameName carries the timestamp', () => {
+  assert.equal(section(30, 120), '*00:00:30-00:02:00');
+  assert.equal(frameName(3, 12.44), '03-t12.4s.jpg');
+});
+
+test('parseArgs rejects unknown flags and bad counts', () => {
+  assert.throws(() => parseArgs(['--nope']), /unknown flag/);
+  assert.throws(() => parseArgs(['--frames', '0']), /at least one frame/);
+  assert.throws(() => parseArgs(['--pick']), /missing value/);
+  const options = parseArgs(['black', 'myth', 'wukong', 'parry', '--frames', '4', '--at', '1:30,0:10']);
+  assert.equal(options.query, 'black myth wukong parry');
+  assert.equal(options.frames, 4);
+  assert.deepEqual(options.at, [10, 90]);
+});
+
+test('the watcher slices frames, a sheet and a manifest from the picked result', () => {
+  const dir = tempDir('watch-');
+  const output = path.join(dir, 'out');
+  const search = JSON.stringify({
+    entries: [
+      { id: 'aaa', title: 'Long playthrough', channel: 'Someone', duration: 3600 },
+      { id: 'bbb', title: 'Parry montage', channel: 'Other', duration: 120 },
+    ],
+  });
+  // Fake tools: yt-dlp answers the search and leaves a clip; ffmpeg writes whatever path it is
+  // given as its last argument, which is how the real one ends a command line.
+  writeExecutable(dir, 'yt-dlp', `#!/bin/sh
+case "$*" in
+  *-J*) printf '%s' '${search}' ;;
+  *)
+    out=''
+    while [ "$#" -gt 0 ]; do
+      if [ "$1" = '-o' ]; then out=$2; fi
+      shift
+    done
+    : > "$(printf '%s' "$out" | sed 's/%(ext)s/mp4/')"
+    ;;
+esac
+`);
+  writeExecutable(dir, 'ffmpeg', `#!/bin/sh
+for a in "$@"; do out=$a; done
+printf '%s' "$*" >> '${path.join(dir, 'ffmpeg.log')}'
+printf '\\n' >> '${path.join(dir, 'ffmpeg.log')}'
+printf 'jpeg' > "$out"
+`);
+  const result = spawnSync(process.execPath, ['scripts/watch.mjs', 'parry', 'timing', '--pick', '1', '--at', '100,130', '--out', output], {
+    cwd: root,
+    encoding: 'utf8',
+    timeout: 60_000,
+    env: { ...process.env, PATH: `${dir}${path.delimiter}${process.env.PATH}` },
+  });
+  assert.equal(result.status, 0, result.stderr);
+  assert.match(result.stdout, /Parry montage/);
+  assert.match(result.stdout, /2 frames/);
+
+  const frames = readdirSync(path.join(output, 'frames'));
+  assert.deepEqual(frames, ['01-t100.0s.jpg', '02-t130.0s.jpg']);
+  assert.ok(existsSync(path.join(output, 'sheet.jpg')));
+  const manifest = JSON.parse(readFileSync(path.join(output, 'watch.json'), 'utf8'));
+  assert.equal(manifest.video.id, 'bbb');
+  assert.equal(manifest.video.url, 'https://www.youtube.com/watch?v=bbb');
+  assert.equal(manifest.segment.from, 99, 'the window starts one second before the first timestamp');
+  assert.equal(manifest.segment.seconds, 32);
+  assert.equal(manifest.segment.sampled, 'timestamps');
+  assert.deepEqual(manifest.frames.map(frame => frame.at), [100, 130]);
+  assert.equal(manifest.frames[0].path, 'frames/01-t100.0s.jpg');
+  assert.equal(manifest.sheet, 'sheet.jpg');
+  assert.equal(manifest.clip, null, 'the clip is thrown away unless --keep asks for it');
+  assert.match(manifest.note, /never commit/);
+  // Seeks must be relative to the segment: yt-dlp rebases a downloaded section to zero.
+  const seeks = readFileSync(path.join(dir, 'ffmpeg.log'), 'utf8').split('\n')
+    .filter(line => line.includes('-ss')).map(line => line.split('-ss ')[1].split(' ')[0]);
+  assert.deepEqual(seeks, ['1.00', '31.00']);
+  cleanup(dir);
+});
+
+test('the watcher refuses a non-empty output folder', () => {
+  const dir = tempDir('watch-');
+  write(dir, 'busy/keep.txt', 'x');
+  const result = spawnSync(process.execPath, ['scripts/watch.mjs', 'parry', '--out', path.join(dir, 'busy')], {
+    cwd: root, encoding: 'utf8', timeout: 60_000,
+    env: { ...process.env, PATH: `${dir}${path.delimiter}${process.env.PATH}` },
+  });
+  assert.equal(result.status, 1);
+  assert.match(result.stderr, /not empty/);
+  cleanup(dir);
+});
+
+test('the watcher needs search words', () => {
+  const result = spawnSync(process.execPath, ['scripts/watch.mjs', '--frames', '4'], { cwd: root, encoding: 'utf8', timeout: 60_000 });
+  assert.equal(result.status, 1);
+  assert.match(result.stderr, /missing search words/);
+});