From a4e55165b8617f7832d05824739d11fe78bfb20e Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Wed, 9 Sep 2026 15:13:54 -0500 Subject: [PATCH 1/5] ADFA-5552: Add in-place media optimizer for documentation.db A single, self-contained script that shrinks every image stored in a documentation.db in place - same Content.path, extension and contentTypeID, so every page reference keeps resolving. Reuses the optimization approach from the Kotlin website pipeline (pngquant + 500px downscale for PNG, JPEG re-encode, animated-GIF-aware resize, WebP re-encode, Scour for SVG) and applies it across all doc sets. Optimizes in place rather than converting formats: non-Kotlin pages reference media by absolute URL with no literal link to the stored path, so renaming (e.g. to WebP) can't be rewritten safely - keeping path and extension fixed avoids the problem entirely. Handles both plain-Brotli and shared-dictionary (CompressionDictionary) databases, reassembles chunked (>1MB) media before optimizing and re-chunks on write, and only ever rewrites a row when the result is smaller. Backs up first, runs in one transaction, VACUUMs, and supports --dry-run. Co-Authored-By: Claude Opus 4.8 --- scripts/optimize_db_media.py | 602 +++++++++++++++++++++++++++++++++++ 1 file changed, 602 insertions(+) create mode 100644 scripts/optimize_db_media.py diff --git a/scripts/optimize_db_media.py b/scripts/optimize_db_media.py new file mode 100644 index 00000000..df7f357b --- /dev/null +++ b/scripts/optimize_db_media.py @@ -0,0 +1,602 @@ +#!/usr/bin/env python3 +""" +optimize_db_media.py + +Applies the image optimization from PR #24 (the Kotlin website's +optimize_media.py) to *every* media file already stored in a +documentation.db, across all doc sets (a/, i/, k/, p/, j/, ...), optimizing +each image in place - same Content.path, same extension, same contentTypeID. + +Why in place (no format changes), unlike insert_optimized_media.py: + insert_optimized_media.py optimizes a *directory* of raw Kotlin media and + reinserts it, and with --webp it renames files (png -> webp, oversized svg + -> png), then rewrites the "/k/html/images/" references baked into + Kotlin's stored JSON pages. That rename+rewrite step only understands + Kotlin's page format. The other doc sets don't share it: an Android page, + for instance, references its media by absolute URL + ("https://developer.android.com/images/foo.gif"), with no literal link to + the stored path ("a/devsite/media/..._foo.gif") that a text substitution + could follow. So renaming media anywhere outside Kotlin can't be done + safely by this kind of tooling. Optimizing in place sidesteps the whole + problem: the path and extension never change, so every reference - however + it's written - keeps resolving to the same, now-smaller, file. That's why + this tool never converts to WEBP or rasterizes an SVG. + +What it does, per media Content row (inside one transaction, rolled back on +any error), mirroring optimize_media.py's own encoders and defaults: + - image/png : normalize mode, pngquant at full res, downscale to + --max-width (never upscaled), pngquant again. Stored raw (no compression). + - image/jpeg: downscale, re-encode JPEG (--jpeg-quality, progressive). Raw. + - image/gif : downscale; animated GIFs get every frame resized (frame + count/durations/loop preserved). Raw. + - image/webp: downscale, re-encode WEBP (--webp-quality). Kept as WEBP + (never a rename). Animated WEBP is left untouched. Brotli-compressed on + store, matching the ContentTypes row. + - image/svg+xml: Scour-optimized (metadata/comment/id stripping, numbers + rounded to --svg-precision) - but never rasterized to PNG, since that + would rename it. Brotli-compressed on store. + - Every other content type (video/mp4, image/x-icon, fonts, html, ...) is + left untouched. + +Each row's bytes are decompressed per its ContentTypes.compression ("brotli" +-> plain brotli, as this database stores it; "none" -> raw), optimized, then +recompressed the same way. A row is only rewritten when the result is +actually smaller than what's already stored - an image is never made larger. + +Reads/writes bytes straight from/to the DB; needs no source directory. Backs +up the database first (VACUUM INTO a timestamped sibling), VACUUMs at the end +to reclaim freed space, and supports --dry-run (do all the work, log what +would change, roll back). Requires the "pngquant" binary on PATH plus Pillow, +scour and brotli: + + uv run --with-requirements requirements.txt scripts/optimize_db_media.py documentation.db +""" +import argparse +import io +import re +import shutil +import subprocess +import sqlite3 +import sys +import tempfile +import time +from pathlib import Path + +import brotli +from PIL import Image + + +# --------------------------------------------------------------------------- +# Content-table chunking protocol (inlined so this optimizer is a single, +# self-contained file). Every tool that reads or rewrites a Content row has to +# agree with what WebServer.kt actually serves: read `path`; if that blob is +# exactly CHUNK_SIZE bytes, keep appending `path-1`, `path-2`, ... in suffix +# order, stopping at the first fragment shorter than CHUNK_SIZE (or the first +# one missing). Two rules fall out of that: +# 1. A "-" path is a continuation only if the base row is exactly +# CHUNK_SIZE bytes - the base merely existing proves nothing. +# 2. A short fragment (or a gap in the numbering) terminates the chain. +# Suffix discovery is deliberately suffix-agnostic (it does not assume the +# chain starts at -1), because real chains numbered from -2 exist in the +# production database (ADFA-5171). +# --------------------------------------------------------------------------- +CHUNK_SIZE = 1024 * 1024 # must match WebServer.kt's contentChunkSize exactly +_FRAGMENT_SUFFIX_RE = re.compile(r"^(.*)-(\d+)$") + + +def _split_fragment_path(path): + """("k/html/a.html", 2) for "k/html/a.html-2", or None if `path` has no + numeric "-" suffix. Purely syntactic.""" + match = _FRAGMENT_SUFFIX_RE.match(path) + return (match.group(1), int(match.group(2))) if match else None + + +def _is_chunked_base(lengths: dict, base_path: str) -> bool: + """Rule 1: whether `base_path` heads a chunked item, given a {path: length} + map. Anything shorter than CHUNK_SIZE owns no continuations.""" + return lengths.get(base_path) == CHUNK_SIZE + + +def is_continuation_path(lengths: dict, path: str) -> bool: + """Whether `path` is a continuation row of some chunked base rather than a + page of its own (rule 1 from the fragment's side) - what a scan over every + row needs to skip fragments without also skipping ordinary pages that + merely look like one.""" + split = _split_fragment_path(path) + if split is None: + return False + return _is_chunked_base(lengths, split[0]) + + +def _discover(conn, base_path: str): + """[(n, path, length)] for every existing "-" row, ordered by + N. The LIKE pattern over-matches (`_`/`%` are wildcards, `-%` doesn't + constrain the tail to digits); the regex re-check makes the result exact.""" + found = [] + for path, length in conn.execute( + "SELECT path, LENGTH(content) FROM Content WHERE path LIKE ?", (f"{base_path}-%",) + ).fetchall(): + split = _split_fragment_path(path) + if split is not None and split[0] == base_path: + found.append((split[1], path, length)) + found.sort(key=lambda item: item[0]) + return found + + +def owned_fragment_paths(conn, base_path: str, base_length: int = None) -> list: + """Every continuation row belonging to `base_path`, in suffix order - the + *ownership* answer (includes fragments past a short one, since a delete or + replace must take the whole tail rather than orphaning it). Empty unless + the base is genuinely chunked (rule 1).""" + if base_length is None: + row = conn.execute("SELECT LENGTH(content) FROM Content WHERE path = ?", (base_path,)).fetchone() + if row is None: + return [] + base_length = row[0] + if not _is_chunked_base({base_path: base_length}, base_path): + return [] + return [path for _n, path, _length in _discover(conn, base_path)] + + +def _served_fragment_paths(conn, base_path: str, base_length: int) -> list: + """The continuation rows WebServer.kt would actually concatenate: rule 1 to + decide there's a chain, then rule 2 - stop at the first fragment shorter + than CHUNK_SIZE and at the first gap in the numbering. Contiguity is + enforced from whatever suffix the chain begins at (not from 1), so an + ADFA-5171 chain numbered from -2 still reads whole. The *reassembly* + answer; use owned_fragment_paths when deleting or replacing instead.""" + if not _is_chunked_base({base_path: base_length}, base_path): + return [] + served = [] + expected = None + for number, path, length in _discover(conn, base_path): + if expected is not None and number != expected: + break # a gap: the server would have stopped at the missing suffix + served.append(path) + if length < CHUNK_SIZE: + break + expected = number + 1 + return served + + +def reassemble(conn, base_path: str, first_content: bytes) -> bytes: + """The full bytes the server would serve for `base_path`, given its + already-read base blob.""" + if len(first_content) < CHUNK_SIZE: + return first_content + parts = [first_content] + for path in _served_fragment_paths(conn, base_path, len(first_content)): + row = conn.execute("SELECT content FROM Content WHERE path = ?", (path,)).fetchone() + if row is None: # raced with a concurrent delete; serve what we have + break + parts.append(row[0]) + return b"".join(parts) + + +try: + RESAMPLE = Image.Resampling.LANCZOS +except AttributeError: # Pillow < 9.1 + RESAMPLE = Image.LANCZOS + +DEFAULTS = { + "max_width": 500, + "jpeg_quality": 82, + "webp_quality": 80, + "pngquant_speed": 4, + "svg_precision": 4, +} + + +# --- primitives, mirroring optimize_media.py --------------------------------- + +def find_pngquant() -> str: + path = shutil.which("pngquant") + if path is None: + raise RuntimeError("pngquant not found on PATH; install it (e.g. `brew install pngquant`) and retry") + return path + + +def quantize_png_bytes(data: bytes, pngquant_path: str, speed: int, name: str) -> bytes: + """pngquant over raw PNG bytes (stdin->stdout). Falls back to the input + bytes if pngquant declines (e.g. exit 99: would fall below the quality + floor) - a slightly larger PNG beats a broken one.""" + result = subprocess.run( + [pngquant_path, "--quality", "65-95", "--speed", str(speed), "--strip", "--force", "--output", "-", "-"], + input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + if result.returncode != 0 or not result.stdout: + return data + return result.stdout + + +def resize_if_needed(img: Image.Image, max_width: int) -> Image.Image: + if img.width <= max_width: + return img + new_height = max(1, round(img.height * (max_width / img.width))) + return img.resize((max_width, new_height), RESAMPLE) + + +def normalize_mode(img: Image.Image) -> Image.Image: + if img.mode == "P": + return img.convert("RGBA") if img.info.get("transparency") is not None else img.convert("RGB") + if img.mode == "CMYK": + return img.convert("RGB") + return img + + +# --- per-format, in-memory optimizers (bytes -> bytes, format preserved) ----- + +def optimize_png(data: bytes, pngquant_path: str, speed: int, max_width: int, name: str) -> bytes: + with Image.open(io.BytesIO(data)) as img: + if getattr(img, "is_animated", False): + # APNG: per-frame handling isn't implemented; leave it untouched + # rather than silently flattening the animation to one frame. + return data + img = normalize_mode(img) + # pngquant at full resolution first (its palette selection sees the + # original color detail), then resize, then pngquant again at the + # delivered size - the same two-pass shape optimize_media.py uses. + buf = io.BytesIO() + img.save(buf, "PNG", optimize=True) + quantized = quantize_png_bytes(buf.getvalue(), pngquant_path, speed, name) + img = Image.open(io.BytesIO(quantized)) + img.load() + img = normalize_mode(img) + img = resize_if_needed(img, max_width) + buf = io.BytesIO() + img.save(buf, "PNG", optimize=True) + return quantize_png_bytes(buf.getvalue(), pngquant_path, speed, name) + + +def optimize_jpeg(data: bytes, quality: int, max_width: int) -> bytes: + with Image.open(io.BytesIO(data)) as img: + img = normalize_mode(img) + img = resize_if_needed(img, max_width) + if img.mode != "RGB": + img = img.convert("RGB") + buf = io.BytesIO() + img.save(buf, "JPEG", quality=quality, optimize=True, progressive=True) + return buf.getvalue() + + +def optimize_gif(data: bytes, max_width: int) -> bytes: + with Image.open(io.BytesIO(data)) as img: + if getattr(img, "is_animated", False): + n_frames = getattr(img, "n_frames", 1) + loop = img.info.get("loop", 0) + frames, durations = [], [] + for i in range(n_frames): + img.seek(i) + frames.append(resize_if_needed(img.convert("RGBA"), max_width)) + durations.append(img.info.get("duration", 100)) + buf = io.BytesIO() + frames[0].save(buf, "GIF", save_all=True, append_images=frames[1:], duration=durations, + loop=loop, disposal=2, optimize=True) + return buf.getvalue() + img = resize_if_needed(img, max_width) + buf = io.BytesIO() + img.save(buf, "GIF", optimize=True) + return buf.getvalue() + + +def optimize_webp(data: bytes, quality: int, max_width: int) -> bytes: + with Image.open(io.BytesIO(data)) as img: + if getattr(img, "is_animated", False): + # Animated WEBP re-encoding isn't implemented (optimize_media.py + # doesn't attempt it either); leave it untouched. + return data + img = normalize_mode(img) + img = resize_if_needed(img, max_width) + buf = io.BytesIO() + img.save(buf, "WEBP", quality=quality, method=6) + return buf.getvalue() + + +def optimize_svg(data: bytes, precision: int) -> bytes: + """Scour-optimize, with optimize_media.py's own aggressive settings, but + never rasterize (that would rename the file).""" + from scour import scour + + options = scour.generateDefaultOptions() + options.remove_metadata = True + options.remove_descriptive_elements = True + options.remove_titles = True + options.remove_descriptions = True + options.strip_comments = True + options.strip_ids = True + options.shorten_ids = True + options.keep_editor_data = False + options.strip_xml_prolog = True + options.enable_viewboxing = True + options.simple_colors = True + options.style_to_xml = True + options.group_collapse = True + options.group_create = True + options.indent_type = "none" + options.newlines = False + options.digits = precision + + return scour.scourString(data.decode("utf-8"), options).encode("utf-8") + + +# Content-type value -> the optimizer to call. Anything not here is left +# untouched (video/mp4, image/x-icon, fonts, text, ...). +def build_optimizers(cfg: dict, pngquant_path: str) -> dict: + return { + "image/png": lambda d, name: optimize_png(d, pngquant_path, cfg["pngquant_speed"], cfg["max_width"], name), + "image/jpeg": lambda d, name: optimize_jpeg(d, cfg["jpeg_quality"], cfg["max_width"]), + "image/gif": lambda d, name: optimize_gif(d, cfg["max_width"]), + "image/webp": lambda d, name: optimize_webp(d, cfg["webp_quality"], cfg["max_width"]), + "image/svg+xml": lambda d, name: optimize_svg(d, cfg["svg_precision"]), + } + + +# --- DB plumbing ------------------------------------------------------------- + +def backup_database(db_path: Path) -> Path: + """VACUUM INTO a timestamped sibling, same approach as populate_db.py.""" + backup_path = db_path.with_name(f"{db_path.name}.backup-{time.strftime('%Y%m%d-%H%M%S')}") + conn = sqlite3.connect(db_path) + try: + conn.execute("VACUUM INTO ?", (str(backup_path),)) + finally: + conn.close() + return backup_path + + +class BrotliCodec: + """Brotli (de)compression for a Content row's "brotli" bytes. + + Two databases exist in this repo's lineage: older ones store plain Brotli + (handled by the Python `brotli` package), and ones populate_db.py has + touched store Brotli against a fixed 256 KiB shared dictionary from the + CompressionDictionary table (ADFA-5153). Those two streams are NOT + interchangeable, and the Python package has no dictionary parameter at + all, so a dictionary database is handled by shelling out to the `brotli` + CLI with `-D ` - exactly as populate_db.py's DictionaryCompressor + does. Every row must be decompressed with the same dictionary it was + compressed against, so the dictionary is read straight from the database + being optimized; there is no other copy to get out of sync with.""" + + def __init__(self, dictionary: bytes | None): + self._dictionary = dictionary + self._work_dir = None + if dictionary is not None: + self._brotli_path = shutil.which("brotli") + if self._brotli_path is None: + raise RuntimeError( + "this database dictionary-compresses its brotli content, which needs the `brotli` CLI " + "on PATH (e.g. `brew install brotli`); install it and retry" + ) + self._work_dir = Path(tempfile.mkdtemp(prefix="optimize_db_media_brotli_")) + self._dict_path = self._work_dir / "dictionary.bin" + self._dict_path.write_bytes(dictionary) + + def _run(self, *extra_args: str, data: bytes) -> bytes: + result = subprocess.run( + [self._brotli_path, "-D", str(self._dict_path), *extra_args, "-c"], + input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"brotli failed: {result.stderr.decode(errors='replace').strip()}") + return result.stdout + + def decompress(self, blob: bytes, compression: str) -> bytes: + if compression != "brotli": + return bytes(blob) + if self._dictionary is None: + return brotli.decompress(blob) + return self._run("-d", data=blob) + + def compress(self, data: bytes, compression: str) -> bytes: + if compression != "brotli": + return data + if self._dictionary is None: + return brotli.compress(data) + return self._run(data=data) + + def close(self) -> None: + if self._work_dir is not None: + shutil.rmtree(self._work_dir, ignore_errors=True) + + +def load_dictionary(conn) -> bytes | None: + """The shared Brotli dictionary bytes from CompressionDictionary if this + database has that table populated, else None (an older plain-Brotli + database). Never trains one - a database that needs a dictionary already + has the exact bytes every existing row was compressed against.""" + has_table = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='CompressionDictionary'" + ).fetchone() + if not has_table: + return None + row = conn.execute("SELECT data FROM CompressionDictionary WHERE id = 1").fetchone() + return row[0] if row else None + + +def human(n: int) -> str: + return f"{n:,}" + + +def write_media_row(conn, base_path: str, base_length: int, new_stored: bytes, + language_id: int, content_type_id: int) -> None: + """Replaces a media item's stored bytes, deleting whatever chunk chain it + used to have and re-chunking the new bytes if they still exceed + CHUNK_SIZE (they almost never do after downscaling, but a correct write + can't assume that). Fragments are written as a clean chain from -1, + matching what WebServer.kt probes for.""" + for fragment_path in owned_fragment_paths(conn, base_path, base_length): + conn.execute("DELETE FROM Content WHERE path = ?", (fragment_path,)) + + if len(new_stored) <= CHUNK_SIZE: + conn.execute("UPDATE Content SET content = ? WHERE path = ?", (new_stored, base_path)) + return + + # Rare: still over a chunk after optimizing. Split into CHUNK_SIZE pieces; + # the base holds the first, continuations are "-1", "-2", ... + chunks = [new_stored[i:i + CHUNK_SIZE] for i in range(0, len(new_stored), CHUNK_SIZE)] + conn.execute("UPDATE Content SET content = ? WHERE path = ?", (chunks[0], base_path)) + for number, chunk in enumerate(chunks[1:], start=1): + conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, 0)", + (f"{base_path}-{number}", language_id, chunk, content_type_id), + ) + + +def run(cfg: dict) -> int: + db_path = cfg["db_path"] + if not db_path.is_file(): + print(f"error: {db_path} does not exist", file=sys.stderr) + return 1 + + try: + pngquant_path = find_pngquant() + except RuntimeError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + optimizers = build_optimizers(cfg, pngquant_path) + + if cfg["dry_run"]: + print(f"Dry run: no backup, no changes committed. Optimizing media in {db_path} ...") + else: + print(f"Backing up {db_path} ...") + backup_path = backup_database(db_path) + print(f"Backup written to {backup_path}") + + conn = sqlite3.connect(db_path) + stats = {"optimized": 0, "no_gain": 0, "skipped_type": 0, "skipped_fragment": 0, "errors": 0, + "before": 0, "after": 0, "saved": 0} + per_type = {} + codec = None + try: + conn.execute("BEGIN") + # languageID/contentTypeID are needed only if a re-chunk write inserts + # new continuation rows; read the language once (this DB has one). + language_id = conn.execute("SELECT id FROM Languages LIMIT 1").fetchone()[0] + codec = BrotliCodec(load_dictionary(conn)) + if codec._dictionary is not None: + print(f"Using the database's {len(codec._dictionary):,}-byte shared Brotli dictionary " + "for image/svg+xml and image/webp rows.") + + rows = conn.execute( + "SELECT c.path, c.content, c.contentTypeID, ct.value, ct.compression " + "FROM Content c JOIN ContentTypes ct ON c.contentTypeID = ct.id " + "WHERE ct.value LIKE 'image/%' ORDER BY c.path" + ).fetchall() + # {path: stored length} over every image row, so a "-" row can + # be recognized as a continuation only when its base is exactly + # CHUNK_SIZE bytes (see content_chunking) rather than by its name alone. + lengths = {path: len(blob) for path, blob, _c, _v, _cmp in rows} + print(f"Scanning {len(rows)} image row(s) ({sum(1 for p in lengths if is_continuation_path(lengths, p))} " + "chunk-continuation row(s) will be folded into their base)...") + + for path, blob, content_type_id, value, compression in rows: + if is_continuation_path(lengths, path): + stats["skipped_fragment"] += 1 # handled as part of its base row + continue + + optimizer = optimizers.get(value) + if optimizer is None: + stats["skipped_type"] += 1 + continue + + name = path.rsplit("/", 1)[-1] + stored_full = reassemble(conn, path, blob) # base + its served chunks + original_stored = len(stored_full) + try: + media_bytes = codec.decompress(stored_full, compression) + new_media = optimizer(media_bytes, name) + new_stored = codec.compress(new_media, compression) + except Exception as exc: # noqa: BLE001 - one bad image is not the whole run + stats["errors"] += 1 + print(f" error: failed to optimize {path}: {exc}", file=sys.stderr) + continue + + stats["before"] += original_stored + if len(new_stored) < original_stored: + if not cfg["dry_run"]: + write_media_row(conn, path, len(blob), new_stored, language_id, content_type_id) + stats["optimized"] += 1 + stats["after"] += len(new_stored) + saved = original_stored - len(new_stored) + stats["saved"] += saved + bucket = per_type.setdefault(value, {"n": 0, "saved": 0}) + bucket["n"] += 1 + bucket["saved"] += saved + if cfg["verbose"]: + pct = saved / original_stored * 100 if original_stored else 0.0 + print(f" [OPT] {path}: {human(original_stored)} -> {human(len(new_stored))} " + f"(saved {human(saved)}, {pct:.1f}%)") + else: + stats["no_gain"] += 1 + stats["after"] += original_stored + + if cfg["dry_run"]: + conn.rollback() + else: + conn.commit() + except Exception: + conn.rollback() + raise + finally: + if codec is not None: + codec.close() + conn.close() + + if not cfg["dry_run"] and stats["optimized"]: + print("Vacuuming database to reclaim freed space...") + vac = sqlite3.connect(db_path) + try: + vac.execute("VACUUM") + finally: + vac.close() + + pct = stats["saved"] / stats["before"] * 100 if stats["before"] else 0.0 + verb = "would optimize" if cfg["dry_run"] else "optimized" + print() + print(f"{'Dry run complete. ' if cfg['dry_run'] else 'Done. '}" + f"{verb} {stats['optimized']} image(s); {stats['no_gain']} already minimal, " + f"{stats['skipped_type']} untouched (non-optimizable type), " + f"{stats['skipped_fragment']} chunk-fragment row(s) folded into their base, " + f"{stats['errors']} error(s).") + for value in sorted(per_type): + b = per_type[value] + print(f" {value}: {b['n']} optimized, saved {human(b['saved'])} bytes") + print(f"Image bytes {'that would go' if cfg['dry_run'] else 'gone'} from " + f"{human(stats['before'])} -> {human(stats['after'])} " + f"(saved {human(stats['saved'])}, {pct:.1f}%).") + return 1 if stats["errors"] else 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("db_path", type=Path, help="SQLite database to optimize in place, e.g. documentation.db") + p.add_argument("--dry-run", action="store_true", + help="Do all the work and report savings, then roll back without writing or backing up") + p.add_argument("--max-width", type=int, default=DEFAULTS["max_width"], + help=f"Max raster width in px, never upscaled (default: {DEFAULTS['max_width']})") + p.add_argument("--jpeg-quality", type=int, default=DEFAULTS["jpeg_quality"], + help=f"JPEG quality 0-95 (default: {DEFAULTS['jpeg_quality']})") + p.add_argument("--webp-quality", type=int, default=DEFAULTS["webp_quality"], + help=f"WEBP quality 0-100 (default: {DEFAULTS['webp_quality']})") + p.add_argument("--pngquant-speed", type=int, default=DEFAULTS["pngquant_speed"], + help=f"pngquant speed/quality 1(best)-11(rough) (default: {DEFAULTS['pngquant_speed']})") + p.add_argument("--svg-precision", type=int, default=DEFAULTS["svg_precision"], + help=f"Decimal places Scour rounds SVG numbers to (default: {DEFAULTS['svg_precision']})") + p.add_argument("--verbose", action="store_true", help="Log every optimized image, with byte sizes") + return p + + +def main() -> None: + args = build_parser().parse_args() + cfg = { + "db_path": args.db_path, "dry_run": args.dry_run, "max_width": args.max_width, + "jpeg_quality": args.jpeg_quality, "webp_quality": args.webp_quality, + "pngquant_speed": args.pngquant_speed, "svg_precision": args.svg_precision, "verbose": args.verbose, + } + sys.exit(run(cfg)) + + +if __name__ == "__main__": + main() From 2dc2484da2bcd4ae5819a0f6225832f2d4b45c78 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Wed, 9 Sep 2026 15:18:23 -0500 Subject: [PATCH 2/5] ADFA-5552: Make the media optimizer runnable one-shot via uv (PEP 723) Declare the Python dependencies (Pillow, scour, brotli) inline with PEP 723 script metadata so `uv run scripts/optimize_db_media.py ` installs them on the fly - no requirements.txt or --with flags needed. Add `from __future__ import annotations` so the `X | None` hints run on Python 3.9+, and update the usage docstring. (pngquant and brotli remain system CLIs the user installs.) Co-Authored-By: Claude Opus 4.8 --- scripts/optimize_db_media.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/scripts/optimize_db_media.py b/scripts/optimize_db_media.py index df7f357b..88ee3f3f 100644 --- a/scripts/optimize_db_media.py +++ b/scripts/optimize_db_media.py @@ -1,4 +1,12 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "scour", +# ] +# /// """ optimize_db_media.py @@ -46,11 +54,19 @@ Reads/writes bytes straight from/to the DB; needs no source directory. Backs up the database first (VACUUM INTO a timestamped sibling), VACUUMs at the end to reclaim freed space, and supports --dry-run (do all the work, log what -would change, roll back). Requires the "pngquant" binary on PATH plus Pillow, -scour and brotli: +would change, roll back). - uv run --with-requirements requirements.txt scripts/optimize_db_media.py documentation.db +Python dependencies (Pillow, scour, brotli) are declared inline above (PEP +723), so uv installs them on the fly - run it one-shot with no setup: + + uv run scripts/optimize_db_media.py documentation.db + +The "pngquant" and "brotli" command-line tools must also be on PATH (e.g. +`brew install pngquant brotli`); those are system binaries, not pip packages, +so uv can't provide them. """ +from __future__ import annotations + import argparse import io import re From 00759efa1329bd2ef366622320a5bb35542b23d3 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Wed, 9 Sep 2026 22:53:40 -0500 Subject: [PATCH 3/5] ADFA-5552: Add --webp conversion, reference updating, and review fixes optimize_db_media.py gains a --webp mode that converts stored media to WEBP under a new path/extension and deletes the old row, alongside the existing in-place mode. update_media_references.py then repairs the links that conversion leaves dangling, across every doc set. The reference updater matches on filename rather than stored path: conversion changes only the extension, and each doc set writes references differently (Android "media/static_..._foo.gif", IntelliJ "./images/foo.gif", Kotlin a JSON-escaped "/k/html/images/..."), with no single form that maps onto Content.path. Filenames are unambiguous here - no two images share a basename within a doc set, and no converted name collides with one left alone. The rename map is a diff against the pre-conversion copy (--before) rather than inferred from the converted database: inferring "every stored X.webp implies a converted X.png" produced 4530 candidate names where only 745 files had really converted, each spurious entry rewriting links to unrelated external assets. Fixes from the /code-review pass: - A file was renamed whenever its extension differed from its type's canonical one, so "photo.jpeg" became "photo.jpg" even in the default mode documented as never renaming. optimize_media now returns the content type it produced, so only a real format change renames anything. - Inserting a "-N" chunk row could hit an unrelated row of that name (content_chunking notes those are legal as separate pages) and abort the whole transaction on UNIQUE. clear_fragment_slots frees the slots first, logging what it removes. - target_path split the extension off the whole path, so a dotted directory with an extensionless file ("a/v1.2/logo") targeted "a/v1.webp". - Writes hardcoded templateId 0, against populate_db's convention that fragments reuse the base row's languageID/contentTypeID/templateId. - --before is validated as the same database lineage before anything is rewritten; a mistyped backup no longer silently rewrites 29k rows. - Reference matching is anchored to a URL-ish delimiter, so a filename in prose ("save it as copy.png") is no longer rewritten. - Images were decoded twice on the default path; the open image is now handed to the encoders. - Animated WEBP/APNG skips are counted separately from unhandled types, backup names take the next free suffix instead of failing on a same-second rerun, and flags that do nothing under --webp now warn. Measured on a 249MB database: 745 files converted, image bytes 130.2MB -> 60.6MB (-53.5%), whole file 249.1MB -> 178.4MB (-28.4%), then 1699 references rewritten across 578 rows. integrity_check/foreign_key_check pass, all 1192 images decode, no row links to a deleted filename, and templateId is preserved (3507 non-zero, unchanged). Co-Authored-By: Claude Opus 4.8 --- scripts/optimize_db_media.py | 548 +++++++++++++++++++++-------- scripts/update_media_references.py | 342 ++++++++++++++++++ 2 files changed, 734 insertions(+), 156 deletions(-) create mode 100644 scripts/update_media_references.py diff --git a/scripts/optimize_db_media.py b/scripts/optimize_db_media.py index 88ee3f3f..f2f17ff2 100644 --- a/scripts/optimize_db_media.py +++ b/scripts/optimize_db_media.py @@ -5,6 +5,7 @@ # "brotli", # "Pillow", # "scour", +# "cairosvg", # ] # /// """ @@ -12,44 +13,57 @@ Applies the image optimization from PR #24 (the Kotlin website's optimize_media.py) to *every* media file already stored in a -documentation.db, across all doc sets (a/, i/, k/, p/, j/, ...), optimizing -each image in place - same Content.path, same extension, same contentTypeID. - -Why in place (no format changes), unlike insert_optimized_media.py: - insert_optimized_media.py optimizes a *directory* of raw Kotlin media and - reinserts it, and with --webp it renames files (png -> webp, oversized svg - -> png), then rewrites the "/k/html/images/" references baked into - Kotlin's stored JSON pages. That rename+rewrite step only understands - Kotlin's page format. The other doc sets don't share it: an Android page, - for instance, references its media by absolute URL - ("https://developer.android.com/images/foo.gif"), with no literal link to - the stored path ("a/devsite/media/..._foo.gif") that a text substitution - could follow. So renaming media anywhere outside Kotlin can't be done - safely by this kind of tooling. Optimizing in place sidesteps the whole - problem: the path and extension never change, so every reference - however - it's written - keeps resolving to the same, now-smaller, file. That's why - this tool never converts to WEBP or rasterizes an SVG. +documentation.db, across all doc sets (a/, i/, k/, p/, j/, ...). + +Two modes: + + * Default - optimize each image **in place**: same Content.path, same + extension, same contentTypeID. Every reference to it keeps resolving, + whatever form that reference takes, because nothing is renamed. + + * --webp - **convert** formats, mirroring insert_optimized_media.py's own + --webp behavior: every static raster becomes WEBP, and an SVG still over + --svg-rasterize-threshold after minifying is rasterized to WEBP. A + converted file is written to a NEW path (same directory and stem, new + extension), its content type set to the format actually produced, and the + old row (plus its chunk chain) is deleted. + + NOTE: --webp does NOT rewrite the references that point at the old + filenames, so pages will link to names that no longer exist until those + references are fixed separately. insert_optimized_media.py can only do + that rewriting for Kotlin, whose pages embed a literal + "/k/html/images/"; the other doc sets don't share that form (an + Android page references media by absolute URL, e.g. + "https://developer.android.com/images/foo.gif", with no literal link to + the stored path "a/devsite/media/..._foo.gif"), so fixing them up is a + separate job this tool deliberately leaves alone. What it does, per media Content row (inside one transaction, rolled back on any error), mirroring optimize_media.py's own encoders and defaults: - image/png : normalize mode, pngquant at full res, downscale to --max-width (never upscaled), pngquant again. Stored raw (no compression). + With --webp: downscaled and re-encoded as WEBP instead (no pngquant, + which only makes sense for PNG output). - image/jpeg: downscale, re-encode JPEG (--jpeg-quality, progressive). Raw. - - image/gif : downscale; animated GIFs get every frame resized (frame - count/durations/loop preserved). Raw. - - image/webp: downscale, re-encode WEBP (--webp-quality). Kept as WEBP - (never a rename). Animated WEBP is left untouched. Brotli-compressed on - store, matching the ContentTypes row. + With --webp: WEBP instead. + - image/gif : downscale. Animated GIFs get every frame resized (frame + count/durations/loop preserved) and stay GIFs even under --webp, since + animated-WEBP encoding isn't implemented here. Raw. + - image/webp: downscale, re-encode WEBP (--webp-quality). Animated WEBP is + left untouched. Brotli-compressed on store, matching the ContentTypes row. - image/svg+xml: Scour-optimized (metadata/comment/id stripping, numbers - rounded to --svg-precision) - but never rasterized to PNG, since that - would rename it. Brotli-compressed on store. + rounded to --svg-precision). With --webp, one still over + --svg-rasterize-threshold is rasterized to WEBP; if that fails (e.g. no + native cairo for cairosvg), the optimized SVG is kept and a note logged. - Every other content type (video/mp4, image/x-icon, fonts, html, ...) is left untouched. Each row's bytes are decompressed per its ContentTypes.compression ("brotli" --> plain brotli, as this database stores it; "none" -> raw), optimized, then -recompressed the same way. A row is only rewritten when the result is -actually smaller than what's already stored - an image is never made larger. +-> plain, or against the shared CompressionDictionary when the database has +one; "none" -> raw), optimized, then compressed the way the *resulting* +format's content type says to. A row is only rewritten when the result is +actually smaller than what's already stored - an image is never made larger, +and never renamed for no benefit. Reads/writes bytes straight from/to the DB; needs no source directory. Backs up the database first (VACUUM INTO a timestamped sibling), VACUUMs at the end @@ -200,6 +214,7 @@ def reassemble(conn, base_path: str, first_content: bytes) -> bytes: "webp_quality": 80, "pngquant_speed": 4, "svg_precision": 4, + "svg_rasterize_threshold": 300 * 1024, # 300KB } @@ -242,75 +257,82 @@ def normalize_mode(img: Image.Image) -> Image.Image: # --- per-format, in-memory optimizers (bytes -> bytes, format preserved) ----- -def optimize_png(data: bytes, pngquant_path: str, speed: int, max_width: int, name: str) -> bytes: - with Image.open(io.BytesIO(data)) as img: - if getattr(img, "is_animated", False): - # APNG: per-frame handling isn't implemented; leave it untouched - # rather than silently flattening the animation to one frame. - return data - img = normalize_mode(img) - # pngquant at full resolution first (its palette selection sees the - # original color detail), then resize, then pngquant again at the - # delivered size - the same two-pass shape optimize_media.py uses. - buf = io.BytesIO() - img.save(buf, "PNG", optimize=True) - quantized = quantize_png_bytes(buf.getvalue(), pngquant_path, speed, name) - img = Image.open(io.BytesIO(quantized)) - img.load() - img = normalize_mode(img) - img = resize_if_needed(img, max_width) - buf = io.BytesIO() - img.save(buf, "PNG", optimize=True) - return quantize_png_bytes(buf.getvalue(), pngquant_path, speed, name) - - -def optimize_jpeg(data: bytes, quality: int, max_width: int) -> bytes: - with Image.open(io.BytesIO(data)) as img: - img = normalize_mode(img) - img = resize_if_needed(img, max_width) - if img.mode != "RGB": - img = img.convert("RGB") - buf = io.BytesIO() - img.save(buf, "JPEG", quality=quality, optimize=True, progressive=True) - return buf.getvalue() - - -def optimize_gif(data: bytes, max_width: int) -> bytes: - with Image.open(io.BytesIO(data)) as img: - if getattr(img, "is_animated", False): - n_frames = getattr(img, "n_frames", 1) - loop = img.info.get("loop", 0) - frames, durations = [], [] - for i in range(n_frames): - img.seek(i) - frames.append(resize_if_needed(img.convert("RGBA"), max_width)) - durations.append(img.info.get("duration", 100)) - buf = io.BytesIO() - frames[0].save(buf, "GIF", save_all=True, append_images=frames[1:], duration=durations, - loop=loop, disposal=2, optimize=True) - return buf.getvalue() - img = resize_if_needed(img, max_width) - buf = io.BytesIO() - img.save(buf, "GIF", optimize=True) - return buf.getvalue() - - -def optimize_webp(data: bytes, quality: int, max_width: int) -> bytes: - with Image.open(io.BytesIO(data)) as img: - if getattr(img, "is_animated", False): - # Animated WEBP re-encoding isn't implemented (optimize_media.py - # doesn't attempt it either); leave it untouched. - return data - img = normalize_mode(img) - img = resize_if_needed(img, max_width) - buf = io.BytesIO() - img.save(buf, "WEBP", quality=quality, method=6) - return buf.getvalue() +def encode_png(img: Image.Image, pngquant_path: str, speed: int, max_width: int, name: str) -> bytes: + """Quantize/resize an already-open static PNG. pngquant runs at full + resolution first (its palette selection sees the original color detail), + then the image is resized and quantized again at the delivered size - the + same two-pass shape optimize_media.py uses.""" + img = normalize_mode(img) + buf = io.BytesIO() + img.save(buf, "PNG", optimize=True) + quantized = quantize_png_bytes(buf.getvalue(), pngquant_path, speed, name) + img = Image.open(io.BytesIO(quantized)) + img.load() + img = normalize_mode(img) + img = resize_if_needed(img, max_width) + buf = io.BytesIO() + img.save(buf, "PNG", optimize=True) + return quantize_png_bytes(buf.getvalue(), pngquant_path, speed, name) + + +def encode_jpeg(img: Image.Image, quality: int, max_width: int) -> bytes: + img = normalize_mode(img) + img = resize_if_needed(img, max_width) + if img.mode != "RGB": + img = img.convert("RGB") + buf = io.BytesIO() + img.save(buf, "JPEG", quality=quality, optimize=True, progressive=True) + return buf.getvalue() + + +def encode_static_gif(img: Image.Image, max_width: int) -> bytes: + img = resize_if_needed(img, max_width) + buf = io.BytesIO() + img.save(buf, "GIF", optimize=True) + return buf.getvalue() + + +def optimize_animated_gif(img: Image.Image, max_width: int) -> bytes: + """Resizes every frame of an already-open animated GIF, preserving frame + count, each frame's own duration, and the loop count.""" + n_frames = getattr(img, "n_frames", 1) + loop = img.info.get("loop", 0) + frames, durations = [], [] + for i in range(n_frames): + img.seek(i) + frames.append(resize_if_needed(img.convert("RGBA"), max_width)) + durations.append(img.info.get("duration", 100)) + buf = io.BytesIO() + frames[0].save(buf, "GIF", save_all=True, append_images=frames[1:], duration=durations, + loop=loop, disposal=2, optimize=True) + return buf.getvalue() + + +def encode_webp(img: Image.Image, quality: int, max_width: int) -> bytes: + """Resize and encode any already-loaded image as WEBP.""" + img = normalize_mode(img) + img = resize_if_needed(img, max_width) + buf = io.BytesIO() + img.save(buf, "WEBP", quality=quality, method=6) + return buf.getvalue() + + +def rasterize_svg(svg_text: str, max_width: int) -> Image.Image: + """Renders SVG markup to a raster at exactly max_width px wide (cairosvg + derives the height from the SVG's own viewBox), for SVGs too large to keep + as vector. Imported lazily: cairosvg needs the native cairo library, and a + machine without it should fall back to keeping the optimized SVG rather + than failing the whole run.""" + import cairosvg + + png_bytes = cairosvg.svg2png(bytestring=svg_text.encode("utf-8"), output_width=max_width) + img = Image.open(io.BytesIO(png_bytes)) + img.load() + return img def optimize_svg(data: bytes, precision: int) -> bytes: - """Scour-optimize, with optimize_media.py's own aggressive settings, but - never rasterize (that would rename the file).""" + """Scour-optimize with optimize_media.py's own aggressive settings.""" from scour import scour options = scour.generateDefaultOptions() @@ -335,23 +357,113 @@ def optimize_svg(data: bytes, precision: int) -> bytes: return scour.scourString(data.decode("utf-8"), options).encode("utf-8") -# Content-type value -> the optimizer to call. Anything not here is left -# untouched (video/mp4, image/x-icon, fonts, text, ...). -def build_optimizers(cfg: dict, pngquant_path: str) -> dict: - return { - "image/png": lambda d, name: optimize_png(d, pngquant_path, cfg["pngquant_speed"], cfg["max_width"], name), - "image/jpeg": lambda d, name: optimize_jpeg(d, cfg["jpeg_quality"], cfg["max_width"]), - "image/gif": lambda d, name: optimize_gif(d, cfg["max_width"]), - "image/webp": lambda d, name: optimize_webp(d, cfg["webp_quality"], cfg["max_width"]), - "image/svg+xml": lambda d, name: optimize_svg(d, cfg["svg_precision"]), - } +WEBP_TYPE = "image/webp" +# Content types this tool knows how to optimize. Anything absent (video/mp4, +# image/x-icon, fonts, text, ...) is left untouched. The value is the extension +# a file of that type is *renamed to* when this run converts something into it - +# it is never used to "correct" a file already stored under that type, so a +# `.jpeg` stays `.jpeg`. +OPTIMIZABLE = { + "image/png": ".png", + "image/jpeg": ".jpg", + "image/gif": ".gif", + WEBP_TYPE: ".webp", + "image/svg+xml": ".svg", +} + + +def optimize_media(data: bytes, value: str, name: str, cfg: dict, pngquant_path: str, logger) -> tuple: + """Optimizes one media file's bytes. Returns (new_bytes, produced_type) - + the CONTENT TYPE actually produced, which differs from `value` only when + --webp converts a raster or rasterizes an oversized SVG - or None when this + file is deliberately left alone (animated WEBP/APNG). + + Returning the produced content type rather than a canonical extension is + what keeps the default mode's promise that nothing is renamed. Returning an + extension meant a file stored as "photo.jpeg" was compared against the + canonical ".jpg" for image/jpeg, came out "different", and was renamed to + photo.jpg - deleting the old row and breaking every link to it, in the mode + documented as never renaming anything. A type is equal to itself, so the + only thing that can now trigger a rename is a genuine format change.""" + if value == "image/svg+xml": + out = optimize_svg(data, cfg["svg_precision"]) + if cfg["webp"] and len(out) > cfg["svg_rasterize_threshold"]: + try: + img = rasterize_svg(out.decode("utf-8"), cfg["max_width"]) + try: + return encode_webp(img, cfg["webp_quality"], cfg["max_width"]), WEBP_TYPE + finally: + img.close() + except Exception as exc: # noqa: BLE001 - keep the vector instead + logger(f" note: could not rasterize {name} ({type(exc).__name__}); keeping optimized SVG") + return out, "image/svg+xml" + + # Opened once and handed to the per-format encoders: testing is_animated + # here and then letting each optimize_* re-open the same bytes decoded every + # image twice, on the slowest path in the tool. + with Image.open(io.BytesIO(data)) as img: + if getattr(img, "is_animated", False): + # Animated GIFs are resized frame by frame and stay GIFs; every + # other animated format (APNG, animated WEBP) is left untouched, + # since re-encoding those animations isn't implemented and + # flattening them to one frame would silently break them. + if value == "image/gif": + return optimize_animated_gif(img, cfg["max_width"]), "image/gif" + return None + if cfg["webp"]: + # --webp: every static raster becomes WEBP regardless of its source + # format (pngquant is skipped - it only makes sense for PNG output). + return encode_webp(img, cfg["webp_quality"], cfg["max_width"]), WEBP_TYPE + if value == "image/png": + return encode_png(img, pngquant_path, cfg["pngquant_speed"], cfg["max_width"], name), "image/png" + if value == "image/jpeg": + return encode_jpeg(img, cfg["jpeg_quality"], cfg["max_width"]), "image/jpeg" + if value == "image/gif": + return encode_static_gif(img, cfg["max_width"]), "image/gif" + return encode_webp(img, cfg["webp_quality"], cfg["max_width"]), WEBP_TYPE + + +def target_path(old_path: str, new_ext: str, claimed: set) -> str: + """The Content.path a converted file should be stored under: same directory + and stem, new extension. Content.path is UNIQUE, so a target that's already + spoken for - by a row that's staying put (an existing foo.webp), or by + another file converting to the same name (foo.png and foo.jpg both wanting + foo.webp) - is disambiguated by folding the source extension into the stem. + + The extension is split off the FILENAME, not the whole path: rpartition(".") + over the full path finds a dot in a *directory* when the filename has no + extension of its own, so "a/v1.2/logo" yielded the target "a/v1.webp" - + a row in the wrong directory, while the real one was deleted.""" + directory, slash, filename = old_path.rpartition("/") + stem, dot, old_ext = filename.rpartition(".") + if not dot: # no extension to replace + return old_path + prefix = f"{directory}{slash}{stem}" + candidate = f"{prefix}{new_ext}" + if candidate.lower() not in claimed: + return candidate + candidate = f"{prefix}-{old_ext.lower()}{new_ext}" + attempt = 1 + while candidate.lower() in claimed: + attempt += 1 + candidate = f"{prefix}-{old_ext.lower()}-{attempt}{new_ext}" + return candidate # --- DB plumbing ------------------------------------------------------------- def backup_database(db_path: Path) -> Path: - """VACUUM INTO a timestamped sibling, same approach as populate_db.py.""" - backup_path = db_path.with_name(f"{db_path.name}.backup-{time.strftime('%Y%m%d-%H%M%S')}") + """VACUUM INTO a timestamped sibling, same approach as populate_db.py. + + VACUUM INTO refuses to write a file that already exists, so a second- + resolution name made two runs inside the same second fail on the backup + before doing any work. Take the next free suffix instead of dying.""" + stamp = time.strftime("%Y%m%d-%H%M%S") + backup_path = db_path.with_name(f"{db_path.name}.backup-{stamp}") + attempt = 1 + while backup_path.exists(): + attempt += 1 + backup_path = db_path.with_name(f"{db_path.name}.backup-{stamp}-{attempt}") conn = sqlite3.connect(db_path) try: conn.execute("VACUUM INTO ?", (str(backup_path),)) @@ -434,44 +546,114 @@ def human(n: int) -> str: return f"{n:,}" -def write_media_row(conn, base_path: str, base_length: int, new_stored: bytes, - language_id: int, content_type_id: int) -> None: - """Replaces a media item's stored bytes, deleting whatever chunk chain it - used to have and re-chunking the new bytes if they still exceed - CHUNK_SIZE (they almost never do after downscaling, but a correct write - can't assume that). Fragments are written as a clean chain from -1, - matching what WebServer.kt probes for.""" +def delete_media_row(conn, base_path: str, base_length: int) -> None: + """Removes a media item entirely: its base row and every continuation row + it owns (see owned_fragment_paths - ownership, not just what the server + would serve, so a gapped chain's tail isn't orphaned).""" for fragment_path in owned_fragment_paths(conn, base_path, base_length): conn.execute("DELETE FROM Content WHERE path = ?", (fragment_path,)) - - if len(new_stored) <= CHUNK_SIZE: - conn.execute("UPDATE Content SET content = ? WHERE path = ?", (new_stored, base_path)) - return - - # Rare: still over a chunk after optimizing. Split into CHUNK_SIZE pieces; - # the base holds the first, continuations are "-1", "-2", ... - chunks = [new_stored[i:i + CHUNK_SIZE] for i in range(0, len(new_stored), CHUNK_SIZE)] - conn.execute("UPDATE Content SET content = ? WHERE path = ?", (chunks[0], base_path)) + conn.execute("DELETE FROM Content WHERE path = ?", (base_path,)) + + +def clear_fragment_slots(conn, base_path: str, count: int, logger) -> None: + """Frees the "-1..count" paths this write is about to insert. + + They are normally already gone (delete_media_row / owned_fragment_paths took + the old chain), but a row can sit at one of those names without being an + owned fragment: content_chunking notes "guide.html" and "guide.html-1" are + legal as two unrelated pages, and owned_fragment_paths deliberately returns + nothing when the base was not previously CHUNK_SIZE. Inserting over that row + raised a UNIQUE violation which propagated out and rolled back the ENTIRE + run - every image optimized so far, lost to one filename coincidence. + + Such a row is unreachable anyway once this item is chunked: the server + concatenates "-1", "-2", ... to serve base_path, so it would be read + as this file's bytes, not as itself. Removing it is what makes the database + consistent; it is logged because it is destructive.""" + for number in range(1, count + 1): + fragment_path = f"{base_path}-{number}" + if conn.execute("SELECT 1 FROM Content WHERE path = ?", (fragment_path,)).fetchone(): + logger(f" warning: removing existing row at {fragment_path}; it collides with a chunk " + f"continuation of {base_path} and the server would serve it as part of that file") + conn.execute("DELETE FROM Content WHERE path = ?", (fragment_path,)) + + +def write_chunks(conn, path: str, stored: bytes, language_id: int, content_type_id: int, + template_id: int, logger, update_base: bool) -> None: + """Writes `stored` at `path`, splitting anything over CHUNK_SIZE across + "-1", "-2", ... Every fragment reuses the base row's languageID, + contentTypeID and templateId, matching populate_db.insert_chunked_content's + documented convention ("Every fragment reuses the first row's + languageID/contentTypeID/templateId for consistency").""" + chunks = [stored[i:i + CHUNK_SIZE] for i in range(0, len(stored), CHUNK_SIZE)] or [b""] + clear_fragment_slots(conn, path, len(chunks) - 1, logger) + if update_base: + conn.execute("UPDATE Content SET content = ?, contentTypeID = ? WHERE path = ?", + (chunks[0], content_type_id, path)) + else: + conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, ?)", + (path, language_id, chunks[0], content_type_id, template_id), + ) for number, chunk in enumerate(chunks[1:], start=1): conn.execute( - "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, 0)", - (f"{base_path}-{number}", language_id, chunk, content_type_id), + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, ?)", + (f"{path}-{number}", language_id, chunk, content_type_id, template_id), ) +def convert_media_row(conn, old_path: str, old_base_length: int, new_path: str, new_stored: bytes, + language_id: int, new_content_type_id: int, template_id: int, logger) -> None: + """Replaces a media item with a converted one stored under a different + path/extension: the old row (and its chunk chain) is deleted and the new + bytes are inserted at the new path with the new content type. Deleting + first keeps Content.path's UNIQUE constraint satisfied even in the corner + case where the new path equals some other row this run already removed.""" + delete_media_row(conn, old_path, old_base_length) + write_chunks(conn, new_path, new_stored, language_id, new_content_type_id, template_id, + logger, update_base=False) + + +def write_media_row(conn, base_path: str, base_length: int, new_stored: bytes, + language_id: int, content_type_id: int, template_id: int, logger) -> None: + """Replaces a media item's stored bytes in place, deleting whatever chunk + chain it used to have and re-chunking the new bytes if they still exceed + CHUNK_SIZE (they almost never do after downscaling, but a correct write + can't assume that).""" + for fragment_path in owned_fragment_paths(conn, base_path, base_length): + conn.execute("DELETE FROM Content WHERE path = ?", (fragment_path,)) + write_chunks(conn, base_path, new_stored, language_id, content_type_id, template_id, + logger, update_base=True) + + def run(cfg: dict) -> int: db_path = cfg["db_path"] if not db_path.is_file(): print(f"error: {db_path} does not exist", file=sys.stderr) return 1 - try: - pngquant_path = find_pngquant() - except RuntimeError as exc: - print(f"error: {exc}", file=sys.stderr) - return 1 - - optimizers = build_optimizers(cfg, pngquant_path) + # pngquant is only ever invoked to quantize PNG *output*. Under --webp no + # PNG is written at all (every static raster becomes WEBP), so requiring + # the binary up front would refuse a run that never needs it. + if cfg["webp"]: + # Under --webp no PNG or JPEG is ever written, so these tuning knobs + # silently do nothing; say so rather than letting someone believe a + # quality setting took effect. + ignored = [flag for flag, key, default in + (("--pngquant-speed", "pngquant_speed", DEFAULTS["pngquant_speed"]), + ("--jpeg-quality", "jpeg_quality", DEFAULTS["jpeg_quality"])) + if cfg[key] != default] + if ignored: + print(f"warning: {', '.join(ignored)} has no effect with --webp (no PNG/JPEG is written); " + "use --webp-quality instead", file=sys.stderr) + + pngquant_path = None + if not cfg["webp"]: + try: + pngquant_path = find_pngquant() + except RuntimeError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 if cfg["dry_run"]: print(f"Dry run: no backup, no changes committed. Optimizing media in {db_path} ...") @@ -481,8 +663,8 @@ def run(cfg: dict) -> int: print(f"Backup written to {backup_path}") conn = sqlite3.connect(db_path) - stats = {"optimized": 0, "no_gain": 0, "skipped_type": 0, "skipped_fragment": 0, "errors": 0, - "before": 0, "after": 0, "saved": 0} + stats = {"optimized": 0, "converted": 0, "no_gain": 0, "skipped_type": 0, "skipped_animated": 0, + "skipped_fragment": 0, "errors": 0, "before": 0, "after": 0, "saved": 0} per_type = {} codec = None try: @@ -490,30 +672,47 @@ def run(cfg: dict) -> int: # languageID/contentTypeID are needed only if a re-chunk write inserts # new continuation rows; read the language once (this DB has one). language_id = conn.execute("SELECT id FROM Languages LIMIT 1").fetchone()[0] + # Content type id + compression for every format a conversion can + # produce, looked up once. Missing here is fatal for that format only + # when something actually converts to it (checked at use). + type_info = {} + for type_value in OPTIMIZABLE: + row = conn.execute("SELECT id, compression FROM ContentTypes WHERE value = ?", (type_value,)).fetchone() + if row: + type_info[type_value] = (row[0], row[1]) + if cfg["webp"] and WEBP_TYPE not in type_info: + print("error: --webp needs an 'image/webp' row in ContentTypes; this database has none", + file=sys.stderr) + conn.rollback() + return 1 + # Every path already in use, so a converted file never collides with a + # row that's staying put. Casefolded: two paths differing only in case + # would still be one UNIQUE key clash risk on a case-insensitive read. + claimed = {row[0].lower() for row in conn.execute("SELECT path FROM Content")} codec = BrotliCodec(load_dictionary(conn)) if codec._dictionary is not None: print(f"Using the database's {len(codec._dictionary):,}-byte shared Brotli dictionary " "for image/svg+xml and image/webp rows.") rows = conn.execute( - "SELECT c.path, c.content, c.contentTypeID, ct.value, ct.compression " + "SELECT c.path, c.content, c.contentTypeID, ct.value, ct.compression, c.templateId " "FROM Content c JOIN ContentTypes ct ON c.contentTypeID = ct.id " "WHERE ct.value LIKE 'image/%' ORDER BY c.path" ).fetchall() # {path: stored length} over every image row, so a "-" row can # be recognized as a continuation only when its base is exactly # CHUNK_SIZE bytes (see content_chunking) rather than by its name alone. - lengths = {path: len(blob) for path, blob, _c, _v, _cmp in rows} + lengths = {path: len(blob) for path, blob, _c, _v, _cmp, _t in rows} print(f"Scanning {len(rows)} image row(s) ({sum(1 for p in lengths if is_continuation_path(lengths, p))} " "chunk-continuation row(s) will be folded into their base)...") + warn = lambda message: print(message, file=sys.stderr) # noqa: E731 - for path, blob, content_type_id, value, compression in rows: + for path, blob, content_type_id, value, compression, template_id in rows: if is_continuation_path(lengths, path): stats["skipped_fragment"] += 1 # handled as part of its base row continue - optimizer = optimizers.get(value) - if optimizer is None: + if value not in OPTIMIZABLE: stats["skipped_type"] += 1 continue @@ -522,31 +721,57 @@ def run(cfg: dict) -> int: original_stored = len(stored_full) try: media_bytes = codec.decompress(stored_full, compression) - new_media = optimizer(media_bytes, name) - new_stored = codec.compress(new_media, compression) + result = optimize_media(media_bytes, value, name, cfg, pngquant_path, warn) + if result is None: # animated WEBP/APNG: deliberately left alone + stats["skipped_animated"] += 1 + continue + new_media, new_type = result + # A converted file is stored under the content type of the + # format actually produced, and compressed the way *that* type + # says to - not however the source happened to be stored. + new_type_id, new_compression = type_info.get(new_type, (content_type_id, compression)) + new_stored = codec.compress(new_media, new_compression) except Exception as exc: # noqa: BLE001 - one bad image is not the whole run stats["errors"] += 1 print(f" error: failed to optimize {path}: {exc}", file=sys.stderr) continue stats["before"] += original_stored - if len(new_stored) < original_stored: - if not cfg["dry_run"]: - write_media_row(conn, path, len(blob), new_stored, language_id, content_type_id) - stats["optimized"] += 1 - stats["after"] += len(new_stored) - saved = original_stored - len(new_stored) - stats["saved"] += saved - bucket = per_type.setdefault(value, {"n": 0, "saved": 0}) - bucket["n"] += 1 - bucket["saved"] += saved - if cfg["verbose"]: - pct = saved / original_stored * 100 if original_stored else 0.0 - print(f" [OPT] {path}: {human(original_stored)} -> {human(len(new_stored))} " - f"(saved {human(saved)}, {pct:.1f}%)") - else: + if len(new_stored) >= original_stored: + # Never grow a file, and never rename one for no benefit. stats["no_gain"] += 1 stats["after"] += original_stored + continue + + # Only a genuine format change renames anything: a file already + # stored under the type it was re-encoded to keeps its own path, + # whatever extension that path happens to use. + new_path = path if new_type == value else target_path(path, OPTIMIZABLE[new_type], claimed) + converted = new_path != path + if not cfg["dry_run"]: + if converted: + convert_media_row(conn, path, len(blob), new_path, new_stored, language_id, + new_type_id, template_id, warn) + else: + write_media_row(conn, path, len(blob), new_stored, language_id, new_type_id, + template_id, warn) + if converted: + claimed.add(new_path.lower()) + claimed.discard(path.lower()) + stats["converted"] += 1 + stats["optimized"] += 1 + stats["after"] += len(new_stored) + saved = original_stored - len(new_stored) + stats["saved"] += saved + bucket = per_type.setdefault(value, {"n": 0, "saved": 0, "converted": 0}) + bucket["n"] += 1 + bucket["saved"] += saved + bucket["converted"] += 1 if converted else 0 + if cfg["verbose"]: + pct = saved / original_stored * 100 if original_stored else 0.0 + arrow = f" -> {new_path}" if converted else "" + print(f" [OPT] {path}{arrow}: {human(original_stored)} -> {human(len(new_stored))} " + f"(saved {human(saved)}, {pct:.1f}%)") if cfg["dry_run"]: conn.rollback() @@ -572,13 +797,15 @@ def run(cfg: dict) -> int: verb = "would optimize" if cfg["dry_run"] else "optimized" print() print(f"{'Dry run complete. ' if cfg['dry_run'] else 'Done. '}" - f"{verb} {stats['optimized']} image(s); {stats['no_gain']} already minimal, " + f"{verb} {stats['optimized']} image(s), of which {stats['converted']} changed format " + f"(new extension, old row deleted); {stats['no_gain']} already minimal, " f"{stats['skipped_type']} untouched (non-optimizable type), " + f"{stats['skipped_animated']} animated WEBP/APNG left alone, " f"{stats['skipped_fragment']} chunk-fragment row(s) folded into their base, " f"{stats['errors']} error(s).") for value in sorted(per_type): b = per_type[value] - print(f" {value}: {b['n']} optimized, saved {human(b['saved'])} bytes") + print(f" {value}: {b['n']} optimized ({b['converted']} converted), saved {human(b['saved'])} bytes") print(f"Image bytes {'that would go' if cfg['dry_run'] else 'gone'} from " f"{human(stats['before'])} -> {human(stats['after'])} " f"(saved {human(stats['saved'])}, {pct:.1f}%).") @@ -590,6 +817,14 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("db_path", type=Path, help="SQLite database to optimize in place, e.g. documentation.db") p.add_argument("--dry-run", action="store_true", help="Do all the work and report savings, then roll back without writing or backing up") + p.add_argument("--webp", action="store_true", + help="Convert every static raster (PNG/JPEG/GIF) to WEBP, and rasterize an SVG still over " + "--svg-rasterize-threshold after minifying. Converted files are stored under a new " + "path ending .webp and their old row is deleted; references to the old name are NOT " + "rewritten") + p.add_argument("--svg-rasterize-threshold", type=int, default=DEFAULTS["svg_rasterize_threshold"], + help="With --webp, rasterize a minified SVG still larger than this many bytes " + f"(default: {DEFAULTS['svg_rasterize_threshold']:,})") p.add_argument("--max-width", type=int, default=DEFAULTS["max_width"], help=f"Max raster width in px, never upscaled (default: {DEFAULTS['max_width']})") p.add_argument("--jpeg-quality", type=int, default=DEFAULTS["jpeg_quality"], @@ -610,6 +845,7 @@ def main() -> None: "db_path": args.db_path, "dry_run": args.dry_run, "max_width": args.max_width, "jpeg_quality": args.jpeg_quality, "webp_quality": args.webp_quality, "pngquant_speed": args.pngquant_speed, "svg_precision": args.svg_precision, "verbose": args.verbose, + "webp": args.webp, "svg_rasterize_threshold": args.svg_rasterize_threshold, } sys.exit(run(cfg)) diff --git a/scripts/update_media_references.py b/scripts/update_media_references.py new file mode 100644 index 00000000..5759c864 --- /dev/null +++ b/scripts/update_media_references.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = [ +# "brotli", +# "Pillow", +# "scour", +# "cairosvg", +# ] +# /// +""" +update_media_references.py + +Fixes up the references left dangling by optimize_db_media.py --webp, which +converts stored media to WEBP under a new path/extension and deletes the old +row without touching the pages that link to it. This script rewrites those +links, across every doc set, so pages stop pointing at filenames that no +longer exist. + +How the rename map is derived - by diffing this database against the +pre-conversion copy given as --before (the backup optimize_db_media.py makes +for itself): an image path present there and gone here, whose directory and +stem now hold a file with a different extension, was converted, and only +those names are rewritten. + +That diff is deliberate rather than inferring renames from this database +alone. Inference - "for every stored .webp, treat .png as a name +that must have converted into it" - looks self-contained but invents renames +for images that were ALWAYS WEBP: measured against a real database it +produced 4530 candidate names where only 745 files had actually been +converted, and each spurious entry would rewrite references to an unrelated +asset this database never stored (an external "/images/foo.png", say) into a +link to a WEBP that has nothing to do with it. The diff cannot make that +mistake, because a name only enters the map when a file really did leave. + +Why matching on the filename is safe here, rather than on the full stored +path: the conversion changes only the extension, and references are written +in whatever form each doc set happens to use - root-relative +("/images/foo.png"), relative ("media/static_..._foo.gif", +"./images/foo.gif"), or absolute URL - with no single form that maps onto +Content.path. The filename is the one component every form shares. It's +unambiguous in this database because no two images share a basename within a +doc set, and no converted basename collides with an image that was left +alone (both verified before this was written). Matches are still anchored so +a filename only counts when it appears as a reference - not as a substring +of a longer name ("my_copy.png" when rewriting "copy.png") and not with +trailing junk. + +Runs in one transaction (rolled back on any error), backs the database up +first, and VACUUMs afterwards. --dry-run does all the work and reports what +would change without writing. Decompression/recompression is parallelised +because a dictionary-compressed database shells out to the brotli CLI per +row, which is where the time goes. + + uv run scripts/update_media_references.py documentation.db --dry-run +""" +from __future__ import annotations + +import argparse +import re +import sqlite3 +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from optimize_db_media import ( + BrotliCodec, CHUNK_SIZE, backup_database, clear_fragment_slots, is_continuation_path, + load_dictionary, owned_fragment_paths, reassemble, +) + +# Content types whose stored text can carry a link to a media file. +TEXT_TYPES = ("text/html", "application/json", "text/css", "text/markdown", + "text/javascript", "text/plain", "application/xml") +# A "-" chunk continuation row, which vanishes with its base rather +# than being renamed. +_FRAGMENT_RE = re.compile(r"-\d+$") + + +def image_paths(conn) -> set: + return { + row[0] for row in conn.execute( + "SELECT c.path FROM Content c JOIN ContentTypes ct ON c.contentTypeID = ct.id " + "WHERE ct.value LIKE 'image/%'" + ) + } + + +def check_same_lineage(conn, before_conn) -> str: + """Returns a complaint if --before doesn't look like a pre-conversion copy + of this same database, or "" if it does. + + Cheap insurance against a mistyped or tab-completed path: several + same-named backups sit side by side, and pointing this at the wrong one + yields a plausible-looking rename map that rewrites thousands of pages to + filenames that never existed - while reporting success. Two markers that + the conversion cannot change: the shared Brotli dictionary bytes, and the + set of non-media paths (converting media never adds or removes a page).""" + if load_dictionary(conn) != load_dictionary(before_conn): + return "their CompressionDictionary bytes differ" + + def pages(c): + return {r[0] for r in c.execute( + "SELECT c.path FROM Content c JOIN ContentTypes ct ON c.contentTypeID = ct.id " + "WHERE ct.value NOT LIKE 'image/%'")} + + here, there = pages(conn), pages(before_conn) + if not there: + return "it contains no non-image content at all" + drift = len(here ^ there) / len(there) + if drift > 0.01: + return (f"{len(here ^ there)} of {len(there)} non-image paths differ ({drift:.0%}); " + "converting media does not add or remove pages") + return "" + + +def build_rename_map(conn, before_conn, logger=print) -> dict: + """{old_basename: new_basename} for every image that left `before_conn` + and came back under a different extension here. See the module docstring + for why this is a diff rather than an inference. + + Chunk continuation rows ("-2") are skipped: they disappear because + their base was rewritten, which is a deletion, not a rename. Anything + whose stem gained no replacement, or gained an ambiguous one, is reported + and left out rather than guessed at.""" + before, after = image_paths(before_conn), image_paths(conn) + gone = {p for p in before - after if not _FRAGMENT_RE.search(p)} + by_stem = {} + for path in after - before: + by_stem.setdefault(path.rsplit(".", 1)[0], []).append(path) + + renames, unresolved = {}, [] + for old_path in sorted(gone): + candidates = by_stem.get(old_path.rsplit(".", 1)[0], []) + if len(candidates) != 1: + unresolved.append(old_path) + continue + old_name = old_path.rsplit("/", 1)[-1] + new_name = candidates[0].rsplit("/", 1)[-1] + # Two doc sets can hold same-named files; both convert to the same new + # name, so an identical mapping is agreement, not a conflict. + if renames.get(old_name, new_name) != new_name: + unresolved.append(old_path) + continue + renames[old_name] = new_name + if unresolved: + logger(f"note: {len(unresolved)} removed image(s) had no unambiguous replacement and are " + f"not being rewritten (e.g. {unresolved[0]})") + return renames + + +def build_pattern(renames: dict) -> re.Pattern: + """One alternation over every old filename, anchored so it only matches a + filename used as a reference: not preceded by a character that would make + it the tail of a longer name (so "copy.png" never matches inside + "my_copy.png"), and not followed by one that would make it a prefix of + something else. Longest names first so an alternation never settles for a + shorter overlapping match.""" + names = sorted(renames, key=len, reverse=True) + body = "|".join(re.escape(n) for n in names) + # The character before must mark the start of a URL or attribute value - + # a path separator, a quote (JSON-escaped or not), "(" for CSS url()/ + # markdown, or "=" for an unquoted attribute. Requiring one keeps a + # filename mentioned in prose or a code sample ("save it as copy.png") + # from being silently rewritten; 39 of the converted names are short and + # generic enough for that to be a real risk. + return re.compile(rf"(?<=[/\"'(=])({body})(?![A-Za-z0-9_])") + + +def rewrite_text(text: str, pattern: re.Pattern, renames: dict) -> tuple: + """Returns (new_text, number_of_replacements) - a single pass over the + original text, so a replacement can never be re-matched by another.""" + count = 0 + + def repl(match): + nonlocal count + count += 1 + return renames[match.group(1)] + + return pattern.sub(repl, text), count + + +def replace_row(conn, path: str, base_length: int, stored: bytes, language_id: int, + content_type_id: int, template_id: int, logger=print) -> None: + """Rewrites one row's stored bytes in place, re-chunking as needed and + preserving its language, content type and templateId (Kotlin's templated + pages carry a non-zero templateId; losing it would stop them rendering).""" + for fragment_path in owned_fragment_paths(conn, path, base_length): + conn.execute("DELETE FROM Content WHERE path = ?", (fragment_path,)) + chunks = [stored[i:i + CHUNK_SIZE] for i in range(0, len(stored), CHUNK_SIZE)] or [b""] + # A rewritten page can cross CHUNK_SIZE when it previously did not, and the + # "-1" slot it then needs may hold an unrelated row that + # owned_fragment_paths never claimed - inserting over it would raise UNIQUE + # and roll back the whole run. See clear_fragment_slots. + clear_fragment_slots(conn, path, len(chunks) - 1, logger) + conn.execute("UPDATE Content SET content = ? WHERE path = ?", (chunks[0], path)) + for number, chunk in enumerate(chunks[1:], start=1): + conn.execute( + "INSERT INTO Content (path, languageID, content, contentTypeID, templateId) VALUES (?, ?, ?, ?, ?)", + (f"{path}-{number}", language_id, chunk, content_type_id, template_id), + ) + + +def run(cfg: dict) -> int: + db_path = cfg["db_path"] + if not db_path.is_file(): + print(f"error: {db_path} does not exist", file=sys.stderr) + return 1 + + before_path = cfg["before"] + if not before_path.is_file(): + print(f"error: --before database {before_path} does not exist", file=sys.stderr) + return 1 + + conn = sqlite3.connect(db_path) + before_conn = sqlite3.connect(before_path) + codec = None + try: + complaint = check_same_lineage(conn, before_conn) + if complaint: + print(f"error: {before_path} does not look like a pre-conversion copy of {db_path}: " + f"{complaint}. Refusing to rewrite references off an unrelated database.", + file=sys.stderr) + return 1 + codec = BrotliCodec(load_dictionary(conn)) + renames = build_rename_map(conn, before_conn) + if not renames: + print("No converted media found - nothing to rewrite.") + return 0 + print(f"Found {len(renames)} converted filename(s) to rewrite references for.") + if cfg["verbose"]: + for old in sorted(renames)[:10]: + print(f" {old} -> {renames[old]}") + pattern = build_pattern(renames) + + placeholders = ",".join("?" * len(TEXT_TYPES)) + rows = conn.execute( + f"SELECT c.path, c.content, c.languageID, c.contentTypeID, c.templateId, ct.compression " + f"FROM Content c JOIN ContentTypes ct ON c.contentTypeID = ct.id " + f"WHERE ct.value IN ({placeholders}) ORDER BY c.path", TEXT_TYPES + ).fetchall() + lengths = {row[0]: len(row[1]) for row in rows} + bases = [r for r in rows if not is_continuation_path(lengths, r[0])] + print(f"Scanning {len(bases)} text row(s) for references " + f"({len(rows) - len(bases)} chunk-continuation row(s) folded into their base)...") + + # Reassembly is done here, on the main thread, and the whole stored blob + # handed to the workers: a sqlite3 connection may only be used from the + # thread that created it, and reassemble() queries for continuation + # rows - so calling it inside the pool blew up on precisely the large, + # chunked pages this most needs to handle (j/html/api/index-all.html). + # The DB work is cheap anyway; the time goes on brotli. + work = [(row[0], reassemble(conn, row[0], row[1]), len(row[1]), *row[2:]) for row in bases] + + def scan(item): + """Decompress one row's bytes and rewrite them if they mention a + converted file. Pure CPU/subprocess - touches no database.""" + path, full, base_len, language_id, content_type_id, template_id, compression = item + try: + raw = codec.decompress(full, compression) + except Exception as exc: # noqa: BLE001 - one unreadable row is not the run + return ("error", path, str(exc)) + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + # Binary content filed under a text content type - this database + # stores .ogv/.webm video as text/plain. It cannot contain a + # textual reference, so skip it rather than failing the run. + return ("binary", path, None) + new_text, hits = rewrite_text(text, pattern, renames) + if not hits: + return None + new_stored = codec.compress(new_text.encode("utf-8"), compression) + return ("ok", path, base_len, new_stored, language_id, content_type_id, template_id, hits) + + with ThreadPoolExecutor(max_workers=cfg["workers"]) as pool: + results = [r for r in pool.map(scan, work) if r is not None] + + errors = [r for r in results if r[0] == "error"] + binary = [r for r in results if r[0] == "binary"] + changes = [r for r in results if r[0] == "ok"] + if binary: + print(f"{len(binary)} row(s) skipped: binary content stored under a text content type.") + for _kind, path, message in errors: + print(f" error: could not read {path}: {message}", file=sys.stderr) + + total_hits = sum(r[7] for r in changes) + print(f"{len(changes)} row(s) reference a converted file; {total_hits} reference(s) to rewrite.") + + if cfg["dry_run"]: + print("Dry run: nothing written.") + else: + print(f"Backing up {db_path} ...") + print(f"Backup written to {backup_database(db_path)}") + conn.execute("BEGIN") + try: + for _kind, path, base_len, new_stored, lang, ctid, tid, hits in changes: + replace_row(conn, path, base_len, new_stored, lang, ctid, tid, + lambda m: print(m, file=sys.stderr)) + if cfg["verbose"]: + print(f" [REF] {path}: {hits} reference(s)") + conn.commit() + except Exception: + conn.rollback() + raise + finally: + if codec is not None: + codec.close() + before_conn.close() + conn.close() + + if not cfg["dry_run"] and changes: + print("Vacuuming database...") + vac = sqlite3.connect(db_path) + try: + vac.execute("VACUUM") + finally: + vac.close() + + verb = "would rewrite" if cfg["dry_run"] else "rewrote" + print(f"\nDone. {verb} {total_hits} reference(s) across {len(changes)} row(s); " + f"{len(renames)} converted filename(s) known; {len(binary)} binary row(s) skipped; " + f"{len(errors)} error(s).") + return 1 if errors else 0 + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("db_path", type=Path, help="SQLite database whose media references should be updated") + p.add_argument("--before", type=Path, required=True, + help="The pre-conversion copy of this database (the backup optimize_db_media.py wrote), " + "diffed against it to learn exactly which files were renamed") + p.add_argument("--dry-run", action="store_true", help="Report what would change without writing") + p.add_argument("--workers", type=int, default=8, + help="Parallel (de)compression workers (default: 8)") + p.add_argument("--verbose", action="store_true", help="List rewritten rows and sample renames") + args = p.parse_args() + sys.exit(run({"db_path": args.db_path, "before": args.before, "dry_run": args.dry_run, + "workers": args.workers, "verbose": args.verbose})) + + +if __name__ == "__main__": + main() From 142349a45a40f60219a43a0c8486e6fc54fb0bc4 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Thu, 10 Sep 2026 07:25:09 -0500 Subject: [PATCH 4/5] ADFA-5552: Scan only the content types that carry media references TEXT_TYPES listed seven content types on the theory that any of them could hold a link. Measured against a full production database, every one of the 578 rewritten rows was text/html (541), text/javascript (29) or text/css (8); text/plain, application/json, text/markdown and application/xml rewrote nothing at all. text/plain was the plausible one - this schema files ~61 "j/html/api/*/module-graph.svg" under it, and SVG can reference a raster via , but none of those module graphs does. Its only measurable contribution was cost: it also files 16 binary .ogv/.webm videos as text, and those are the sole reason the UnicodeDecodeError branch had anything to catch. The one text/plain row that even resembles a reference is a false positive - "javax.imageio.plugins.jpeg", a Java package name in j/html/api/element-list, which is exactly the class of match the delimiter anchor already rejects. Narrowing the list is behavior-neutral and verified so: re-run against the same pre-rewrite database it still finds 1699 references across 578 rows, scanning 28982 rows instead of 29069 (the 87 dropped rows are the 80 text/plain, 5 markdown and 2 json) and skipping 0 binary rows instead of 16. The decode guard stays as insurance, since this schema has been seen filing binary under a text content type. Co-Authored-By: Claude Opus 4.8 --- scripts/update_media_references.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/scripts/update_media_references.py b/scripts/update_media_references.py index 5759c864..ba6ca752 100644 --- a/scripts/update_media_references.py +++ b/scripts/update_media_references.py @@ -68,9 +68,20 @@ load_dictionary, owned_fragment_paths, reassemble, ) -# Content types whose stored text can carry a link to a media file. -TEXT_TYPES = ("text/html", "application/json", "text/css", "text/markdown", - "text/javascript", "text/plain", "application/xml") +# Content types that actually carry links to media, measured rather than +# guessed: across a full production database every rewritten row was one of +# these three - 541 text/html (page markup), 29 text/javascript (javadoc's +# search UI referencing glass.png/x.png), 8 text/css (url(...) backgrounds). +# +# text/plain, application/json, text/markdown and application/xml were on this +# list first and are deliberately off it now: all four were scanned over that +# same database and rewrote nothing. text/plain was the tempting one - this +# schema files ~61 "j/html/api/*/module-graph.svg" under it, and SVG genuinely +# can reference a raster via , but none of those module +# graphs does. What it did contribute was cost: it also files 16 binary .ogv/ +# .webm videos as text, every one of which had to be decoded and discarded. +# Add a type back when a row of that type is shown to hold a reference. +TEXT_TYPES = ("text/html", "text/javascript", "text/css") # A "-" chunk continuation row, which vanishes with its base rather # than being renamed. _FRAGMENT_RE = re.compile(r"-\d+$") @@ -262,9 +273,11 @@ def scan(item): try: text = raw.decode("utf-8") except UnicodeDecodeError: - # Binary content filed under a text content type - this database - # stores .ogv/.webm video as text/plain. It cannot contain a - # textual reference, so skip it rather than failing the run. + # Insurance, not a known case: with TEXT_TYPES narrowed to + # markup/JS/CSS nothing here should be binary, but this schema + # has been seen filing binary video under a text content type, + # so a row that isn't text is skipped rather than failing the + # whole run. return ("binary", path, None) new_text, hits = rewrite_text(text, pattern, renames) if not hits: From f9934bd733baf6d13009917d47f05ed08d11cb31 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Thu, 10 Sep 2026 07:42:42 -0500 Subject: [PATCH 5/5] ADFA-5552: Keep JSON/markdown/XML in the reference scan, drop only text/plain Narrowing TEXT_TYPES to html/javascript/css went too far. Those three are the only ones that rewrote anything in the database measured, but JSON, markdown and XML are real carriers in this schema, not hypotheticals: Kotlin's nav row is application/json holding literal "/k/html/images/" links - the row insert_optimized_media.py exists to rewrite - markdown embeds images as ![](foo.png), and XML does so in an attribute. A database populated by a different pipeline would put references in exactly those rows, and scanning them costs one decode of a handful of rows (7 here). text/plain stays out, because its exclusion rests on what it holds rather than on it having scored zero: ~61 "j/html/api/*/module-graph.svg" that reference no rasters, and 16 binary .ogv/.webm videos filed as text - the only rows that ever hit the decode guard. Its one reference-shaped string is a false positive, "javax.imageio.plugins.jpeg" in j/html/api/element-list. Verified against the same pre-rewrite database: still 1699 references across 578 rows, scanning 28989 rows (vs 29069 with text/plain included and 28982 without JSON/markdown/XML) and skipping 0 binary rows. Co-Authored-By: Claude Opus 4.8 --- scripts/update_media_references.py | 34 ++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/scripts/update_media_references.py b/scripts/update_media_references.py index ba6ca752..21dbe039 100644 --- a/scripts/update_media_references.py +++ b/scripts/update_media_references.py @@ -68,20 +68,28 @@ load_dictionary, owned_fragment_paths, reassemble, ) -# Content types that actually carry links to media, measured rather than -# guessed: across a full production database every rewritten row was one of -# these three - 541 text/html (page markup), 29 text/javascript (javadoc's -# search UI referencing glass.png/x.png), 8 text/css (url(...) backgrounds). +# Content types whose stored text can carry a link to a media file. # -# text/plain, application/json, text/markdown and application/xml were on this -# list first and are deliberately off it now: all four were scanned over that -# same database and rewrote nothing. text/plain was the tempting one - this -# schema files ~61 "j/html/api/*/module-graph.svg" under it, and SVG genuinely -# can reference a raster via , but none of those module -# graphs does. What it did contribute was cost: it also files 16 binary .ogv/ -# .webm videos as text, every one of which had to be decoded and discarded. -# Add a type back when a row of that type is shown to hold a reference. -TEXT_TYPES = ("text/html", "text/javascript", "text/css") +# Three are measured carriers: over a full production database every rewritten +# row was text/html (541, page markup), text/javascript (29, javadoc's search +# UI referencing glass.png/x.png) or text/css (8, url(...) backgrounds). +# +# application/json, text/markdown and application/xml rewrote nothing in that +# particular database but are kept, because each genuinely embeds references +# elsewhere in this schema: Kotlin's nav row is application/json carrying +# literal "/k/html/images/" links (insert_optimized_media.py rewrites +# exactly that row), markdown embeds images as ![](foo.png), and XML does so +# in an attribute. Scanning them costs one decode of a handful of rows. +# +# text/plain is deliberately NOT here, having been measured and found to hold +# no references at all: this schema files ~61 "j/html/api/*/module-graph.svg" +# under it - and SVG can reference a raster via - but none +# of those module graphs does, while it also files 16 binary .ogv/.webm videos +# as text, the only rows that ever hit the decode guard below. The one +# text/plain row that even resembles a reference is a false positive: +# "javax.imageio.plugins.jpeg", a Java package name in j/html/api/element-list. +TEXT_TYPES = ("text/html", "application/json", "text/css", "text/markdown", + "text/javascript", "application/xml") # A "-" chunk continuation row, which vanishes with its base rather # than being renamed. _FRAGMENT_RE = re.compile(r"-\d+$")