From 9436960df5d9a22bb1d04bcb812421c82a50ead1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:11:54 +0000 Subject: [PATCH 1/5] Intro and outro phrases: cut from the start through a phrase, or from a phrase to the end Each podcast's Settings tab lists phrases that end its intro and phrases that begin its outro. When an episode is cut, its timed transcript (the feed's, or captions) is searched for them, ignoring case and punctuation: an intro phrase in the first third cuts from the start through its first hearing, an outro phrase in the last third from its last hearing to the end. The cuts are saved on the episode, written into its chapters, and shown on the timeline as Intro (phrase) and Outro (phrase). The captions job now keeps whisper's time for each word beside the captions, so these cuts land between words; feed transcripts, timed by cue, place words by where they fall in the cue. Captions arrive after the first cut, so a phrase heard in them queues the episode to be cut again; saving new phrases queues the ready episodes they would cut differently. The transcription service is unchanged: it already sent word times. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015gCh3c3PYL5Y4shMugurtB --- README.md | 3 +- WEBAPP.md | 26 +++- podblock/app/captions.py | 11 +- podblock/app/chapters.py | 14 +- podblock/app/db.py | 6 + podblock/app/phrases.py | 202 ++++++++++++++++++++++++++++ podblock/app/processing.py | 43 +++--- podblock/web/routes/podcasts.py | 23 +++- podblock/web/static/app.css | 1 + podblock/web/static/timeline.js | 14 +- podblock/web/templates/podcast.html | 18 +++ tests/test_captions.py | 19 +++ tests/test_phrases.py | 61 +++++++++ tests/test_processing.py | 31 +++++ tests/test_schema.py | 2 +- tests/test_web.py | 5 +- 16 files changed, 442 insertions(+), 37 deletions(-) create mode 100644 podblock/app/phrases.py create mode 100644 tests/test_phrases.py diff --git a/README.md b/README.md index ea0efa8..08d8e94 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,8 @@ feeds or audio files and it lists what repeats. See [`CLI.md`](CLI.md). podcasts. - **Captions.** Optional: run the [`whisper/`](whisper/) container beside it and each download gets captions in the private feed and under the - episode page's player, made on the CPU. + episode page's player, made on the CPU. A show can also list phrases + that end its intro or start its outro; everything past them is cut. - **Stays small.** Downloads are cleaned up a week after they are cut (set per podcast), unless you keep them. diff --git a/WEBAPP.md b/WEBAPP.md index b717772..2d49bd3 100644 --- a/WEBAPP.md +++ b/WEBAPP.md @@ -241,7 +241,20 @@ on Kubernetes, with backups and upgrades, see `deployment/README.md`. meanwhile, and captions jobs take a worker thread of their own. When the service has speakers on, the turns it finds are saved beside the captions (`podblock/app/speakers.py`), for `podblock speakers` to set beside the - repeats; they don't go into the captions or the cuts yet. + repeats; they don't go into the captions or the cuts yet. Each word's + time is saved beside the captions too, for intro and outro phrases. +- Intro and outro phrases (`podblock/app/phrases.py`): a podcast can list + phrases that end its intro and phrases that begin its outro. An episode + with a timed transcript is searched for them each time it is cut, + ignoring case and punctuation. An intro phrase heard in the first third + cuts from the start through its first hearing; an outro phrase heard in + the last third cuts from its last hearing to the end. Captions place the + cut between words; a feed's transcript, timed by cue, places it by where + the words fall in their cue. Captions arrive after the first cut, so + hearing a phrase in them queues the episode to be cut again, and saving + new phrases queues the ready episodes they cut differently. The timeline + shows these cuts as Intro (phrase) and Outro (phrase), changed only + through the phrases. - Each join gets a chapter saying what was cut and how much: "1:32 ad break cut", or, inside a chapter of the source's, "Interview (after 1:32 ad break cut)". An episode with no chapters of its own gets a "Start" @@ -492,8 +505,8 @@ on Kubernetes, with backups and upgrades, see `deployment/README.md`. downloaded is left alone. - The Settings tab sets days before clean-up, how often to check the feed (15 minutes to a week), how close two repeats must be to be combined, - oldest first, and whether new episodes download on their own. Settings - are shared. + oldest first, and whether new episodes download on their own, and lists + the show's intro and outro phrases. Settings are shared. ### Look and feel @@ -543,7 +556,8 @@ podcasts + author, description, image_url, link, language, categories (JSON), e followed, refresh_minutes, feed_etag, feed_last_modified, last_refreshed_at, last_error, added_at, public_token, -- guards audio links - retention_days, auto_download, fill_gaps, sort_oldest + retention_days, auto_download, fill_gaps, sort_oldest, + intro_phrases, outro_phrases -- JSON lists (phrases.py) episodes + pub_ts, summary, image_url, link, audio_type, audio_length, feed_duration, season, episode_number, episode_type, explicit, item_xml, -- raw @@ -554,7 +568,8 @@ episodes + pub_ts, summary, image_url, link, audio_type, audio_length, feed_dur peaks_path, -- the waveform source_chapters, transcripts, -- the source's, as found when first cut cut_chapters, -- the chapters written into the cut file - held_by -- the running job working on it + held_by, -- the running job working on it + phrase_cuts, -- the intro and outro its phrases cut, as last cut ready_at, expired_at -- the retention clock jobs(id, kind, episode_id, state, step, progress, error, cancel_requested, attempts, @@ -594,6 +609,7 @@ only looks at fingerprinted ones, so they don't affect it. | `POST /podcasts/{id}/settings` | yes | clean-up days, refresh interval, gap, auto-download, `sort` (newest or oldest) | | `POST /podcasts/{id}/subscribe` | yes | `on` = true or false: put it on your home page or take it off | | `POST /podcasts/{id}/feed-token` | yes | new address for your own private feed | +| `POST /podcasts/{id}/phrases` | yes | the show's `intro` and `outro` phrases, one a line; queues the episodes they cut differently | | `POST /podcasts/{id}/filters` | yes | add a filter: `pattern`, `scope` (podcast or all), `processing` | | `POST /podcasts/{id}/remove` | admin | stop following | | `POST /filters/{id}/remove` | owner or admin | remove a filter | diff --git a/podblock/app/captions.py b/podblock/app/captions.py index b0c1523..5521153 100644 --- a/podblock/app/captions.py +++ b/podblock/app/captions.py @@ -14,6 +14,9 @@ edit can be saved meanwhile. Captions jobs run in a thread of their own (worker.py), so a long transcription doesn't hold up downloads. +Each word's time is saved beside the captions too, for the show's intro and +outro phrases (`phrases.py`); hearing one queues the episode to be cut again. + When the service has speakers turned on, it also says who speaks when; those turns are saved beside the captions (`speakers.py`) for `podblock speakers` to compare with the repeats found. They don't go into the captions yet. @@ -30,7 +33,7 @@ import requests -from . import chapters, speakers +from . import chapters, phrases, speakers from .config import Settings from .db import transaction from .jobs import Context, enqueue_task @@ -215,7 +218,9 @@ def run(ctx: Context, settings: Settings) -> None: stem = settings.media.transcripts(ep) path = stem.with_name(stem.name + ".captions.vtt") _write(path, webvtt(cues(heard["segments"]))) - saved = {"format": "vtt", "path": str(path), "rel": "captions", "made": "whisper", + timed = stem.with_name(stem.name + ".words.json") # each word's time, for finding intro and outro phrases + _write(timed, json.dumps([w for seg in heard["segments"] for w in _words(seg)], separators=(",", ":"))) + saved = {"format": "vtt", "path": str(path), "rel": "captions", "made": "whisper", "words": str(timed), **({"language": heard["language"]} if heard["language"] else {})} if heard["speakers"] is not None: saved["speakers"] = str(speakers.save(stem, heard["speakers"], heard["options"])) @@ -228,6 +233,8 @@ def run(ctx: Context, settings: Settings) -> None: log.info("captions for episode %d: %d segments, %s%s", episode_id, len(heard["segments"]), heard["language"] or "language unknown", f", {len({t[2] for t in heard['speakers']})} speakers" if heard["speakers"] else "") + phrases.requeue(ctx.db, ep["podcast_id"], ctx.job["requested_by"], [episode_id], + note="Waiting to be cut again: an intro or outro phrase was heard") def _write(path: Path, text: str) -> None: diff --git a/podblock/app/chapters.py b/podblock/app/chapters.py index 2eb439e..1f94a00 100644 --- a/podblock/app/chapters.py +++ b/podblock/app/chapters.py @@ -205,18 +205,21 @@ def gather(db: sqlite3.Connection, ep: sqlite3.Row, stem: Path) -> None: def forget(ep: sqlite3.Row | dict) -> None: - """Delete an episode's saved transcripts, and the speaker turns saved with captions, as its download goes.""" + """Delete an episode's saved transcripts, and the speaker turns and word times saved with captions, as its + download goes.""" for t in json.loads(ep["transcripts"] or "[]"): - for path in (t["path"], t.get("speakers")): + for path in (t["path"], t.get("speakers"), t.get("words")): if path: Path(path).unlink(missing_ok=True) # ---- chapters for the cut ------------------------------------------------------ -def cuts(report: dict | None, edits: list) -> list[tuple[float, float, set[str]]]: - """What was cut where: each section of the report with its types, and each stretch cut by hand.""" +def cuts(report: dict | None, edits: list, phrase_cuts: list[dict] = ()) -> list[tuple[float, float, set[str]]]: + """What was cut where: each section of the report with its types, each stretch cut by hand, and each intro + or outro its phrases cut.""" found = [(s["start"], s["end"], set(s["types"])) for s in (report or {}).get("sections", [])] + found += [(p["start"], p["end"], {p["type"]}) for p in phrase_cuts] return found + [(r["start"], r["end"], {"manual"}) for r in edits] @@ -272,7 +275,8 @@ def for_episode(db: sqlite3.Connection, ep: sqlite3.Row) -> list[dict]: return json.loads(ep["cut_chapters"]) saved = db.execute("SELECT report FROM reports WHERE episode_id = ?", (ep["id"],)).fetchone() edits = db.execute("SELECT start, end FROM edits WHERE episode_id = ?", (ep["id"],)).fetchall() - return build(keep_of(ep), cuts(json.loads(saved["report"]) if saved else None, edits), + return build(keep_of(ep), cuts(json.loads(saved["report"]) if saved else None, edits, + json.loads(ep["phrase_cuts"] or "[]")), json.loads(ep["source_chapters"] or "[]")) diff --git a/podblock/app/db.py b/podblock/app/db.py index a8b7a74..a772ec0 100644 --- a/podblock/app/db.py +++ b/podblock/app/db.py @@ -227,6 +227,12 @@ """ ALTER TABLE episodes ADD COLUMN held_by INTEGER; -- the running job working on it: its own, or taken along """, + # 16: phrases that end a show's intro or begin its outro, and the cuts they made in each episode (phrases.py) + """ + ALTER TABLE podcasts ADD COLUMN intro_phrases TEXT; -- JSON list + ALTER TABLE podcasts ADD COLUMN outro_phrases TEXT; -- JSON list + ALTER TABLE episodes ADD COLUMN phrase_cuts TEXT; -- JSON list of {type, start, end, phrase}, as last cut + """, ] diff --git a/podblock/app/phrases.py b/podblock/app/phrases.py new file mode 100644 index 0000000..13caf73 --- /dev/null +++ b/podblock/app/phrases.py @@ -0,0 +1,202 @@ +"""Intro and outro phrases: words a show says at the edges of its talk, cut along with everything past them. + +Each podcast can list phrases that end its intro ("let's get into it") and +phrases that begin its outro ("that's all for this week"). Once an episode +has a timed transcript (its feed's, or captions from the transcription +service), it is searched for them: + +- an intro phrase heard in the first third cuts from the start to the + phrase's last word, at its first hearing; +- an outro phrase heard in the last third cuts from the phrase's first word + to the end, at its last hearing. + +So a phrase said again mid-episode cuts nothing. Matching ignores case and +punctuation ("Let's get into it!" is "lets get into it"), word for word. + +Captions keep whisper's time for each word (`words.json` beside them), so +cuts land between words; a feed's transcript only times whole cues, so +there each word's time is guessed from where it falls in its cue. + +What is found is saved on the episode (`episodes.phrase_cuts`) each time it +is cut, as [{type: "intro" | "outro", start, end, phrase}], and cut with +the repeats. Captions come after the first cut, so finding a phrase in them +queues the episode to be cut again; so does changing the phrases. +""" + +from __future__ import annotations + +import json +import re +import sqlite3 +from pathlib import Path + +from . import chapters +from .db import transaction +from .jobs import enqueue +from .states import READY + +MAX_PHRASES = 20 # of each kind, per podcast +MAX_CHARS = 100 # in a phrase +REACH = 1 / 3 # of the episode, from its edge, where a phrase counts +PAD = 0.25 # seconds past a phrase's edge also cut, when the next word isn't nearer: whisper's times run tight +INTRO, OUTRO = "intro", "outro" + + +class PhraseError(ValueError): + """Phrases that can't be saved; the message says why, for the page.""" + + +def tokens(text: str) -> list[str]: + """Words to match: lower case, without punctuation, "let's" as "lets".""" + return re.findall(r"\w+", re.sub(r"['’ʼ]", "", text.casefold())) + + +def parse(text: str) -> list[str]: + """Phrases from a box of them, one a line. Blank lines and repeats go.""" + found: dict[tuple, str] = {} + for line in text.splitlines(): + line = " ".join(line.split()) + if not line: + continue + if len(line) > MAX_CHARS: + raise PhraseError(f"Phrases can be at most {MAX_CHARS} characters.") + if not tokens(line): + raise PhraseError(f"“{line}” has no words to listen for.") + found.setdefault(tuple(tokens(line)), line) + if len(found) > MAX_PHRASES: + raise PhraseError(f"At most {MAX_PHRASES} phrases of each kind.") + return list(found.values()) + + +def of_podcast(db: sqlite3.Connection, podcast_id: int | None) -> tuple[list[str], list[str]]: + row = db.execute("SELECT intro_phrases, outro_phrases FROM podcasts WHERE id = ?", (podcast_id,)).fetchone() + if row is None: + return [], [] + return json.loads(row["intro_phrases"] or "[]"), json.loads(row["outro_phrases"] or "[]") + + +# ---- the words of a transcript, with their times ----------------------------------- + +def _from_cue(start: float, end: float, text: str) -> list[tuple[float, float, str]]: + """A cue's words, each given a share of its time by where it falls in the text.""" + text = chapters._unescape(re.sub(r"<[^>]*>", "", text)) + spans = [(m.start(), m.end(), w) for m in re.finditer(r"\S+", text) for w in tokens(m[0])] + if not spans or end <= start: + return [] + size = max(1, len(text.rstrip())) + return [(start + (end - start) * a / size, start + (end - start) * b / size, w) for a, b, w in spans] + + +def _cue_words(text: str) -> list[tuple[float, float, str]]: + out = [] + for block in re.split(r"\n[ \t]*\n", text.replace("\r\n", "\n").replace("\r", "\n").strip()): + lines = block.split("\n") + at = next((i for i, line in enumerate(lines) if "-->" in line), None) + m = chapters.CUE.match(lines[at]) if at is not None else None + start, end = (chapters._clock(m[1]), chapters._clock(m[2])) if m else (None, None) + if start is not None and end is not None: + out += _from_cue(start, end, " ".join(lines[at + 1:])) + return out + + +def _json_words(text: str) -> list[tuple[float, float, str]]: + segments = [s for s in json.loads(text).get("segments", []) + if isinstance(s, dict) and isinstance(s.get("startTime"), (int, float))] + out = [] + for i, seg in enumerate(segments): + nxt = segments[i + 1]["startTime"] if i + 1 < len(segments) else seg["startTime"] + 5 + end = seg["endTime"] if isinstance(seg.get("endTime"), (int, float)) else nxt + out += _from_cue(seg["startTime"], end, str(seg.get("body") or "")) + return out + + +def words(ep: sqlite3.Row) -> list[tuple[float, float, str]]: + """The episode's words in the uncut audio, in order, as (start, end, word) with `word` as `tokens` gives it: + whisper's own times when its captions kept them, else guessed from a transcript's cues. Empty without one.""" + saved = chapters.transcripts(ep) + for t in saved: + if t.get("words") and Path(t["words"]).exists(): + return [(a, b, w) for a, b, word in json.loads(Path(t["words"]).read_text()) for w in tokens(word)] + for fmt in ("vtt", "srt", "json"): + if any(t["format"] == fmt for t in saved): + text = chapters._read(ep, fmt) + try: + return _json_words(text) if fmt == "json" else _cue_words(text) + except (ValueError, AttributeError): + return [] + return [] + + +# ---- finding them ---------------------------------------------------------------- + +def _hits(said: list[str], phrase: list[str]) -> list[int]: + n = len(phrase) + return [i for i in range(len(said) - n + 1) if said[i:i + n] == phrase] + + +def find(heard: list[tuple[float, float, str]], duration: float, intro: list[str], + outro: list[str]) -> list[dict]: + """The cuts the phrases make: at most one intro and one outro, as {type, start, end, phrase}.""" + if not heard or not duration: + return [] + said = [w for _, _, w in heard] + found = [] + first = None # (index of the phrase's last word, phrase) + for phrase in intro: + hits = _hits(said, tokens(phrase)) + last = hits[0] + len(tokens(phrase)) - 1 if hits else None + if last is not None and heard[last][1] <= duration * REACH and (first is None or last < first[0]): + first = (last, phrase) + if first is not None: + at, phrase = first + nxt = heard[at + 1][0] if at + 1 < len(heard) else duration + end = min(max(heard[at][1], min(heard[at][1] + PAD, nxt)), duration) + found.append({"type": INTRO, "start": 0.0, "end": round(end, 2), "phrase": phrase}) + final = None # (index of the phrase's first word, phrase) + for phrase in outro: + hits = _hits(said, tokens(phrase)) + if hits and heard[hits[-1]][0] >= duration * (1 - REACH) and (final is None or hits[-1] > final[0]): + final = (hits[-1], phrase) + if final is not None: + at, phrase = final + prev = heard[at - 1][1] if at > 0 else 0.0 + start = max(min(heard[at][0], max(heard[at][0] - PAD, prev)), 0.0) + if not found or start > found[0]["end"]: + found.append({"type": OUTRO, "start": round(start, 2), "end": round(duration, 2), "phrase": phrase}) + return found + + +def for_episode(db: sqlite3.Connection, ep: sqlite3.Row) -> list[dict]: + """The cuts the podcast's phrases make in this episode, as `find` gives them.""" + intro, outro = of_podcast(db, ep["podcast_id"]) + if not (intro or outro) or not ep["duration"]: + return [] + return find(words(ep), ep["duration"], intro, outro) + + +def saved(ep: sqlite3.Row) -> list[dict]: + return json.loads(ep["phrase_cuts"] or "[]") + + +# ---- when the phrases change ------------------------------------------------------ + +def save(db: sqlite3.Connection, podcast_id: int, intro: list[str], outro: list[str], user_id: int | None) -> int: + """Save a podcast's phrases, and queue its ready episodes to be cut again where they now cut differently. + Returns how many were queued.""" + with transaction(db): + db.execute("UPDATE podcasts SET intro_phrases = ?, outro_phrases = ? WHERE id = ?", + (json.dumps(intro), json.dumps(outro), podcast_id)) + return requeue(db, podcast_id, user_id, note="Waiting to be cut again: the intro and outro phrases changed") + + +def requeue(db: sqlite3.Connection, podcast_id: int, user_id: int | None, episode_ids: list[int] | None = None, + note: str | None = None) -> int: + """Queue the podcast's ready episodes (or these of them) whose phrase cuts are no longer what they were cut + with. Returns how many were queued.""" + rows = db.execute("SELECT * FROM episodes WHERE podcast_id = ? AND state = ? AND transcripts IS NOT NULL " + "AND transcripts != '[]'", (podcast_id, READY)).fetchall() + want = set(episode_ids) if episode_ids is not None else None + stale = [ep["id"] for ep in rows + if (want is None or ep["id"] in want) and for_episode(db, ep) != saved(ep)] + return len(enqueue(db, "process", stale, user_id, skip_ready=False, payload=json.dumps({"recut": True}), + note=note)) diff --git a/podblock/app/processing.py b/podblock/app/processing.py index 3a761aa..892fb60 100644 --- a/podblock/app/processing.py +++ b/podblock/app/processing.py @@ -22,7 +22,7 @@ from ..matcher import MatchParams from ..pipeline import Paths, episode_report, find_common_blocks, load_fingerprint, rescan from ..store import BUMPER, BUMPER_IN, BUMPER_OUT, IGNORED, INTRO, KEPT, OUTRO, PODCAST_TYPES, SHOW, Store -from . import captions, chapters +from . import captions, chapters, phrases from .config import Settings from .db import transaction from .jobs import Cancelled, Context, enqueue, settle_episode, take_along @@ -100,9 +100,12 @@ def _fill_gaps(db: sqlite3.Connection, podcast_id: int | None) -> float: return row["fill_gaps"] if row is not None else FILL_GAPS -def cuts(db: sqlite3.Connection, episode_id: int, report: dict) -> list[tuple[float, float]]: - """Everything to cut from an episode: the report's sections and the ones cut by hand.""" +def cuts(db: sqlite3.Connection, episode_id: int, report: dict, + phrase_cuts: list[dict] = ()) -> list[tuple[float, float]]: + """Everything to cut from an episode: the report's sections, the intro and outro its phrases mark, and the + ones cut by hand.""" found = [(s["start"], s["end"]) for s in report["sections"]] + found += [(p["start"], p["end"]) for p in phrase_cuts] found += [(r["start"], r["end"]) for r in db.execute("SELECT start, end FROM edits WHERE episode_id = ?", (episode_id,))] return found @@ -110,21 +113,22 @@ def cuts(db: sqlite3.Connection, episode_id: int, report: dict) -> list[tuple[fl def _render(ctx: Context, settings: Settings, episode_id: int, report: dict) -> None: ep = _episode(ctx.db, episode_id) - keep = cut.keep_spans(cuts(ctx.db, episode_id, report), ep["duration"]) - if not keep: - raise ProcessError("the whole episode was found to repeat, so nothing would be left") - spec = json.dumps([[round(a, 2), round(b, 2)] for a, b in keep]) - if ep["source_chapters"] is None: + if ep["source_chapters"] is None: # with the feed's transcripts, for its phrases ctx.step("Reading chapters", episode_id) chapters.gather(ctx.db, ep, settings.media.transcripts(ep)) ep = _episode(ctx.db, episode_id) + said = phrases.for_episode(ctx.db, ep) + keep = cut.keep_spans(cuts(ctx.db, episode_id, report, said), ep["duration"]) + if not keep: + raise ProcessError("the whole episode was found to repeat, so nothing would be left") + spec = json.dumps([[round(a, 2), round(b, 2)] for a, b in keep]) edits = ctx.db.execute("SELECT start, end FROM edits WHERE episode_id = ?", (episode_id,)).fetchall() - marks = chapters.build(json.loads(spec), chapters.cuts(report, edits), json.loads(ep["source_chapters"])) + marks = chapters.build(json.loads(spec), chapters.cuts(report, edits, said), json.loads(ep["source_chapters"])) if (spec == ep["cut_spec"] and json.dumps(marks) == ep["cut_chapters"] and ep["cut_path"] and Path(ep["cut_path"]).exists()): with transaction(ctx.db): # nothing to cut differently - ctx.db.execute("UPDATE episodes SET state = 'ready', state_note = NULL, " - "ready_at = COALESCE(ready_at, ?) WHERE id = ?", (time.time(), episode_id)) + ctx.db.execute("UPDATE episodes SET state = 'ready', state_note = NULL, phrase_cuts = ?, " + "ready_at = COALESCE(ready_at, ?) WHERE id = ?", (json.dumps(said), time.time(), episode_id)) return ctx.step("Cutting", episode_id) try: @@ -137,10 +141,10 @@ def _render(ctx: Context, settings: Settings, episode_id: int, report: dict) -> with transaction(ctx.db): ctx.db.execute( """UPDATE episodes SET state = 'ready', state_note = NULL, cut_path = ?, cut_type = ?, cut_duration = ?, - cut_bytes = ?, cut_spec = ?, cut_chapters = ?, processed_at = ?, + cut_bytes = ?, cut_spec = ?, cut_chapters = ?, phrase_cuts = ?, processed_at = ?, ready_at = COALESCE(ready_at, ?), expired_at = NULL, ever_downloaded = 1 WHERE id = ?""", - (str(out.path), out.mime, out.duration, out.size, spec, json.dumps(marks), time.time(), time.time(), - episode_id)) + (str(out.path), out.mime, out.duration, out.size, spec, json.dumps(marks), json.dumps(said), time.time(), + time.time(), episode_id)) def ensure_peaks(db: sqlite3.Connection, settings: Settings, episode_id: int) -> Path | None: @@ -423,9 +427,11 @@ def run_edit(ctx: Context, settings: Settings) -> None: def timeline(db: sqlite3.Connection, episode_id: int, report: dict | None) -> dict: """What the episode page's timeline draws: each cut section, what it is, and the blocks and bumpers in it; - the sections kept in this podcast; and the false positives that play in it, for a list of their own. - `custom` marks a section of this podcast's own types or blocks; `own` lists its blocks marked by hand.""" - podcast = db.execute("SELECT podcast_id FROM episodes WHERE id = ?", (episode_id,)).fetchone()[0] + the sections kept in this podcast; the intro and outro its phrases cut; and the false positives that play in + it, for a list of their own. `custom` marks a section of this podcast's own types or blocks; `own` lists its + blocks marked by hand.""" + ep = _episode(db, episode_id) + podcast = ep["podcast_id"] store = Store.on(db) types = store.types(podcast) custom = set(store.podcast_types(podcast)) if podcast is not None else set() @@ -461,5 +467,6 @@ def timeline(db: sqlite3.Connection, episode_id: int, report: dict | None) -> di for i, x in enumerate((report or {}).get("not_repeats", []))] manual = [{"id": f"m{r['id']}", "edit": r["id"], "start": r["start"], "end": r["end"]} for r in db.execute("SELECT * FROM edits WHERE episode_id = ? ORDER BY start", (episode_id,))] - return {"sections": sections, "manual": manual, "not_repeats": not_repeats} + said = [{"id": f"p{i}", **p} for i, p in enumerate(phrases.saved(ep))] + return {"sections": sections, "manual": manual, "not_repeats": not_repeats, "phrases": said} diff --git a/podblock/web/routes/podcasts.py b/podblock/web/routes/podcasts.py index 2fd1733..70da3c0 100644 --- a/podblock/web/routes/podcasts.py +++ b/podblock/web/routes/podcasts.py @@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, Form, HTTPException, Query, Request from fastapi.responses import JSONResponse, RedirectResponse, Response -from ...app import directory, filters, following, library, private_feeds, storage +from ...app import directory, filters, following, library, phrases, private_feeds, storage from ..deps import admin_user, current_user, feed_url, get_db, wake_worker from ..templating import flash, page, plural @@ -154,6 +154,10 @@ def show_podcast(request: Request, podcast_id: int, view: str = "active", q: str return page(request, "_episodes.html", **context) if tab == "feed": context.update(private_addresses=_addresses(request, private_feeds.feed_token(db, user["id"], podcast_id))) + if tab == "settings": + intro, outro = phrases.of_podcast(db, podcast_id) + context.update(intro_phrases=params.get("intro", "\n".join(intro)), + outro_phrases=params.get("outro", "\n".join(outro))) return page(request, "podcast.html", title=pod["name"], nav="podcasts", user=user, cut_sec=library.cut_seconds(db, podcast_id), subscribed=library.subscribed(db, user["id"], podcast_id), @@ -178,6 +182,23 @@ def podcast_settings(request: Request, podcast_id: int, retention_days: int = Fo return RedirectResponse(f"/podcasts/{podcast_id}?tab=settings#tabs", status_code=303) +@router.post("/podcasts/{podcast_id}/phrases") +def podcast_phrases(request: Request, podcast_id: int, intro: str = Form(""), outro: str = Form(""), + user=Depends(current_user), db: sqlite3.Connection = Depends(get_db)) -> Response: + """Save the phrases that end the show's intro and begin its outro, and cut its episodes again to fit.""" + _podcast_or_404(db, podcast_id) + try: + queued = phrases.save(db, podcast_id, phrases.parse(intro), phrases.parse(outro), user["id"]) + except phrases.PhraseError as e: + flash(request, str(e), "error") + typed = urlencode({"tab": "settings", "intro": intro, "outro": outro}) + return RedirectResponse(f"/podcasts/{podcast_id}?{typed}#phrases", status_code=303) + if queued: + wake_worker(request) + flash(request, "Saved." + (f" Cutting {plural(queued, 'episode')} again to fit." if queued else "")) + return RedirectResponse(f"/podcasts/{podcast_id}?tab=settings#phrases", status_code=303) + + @router.post("/podcasts/{podcast_id}/filters") def add_filter(request: Request, podcast_id: int, pattern: str = Form(""), scope: str = Form("podcast"), processing: bool = Form(False), user=Depends(current_user), diff --git a/podblock/web/static/app.css b/podblock/web/static/app.css index 776c0b0..b0446dc 100644 --- a/podblock/web/static/app.css +++ b/podblock/web/static/app.css @@ -833,6 +833,7 @@ button.transcript-time:hover { color: var(--accent); text-decoration: underline; .settings-form .field { flex: 1 1 240px; } .settings-form .btn { align-self: flex-end; margin-bottom: 22px; } .input-with-unit { display: flex; align-items: center; gap: 10px; } +textarea.phrase-input { padding: 10px 14px; resize: vertical; line-height: 1.4; } .input-with-unit .input { max-width: 120px; } select.input { appearance: auto; } @media (max-width: 720px) { diff --git a/podblock/web/static/timeline.js b/podblock/web/static/timeline.js index b3c87f3..554c570 100644 --- a/podblock/web/static/timeline.js +++ b/podblock/web/static/timeline.js @@ -27,6 +27,8 @@ const sections = [ ...data.sections.map((s) => ({ ...s, saved: s.value, deleted: false })), ...data.manual.map((m) => ({ ...m, manual: true, removed: false })), + // Cut by the show's intro and outro phrases: shown, and changed only through the phrases. + ...(data.phrases || []).map((p) => ({ ...p, byPhrase: true, value: p.type, saved: p.type, blocks: [], bumpers: [], own: [] })), ]; // False positives that play here: not on the timeline, only in a list of their own, to undo the flag. const falses = data.not_repeats.map((x) => ({ ...x, value: "ignored", saved: "ignored", bumpers: [] })); @@ -89,12 +91,14 @@ const isCut = (s) => !s.removed && !s.deleted && !(!s.manual && (s.value === "ignored" || s.value === "kept")); const allOwn = (s) => !s.manual && s.own.length > 0 && s.own.length === s.blocks.length; // only blocks marked by hand function kind(s) { + if (s.byPhrase) return s.type; if (s.manual) return s.removed ? "removed" : "manual"; if (s.deleted) return "removed"; if (s.value === "auto") return s.is_break ? "break" : "repeated"; return s.value === "kept" ? "ignored" : s.value; } function name(s) { + if (s.byPhrase) return s.type === "intro" ? "Intro (phrase)" : "Outro (phrase)"; if (s.manual) return isNew(s) ? "New section" : "Cut by hand"; if (s.value === "auto") return s.is_break ? "Ad break" : "Repeat"; return { intro: "Intro", outro: "Outro", show: "Local", kept: "Kept", ignored: "Not a repeat" }[s.value]; @@ -105,6 +109,7 @@ function changes() { const types = [], add = [], remove = [], del = []; for (const s of [...sections, ...falses]) { + if (s.byPhrase) continue; if (isNew(s)) add.push({ start: s.start, end: s.end, as_block: !!s.asBlock, type: s.blockType || "show" }); else if (s.manual && s.removed) remove.push(s.edit); else if (s.deleted) del.push(...s.own); @@ -184,7 +189,10 @@ const li = h("li", { class: `tl-item k-${kind(s)}${isNew(s) ? " tl-item-new" : ""}`, "data-id": s.id }, head, playButton(s, "icon-btn tl-item-play")); li.addEventListener("click", (e) => { if (!e.target.closest("button, select, input, label")) focusSection(s); }); - if (!s.manual) { + if (s.byPhrase) { + li.append(h("div", { class: "tl-item-row" }, + h("span", { class: "hint" }, `Heard “${s.phrase}”. Change it in the show's settings.`))); + } else if (!s.manual) { const row = h("div", { class: "tl-item-row" }, allOwn(s) ? null : valueSelect(s)); if (s.bumpers.length) row.append(h("span", { class: "hint" }, "Found from the bumpers around it")); if (editable && s.custom && !allOwn(s) && s.saved !== "auto" && s.value === s.saved) { @@ -254,7 +262,7 @@ } wave.append(s.region); s.label = h("div", { class: `tl-label k-${kind(s)}` }, - !s.manual && !allOwn(s) && editable ? valueSelect(s, true) : h("span", { class: "tl-label-text" }, name(s)), + !s.manual && !s.byPhrase && !allOwn(s) && editable ? valueSelect(s, true) : h("span", { class: "tl-label-text" }, name(s)), h("button", { type: "button", class: "tl-chip", title: `${name(s)}, ${clock(s.start)}: click to zoom in`, "aria-label": `${name(s)} at ${clock(s.start)}`, onclick: () => focusSection(s) }, icon(KIND_ICONS[kind(s)] || KIND_ICONS.repeated))); @@ -341,7 +349,7 @@ for (const [band, b] of s.bands) { Object.assign(band.style, { left: `${xAt(b.start)}px`, width: `${Math.max(2, xAt(b.end) - xAt(b.start))}px` }); } - const fullWidth = s.manual || !editable ? LABEL_W.text : LABEL_W.full; + const fullWidth = s.manual || s.byPhrase || !editable ? LABEL_W.text : LABEL_W.full; let placed = null; for (const [mode, width] of [["full", fullWidth], ["chip", LABEL_W.chip]]) { const row = rows.findIndex((taken) => free(taken, left, left + width)); diff --git a/podblock/web/templates/podcast.html b/podblock/web/templates/podcast.html index 5e47123..a8e67ac 100644 --- a/podblock/web/templates/podcast.html +++ b/podblock/web/templates/podcast.html @@ -177,6 +177,24 @@

Settings

+
+

Intro and outro phrases

+

Listened for in captions and transcripts. One phrase a line; case and punctuation don't matter.

+
+ + + +
+
+ {% if user.is_admin %}
diff --git a/tests/test_captions.py b/tests/test_captions.py index 08f1739..c89b4aa 100644 --- a/tests/test_captions.py +++ b/tests/test_captions.py @@ -184,3 +184,22 @@ def test_speaker_turns_are_saved_with_the_captions_and_compared_with_the_repeats from podblock.app import retention retention.clear(db, episode(db)) assert not list((setup.media_dir / "transcripts").iterdir()) # the turns go with the download + + +def test_captions_keep_each_words_time_and_an_intro_phrase_heard_cuts_again(db, setup, service): + from podblock.app import phrases + db.execute("UPDATE episodes SET duration = 20 WHERE id = 1") + db.execute("UPDATE podcasts SET intro_phrases = ? WHERE id = 1", (json.dumps(["You will hear AGAIN"]),)) + db.commit() + captions.queue(db, setup, [1], None) + assert worker.run_job(setup, db, jobs.claim(db, kinds=(captions.KIND,))) == "done" + assert phrases.words(episode(db))[:3] == [(1.0, 1.3, "an"), (1.3, 1.6, "ad"), (1.6, 1.9, "you")] + assert phrases.for_episode(db, episode(db)) == [ + {"type": "intro", "start": 0.0, "end": 3.05, "phrase": "You will hear AGAIN"}] + queued = db.execute("SELECT kind, episode_id, payload FROM jobs WHERE state = 'queued'").fetchall() + assert [tuple(r) for r in queued] == [("process", 1, '{"recut": true}')] + assert episode(db)["state_note"] == "Waiting to be cut again: an intro or outro phrase was heard" + + from podblock.app import retention + retention.clear(db, episode(db)) + assert not list((setup.media_dir / "transcripts").iterdir()) # the word times go with the download diff --git a/tests/test_phrases.py b/tests/test_phrases.py new file mode 100644 index 0000000..cf7db6e --- /dev/null +++ b/tests/test_phrases.py @@ -0,0 +1,61 @@ +"""Intro and outro phrases: found in a transcript's words, and the cuts they make.""" + +import json + +import pytest + +from podblock.app import phrases + + +def said(start, text, each=0.5): + return [(start + i * each, start + (i + 1) * each - 0.1, w) for i, w in enumerate(phrases.tokens(text))] + + +def test_words_match_without_case_or_punctuation(): + assert phrases.tokens("Let’s get INTO it!") == ["lets", "get", "into", "it"] + assert phrases.parse(" let's get into it \n\nLETS GET INTO IT\nthat's all, folks\n") == [ + "let's get into it", "that's all, folks"] + for bad in ("?!", "x" * 101, "\n".join(f"phrase {i}" for i in range(21))): + with pytest.raises(phrases.PhraseError): + phrases.parse(bad) + + +def test_an_intro_ends_at_its_first_hearing_and_an_outro_starts_at_its_last(): + heard = [*said(0, "welcome back everybody lets get into it today we talk"), + *said(300, "and lets get into it once more"), # past the first third: not the intro's end + *said(500, "thanks for listening see you next week"), + *said(560, "thanks for listening this was a production of somebody")] + found = phrases.find(heard, 600, ["let's get into it", "never said"], ["Thanks for listening!"]) + assert found == [ + {"type": "intro", "start": 0.0, "end": 3.5, "phrase": "let's get into it"}, + {"type": "outro", "start": pytest.approx(559.75), "end": 600, "phrase": "Thanks for listening!"}, + ] + assert found[0]["end"] == heard[7][0] # padded, but not into the next word + + assert phrases.find(said(300, "lets get into it"), 600, ["lets get into it"], []) == [] + assert phrases.find(said(100, "thanks for listening"), 600, [], ["thanks for listening"]) == [] # too early + assert phrases.find([], 600, ["anything"], ["anything"]) == [] + + +def test_a_feeds_transcript_times_words_by_where_they_fall_in_a_cue(tmp_path): + vtt = tmp_path / "t.vtt" + vtt.write_text("WEBVTT\n\n00:00:10.000 --> 00:00:14.000\nOkay, let's & go\n") + ep = {"transcripts": json.dumps([{"format": "vtt", "path": str(vtt)}])} + words = phrases.words(ep) + assert [w for _, _, w in words] == ["okay", "lets", "go"] + assert words[0][0] == pytest.approx(10) and words[-1][1] == pytest.approx(14) + assert words[1][0] == pytest.approx(10 + 4 * 6 / 16) + + js = tmp_path / "t.json" + js.write_text(json.dumps({"segments": [{"startTime": 1, "endTime": 2, "body": "one two"}, + {"startTime": 5, "body": "three"}]})) + words = phrases.words({"transcripts": json.dumps([{"format": "json", "path": str(js)}])}) + assert [(w, round(a, 2)) for a, _, w in words] == [("one", 1), ("two", 1.57), ("three", 5)] + + +def test_captions_word_times_win_over_their_cues(tmp_path): + vtt, timed = tmp_path / "c.vtt", tmp_path / "c.words.json" + vtt.write_text("WEBVTT\n\n00:00:00.000 --> 00:00:09.000\nwrong times\n") + timed.write_text(json.dumps([[1.0, 1.4, " That's"], [1.5, 2.0, " it."]])) + ep = {"transcripts": json.dumps([{"format": "vtt", "path": str(vtt), "words": str(timed)}])} + assert phrases.words(ep) == [(1.0, 1.4, "thats"), (1.5, 2.0, "it")] diff --git a/tests/test_processing.py b/tests/test_processing.py index 8be8f63..2f65f64 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -1025,6 +1025,37 @@ def test_chapters_and_transcripts_fit_the_cut_audio(env): assert not saved.exists() and episode(env, "ep-3")["source_chapters"] is None +def test_an_outro_phrase_in_the_transcript_cuts_it_and_everything_after(env): + with_extras(env) + res = env.client.post(f"/podcasts/{env.pod}/phrases", data={"intro": "", "outro": "AFTER the ad.\n"}) + assert res.status_code == 303 + processed(env) + three = episode(env, "ep-3") + total = JINGLE + AD + 2 * FILLER + assert json.loads(three["phrase_cuts"]) == [ + {"type": "outro", "start": 79.75, "end": pytest.approx(total, abs=0.5), "phrase": "AFTER the ad."}] + assert three["cut_duration"] == pytest.approx(79.75 - JINGLE - AD, abs=3) + assert episode(env, "ep-2")["phrase_cuts"] == "[]" # no transcript to hear it in + assert "After the ad" not in env.client.get(f"/episodes/{three['id']}/captions.vtt").text + + page = env.client.get(f"/episodes/{three['id']}").text + data = json.loads(re.search(r'id="timeline-data"[^>]*>(.*?)', page, re.S)[1]) + assert [(p["id"], p["type"], p["start"]) for p in data["phrases"]] == [("p0", "outro", 79.75)] + settings = env.client.get(f"/podcasts/{env.pod}?tab=settings").text + assert ">AFTER the ad." in settings + + # Taking the phrase away cuts again only the episode it changes. + res = env.client.post(f"/podcasts/{env.pod}/phrases", data={"intro": "", "outro": ""}) + assert res.status_code == 303 and episode(env, "ep-3")["state"] == "queued" + assert episode(env, "ep-2")["state"] == "ready" + worker.run_queue(env.settings) + assert episode(env, "ep-3")["cut_duration"] == pytest.approx(total - JINGLE - AD, abs=3) + + bad = env.client.post(f"/podcasts/{env.pod}/phrases", data={"intro": "?!", "outro": "kept"}) + assert bad.status_code == 303 and "outro=kept" in bad.headers["location"] + assert ">kept" in env.client.get(bad.headers["location"]).text # what was typed, to fix + + def test_episodes_cut_before_chapters_get_them_when_cut_again(env): processed(env) env.db.execute("UPDATE episodes SET cut_chapters = NULL, source_chapters = NULL") # as cut by an older version diff --git a/tests/test_schema.py b/tests/test_schema.py index eb3d304..db034a8 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -30,7 +30,7 @@ SHIPPED = ["9e41bd10322d1b18", "25b49003da210e9e", "a3b543510a6e79fe", "52c4ef83a590799a", "95f4530505c45b89", "da86379def3017da", "579499483c73fce2", "8584ad1da8ec8f0d", "0ecaa28be5177004", "3d3d0fb58e988193", "894c592d023cff02", "2279df03ba58202e", "6746b5025296d568", "03af274d2e0e3f79", "a4894b75bb3c8de0", - "5dc3b8c724b904c2"] + "5dc3b8c724b904c2", "6b5eaf907368114f"] def tables(db: sqlite3.Connection) -> set[str]: diff --git a/tests/test_web.py b/tests/test_web.py index b6dcf0c..b83c6a8 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -232,7 +232,10 @@ def test_the_shared_feed_becomes_the_admins_on_upgrade(tmp_path): DROP TABLE grabs; ALTER TABLE episodes DROP COLUMN source_chapters; ALTER TABLE episodes DROP COLUMN transcripts; ALTER TABLE episodes DROP COLUMN cut_chapters; - ALTER TABLE episodes DROP COLUMN held_by;""") # undo every step from per-user feeds on + ALTER TABLE episodes DROP COLUMN held_by; + ALTER TABLE podcasts DROP COLUMN intro_phrases; + ALTER TABLE podcasts DROP COLUMN outro_phrases; + ALTER TABLE episodes DROP COLUMN phrase_cuts;""") # undo every step from per-user feeds on before = next(n for n, m in enumerate(MIGRATIONS) if isinstance(m, str) and "CREATE TABLE user_feeds" in m) db.execute("UPDATE settings SET value = ? WHERE key = 'app_schema'", (str(before),)) db.commit() From dd8ee004854b980e0f9187c29672fe01836797f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:13:26 +0000 Subject: [PATCH 2/5] Intro and outro phrases: repeats close together count as one, so a chant is cut whole A show's outro chant sung three times was cut only from its last time, leaving the first two in. Hearings of any of a show's intro (or outro) phrases less than 20 seconds apart now make one run, and the cut covers the whole run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015gCh3c3PYL5Y4shMugurtB --- WEBAPP.md | 3 ++- podblock/app/phrases.py | 39 +++++++++++++++++++++++---------------- tests/test_phrases.py | 14 ++++++++++++++ 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/WEBAPP.md b/WEBAPP.md index 2d49bd3..6e9fab7 100644 --- a/WEBAPP.md +++ b/WEBAPP.md @@ -248,7 +248,8 @@ on Kubernetes, with backups and upgrades, see `deployment/README.md`. with a timed transcript is searched for them each time it is cut, ignoring case and punctuation. An intro phrase heard in the first third cuts from the start through its first hearing; an outro phrase heard in - the last third cuts from its last hearing to the end. Captions place the + the last third cuts from its last hearing to the end. Hearings under 20 + seconds apart count as one, so a chant repeated is cut whole. Captions place the cut between words; a feed's transcript, timed by cue, places it by where the words fall in their cue. Captions arrive after the first cut, so hearing a phrase in them queues the episode to be cut again, and saving diff --git a/podblock/app/phrases.py b/podblock/app/phrases.py index 13caf73..27661e0 100644 --- a/podblock/app/phrases.py +++ b/podblock/app/phrases.py @@ -10,7 +10,9 @@ - an outro phrase heard in the last third cuts from the phrase's first word to the end, at its last hearing. -So a phrase said again mid-episode cuts nothing. Matching ignores case and +Hearings less than CHAIN seconds apart count as one, so a chant sung over +and over is cut whole, and so is a host saying the line twice. A phrase +said again mid-episode cuts nothing. Matching ignores case and punctuation ("Let's get into it!" is "lets get into it"), word for word. Captions keep whisper's time for each word (`words.json` beside them), so @@ -38,6 +40,7 @@ MAX_PHRASES = 20 # of each kind, per podcast MAX_CHARS = 100 # in a phrase REACH = 1 / 3 # of the episode, from its edge, where a phrase counts +CHAIN = 20.0 # seconds between two hearings of a phrase that still make one run of it, as in a chant PAD = 0.25 # seconds past a phrase's edge also cut, when the next word isn't nearer: whisper's times run tight INTRO, OUTRO = "intro", "outro" @@ -134,6 +137,11 @@ def _hits(said: list[str], phrase: list[str]) -> list[int]: return [i for i in range(len(said) - n + 1) if said[i:i + n] == phrase] +def _all_hits(said: list[str], phrases: list[str]) -> list[tuple[int, int, str]]: + """Every hearing of any of the phrases, in order, as (index of its first word, of its last, phrase).""" + return sorted((i, i + len(tokens(p)) - 1, p) for p in phrases for i in _hits(said, tokens(p))) + + def find(heard: list[tuple[float, float, str]], duration: float, intro: list[str], outro: list[str]) -> list[dict]: """The cuts the phrases make: at most one intro and one outro, as {type, start, end, phrase}.""" @@ -141,24 +149,23 @@ def find(heard: list[tuple[float, float, str]], duration: float, intro: list[str return [] said = [w for _, _, w in heard] found = [] - first = None # (index of the phrase's last word, phrase) - for phrase in intro: - hits = _hits(said, tokens(phrase)) - last = hits[0] + len(tokens(phrase)) - 1 if hits else None - if last is not None and heard[last][1] <= duration * REACH and (first is None or last < first[0]): - first = (last, phrase) - if first is not None: - at, phrase = first + hits = [h for h in _all_hits(said, intro) if heard[h[1]][1] <= duration * REACH] + if hits: + at, phrase = hits[0][1], hits[0][2] + for first, last, _ in hits[1:]: # said again straight after: a chant, or a host repeating it + if heard[first][0] - heard[at][1] > CHAIN: + break + at = max(at, last) nxt = heard[at + 1][0] if at + 1 < len(heard) else duration end = min(max(heard[at][1], min(heard[at][1] + PAD, nxt)), duration) found.append({"type": INTRO, "start": 0.0, "end": round(end, 2), "phrase": phrase}) - final = None # (index of the phrase's first word, phrase) - for phrase in outro: - hits = _hits(said, tokens(phrase)) - if hits and heard[hits[-1]][0] >= duration * (1 - REACH) and (final is None or hits[-1] > final[0]): - final = (hits[-1], phrase) - if final is not None: - at, phrase = final + hits = [h for h in _all_hits(said, outro) if heard[h[0]][0] >= duration * (1 - REACH)] + if hits: + at, phrase = hits[-1][0], hits[-1][2] + for first, last, said_as in reversed(hits[:-1]): + if heard[at][0] - heard[last][1] > CHAIN: + break + at, phrase = min(at, first), said_as prev = heard[at - 1][1] if at > 0 else 0.0 start = max(min(heard[at][0], max(heard[at][0] - PAD, prev)), 0.0) if not found or start > found[0]["end"]: diff --git a/tests/test_phrases.py b/tests/test_phrases.py index cf7db6e..ce4a549 100644 --- a/tests/test_phrases.py +++ b/tests/test_phrases.py @@ -59,3 +59,17 @@ def test_captions_word_times_win_over_their_cues(tmp_path): timed.write_text(json.dumps([[1.0, 1.4, " That's"], [1.5, 2.0, " it."]])) ep = {"transcripts": json.dumps([{"format": "vtt", "path": str(vtt), "words": str(timed)}])} assert phrases.words(ep) == [(1.0, 1.4, "thats"), (1.5, 2.0, "it")] + + +def test_a_chant_sung_over_and_over_is_cut_whole(): + chant = "we are we are the youth of the nation" + heard = [*said(0, "and that is the end of the story"), + *said(800, "thanks everybody"), + *said(880, chant), *said(886, "youth of the nation"), *said(891, chant), *said(897, "yeah")] + found = phrases.find(heard, 900, [], ["We are, we are, the youth of the nation", "youth of the nation"]) + assert found == [{"type": "outro", "start": pytest.approx(879.75), "end": 900, + "phrase": "We are, we are, the youth of the nation"}] + + intro = [*said(0, chant), *said(5, chant), *said(10, "welcome to the show"), *said(60, chant)] + assert phrases.find(intro, 900, [chant], [])[0]["end"] == pytest.approx(9.65) # through the second, not the third + assert phrases.find(said(880, "youth of a nation"), 900, [], [chant]) == [] # every word has to match From df98f87b23d6ed9ac5094e3e297628587a8ec2d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 16:28:20 +0000 Subject: [PATCH 3/5] Intro and outro phrases: a phrase between slashes is a regular expression `/youth of (the|a) nation/` is matched against the transcript's words as plain phrases are: lower case, without punctuation, one space apart. It matches whole words only, and at most 30 words at once, so a loose pattern can't reach far. Invalid patterns, and ones that match nothing at all, are refused when saving. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015gCh3c3PYL5Y4shMugurtB --- WEBAPP.md | 4 ++- podblock/app/phrases.py | 56 ++++++++++++++++++++++++++++- podblock/web/templates/podcast.html | 3 +- tests/test_phrases.py | 17 +++++++++ tests/test_processing.py | 8 ++--- 5 files changed, 81 insertions(+), 7 deletions(-) diff --git a/WEBAPP.md b/WEBAPP.md index 6e9fab7..a46cc32 100644 --- a/WEBAPP.md +++ b/WEBAPP.md @@ -249,7 +249,9 @@ on Kubernetes, with backups and upgrades, see `deployment/README.md`. ignoring case and punctuation. An intro phrase heard in the first third cuts from the start through its first hearing; an outro phrase heard in the last third cuts from its last hearing to the end. Hearings under 20 - seconds apart count as one, so a chant repeated is cut whole. Captions place the + seconds apart count as one, so a chant repeated is cut whole. A phrase + between slashes is a regular expression over the same lower-case words, + matching whole words and at most 30 at a time. Captions place the cut between words; a feed's transcript, timed by cue, places it by where the words fall in their cue. Captions arrive after the first cut, so hearing a phrase in them queues the episode to be cut again, and saving diff --git a/podblock/app/phrases.py b/podblock/app/phrases.py index 27661e0..0f04c00 100644 --- a/podblock/app/phrases.py +++ b/podblock/app/phrases.py @@ -15,6 +15,12 @@ said again mid-episode cuts nothing. Matching ignores case and punctuation ("Let's get into it!" is "lets get into it"), word for word. +A phrase between slashes is a regular expression instead, matched against +the same words: lower case, without punctuation, one space apart. It +matches whole words only, and at most MAX_WORDS of them at once, so +`/youth of (the|a) nation/` matches "the youth of the nation" but not +"youth of the nations". + Captions keep whisper's time for each word (`words.json` beside them), so cuts land between words; a feed's transcript only times whole cues, so there each word's time is guessed from where it falls in its cue. @@ -27,6 +33,8 @@ from __future__ import annotations +import bisect +import functools import json import re import sqlite3 @@ -39,6 +47,7 @@ MAX_PHRASES = 20 # of each kind, per podcast MAX_CHARS = 100 # in a phrase +MAX_WORDS = 30 # a regular expression's match spans at most this many words REACH = 1 / 3 # of the episode, from its edge, where a phrase counts CHAIN = 20.0 # seconds between two hearings of a phrase that still make one run of it, as in a chant PAD = 0.25 # seconds past a phrase's edge also cut, when the next word isn't nearer: whisper's times run tight @@ -54,6 +63,15 @@ def tokens(text: str) -> list[str]: return re.findall(r"\w+", re.sub(r"['’ʼ]", "", text.casefold())) +@functools.lru_cache(maxsize=256) +def pattern(phrase: str) -> re.Pattern | None: + """A phrase between slashes as the regular expression it matches the words with; None for a plain phrase. + Raises re.error for one that isn't valid.""" + if len(phrase) < 3 or not (phrase.startswith("/") and phrase.endswith("/")): + return None + return re.compile(rf"(? list[str]: """Phrases from a box of them, one a line. Blank lines and repeats go.""" found: dict[tuple, str] = {} @@ -63,6 +81,15 @@ def parse(text: str) -> list[str]: continue if len(line) > MAX_CHARS: raise PhraseError(f"Phrases can be at most {MAX_CHARS} characters.") + try: + compiled = pattern(line) + except re.error as e: + raise PhraseError(f"“{line}” isn't a valid regular expression: {e}.") from None + if compiled is not None: + if compiled.search("") or compiled.search(" "): + raise PhraseError(f"“{line}” matches nothing at all; it needs a word.") + found.setdefault(("/", line), line) + continue if not tokens(line): raise PhraseError(f"“{line}” has no words to listen for.") found.setdefault(tuple(tokens(line)), line) @@ -137,9 +164,36 @@ def _hits(said: list[str], phrase: list[str]) -> list[int]: return [i for i in range(len(said) - n + 1) if said[i:i + n] == phrase] +def _regex_hits(said: list[str], compiled: re.Pattern) -> list[tuple[int, int]]: + """Where a regular expression matches the words, as (index of the first word, of the last).""" + text = " ".join(said) + starts = [0] + for w in said[:-1]: + starts.append(starts[-1] + len(w) + 1) + found = [] + for m in compiled.finditer(text): + if m.end() <= m.start(): + continue + first = bisect.bisect_right(starts, m.start()) - 1 + last = bisect.bisect_right(starts, m.end() - 1) - 1 + if last - first < MAX_WORDS: + found.append((first, last)) + return found + + def _all_hits(said: list[str], phrases: list[str]) -> list[tuple[int, int, str]]: """Every hearing of any of the phrases, in order, as (index of its first word, of its last, phrase).""" - return sorted((i, i + len(tokens(p)) - 1, p) for p in phrases for i in _hits(said, tokens(p))) + found = [] + for p in phrases: + try: + compiled = pattern(p) + except re.error: + continue + if compiled is not None: + found += [(a, b, p) for a, b in _regex_hits(said, compiled)] + else: + found += [(i, i + len(tokens(p)) - 1, p) for i in _hits(said, tokens(p))] + return sorted(found) def find(heard: list[tuple[float, float, str]], duration: float, intro: list[str], diff --git a/podblock/web/templates/podcast.html b/podblock/web/templates/podcast.html index a8e67ac..b04754f 100644 --- a/podblock/web/templates/podcast.html +++ b/podblock/web/templates/podcast.html @@ -179,7 +179,8 @@

Settings

Intro and outro phrases

-

Listened for in captions and transcripts. One phrase a line; case and punctuation don't matter.

+

Listened for in captions and transcripts. One phrase a line; case and punctuation don't matter. + Between slashes, a regular expression over the words: /youth of (the|a) nation/.