caption.js 2.3.0: dual-speaker captions, leading sentences, and the speaker of each line

README.md

@@ -400,9 +400,13 @@ cap1.init("hypertranscript", "hyperplayer", '37', '21', undefined, undefined, nu
   joinSentences: true,              // a short sentence shares the caption before it when the whole sentence fits
   maxJoinGap: 1,                    // ...unless more than this many seconds of silence separate them
   paragraphBreaks: true,            // a new paragraph always starts a new caption
+  dualSpeakers: true,               // two speakers may share a caption: one per line, each line opening with a hyphen
+  leadSentences: true,              // a short sentence with nothing to join leads the long sentence after it
 });
 ```
 
+Each cue in the result's `data` also carries `speakers`: the speaker of each of its lines (`["Ann", "Bob"]` for a dual-speaker caption, `""` where the transcript names none).
+
 ## :money_with_wings: Web Monetization Support :money_with_wings:
 
 [Web Monetization](https://webmonetization.org/) is a browser API, stewarded by the [Interledger Foundation](https://interledger.org/), that lets visitors stream micropayments to the sites they're reading. There is currently no native browser support — visitors need a [Web Monetization agent](https://webmonetization.org/supporters/get-started/) (browser extension) installed to actually pay.

__TEST__/caption.test.js

@@ -389,3 +389,144 @@ test("joined captions never overlap and keep the cue count honest", () => {
   expect(cues.map((c) => c.text).join(" "))
     .toBe("Yes. No. Maybe. I see. Go on. This one is a good deal longer than a line allows. Fine.");
 });
+
+// ---- 2.3.0: the speaker of each line, dual-speaker captions, leading sentences
+
+const lineTexts = (result) => result.data.map((c) => c.text.replace(/\n+$/, "").split("\n").map((l) => l.trim()));
+const INTERVIEW = "@speaker-A to start? | @speaker-B Sure. So to start off, Dr. Ashby, can you just introduce yourself and give us a little insight into your background?";
+
+test("with no new options, captions are exactly what 2.2.0 generated", () => {
+  buildFromText(INTERVIEW);
+  const plain = run({ joinSentences: true, detectAbbreviations: true, abbreviations: ["Dr."] });
+  expect(cueTexts(plain).slice(0, 2)).toEqual(["to start?", "Sure."]);
+  expect(plain.vtt).not.toContain("-to start?");
+  // a label repeating the speaker already talking still starts a new caption
+  buildFromText("@Ann Yes. @Ann No.");
+  expect(cueTexts(run({ joinSentences: true }))).toEqual(["Yes.", "No."]);
+});
+
+test("every cue names the speaker of each of its lines, deep into a turn", () => {
+  buildFromText("@Ann This sentence is far too long to sit on one line of a caption. Yes. | @Bob No. Never. It was not like that at all in those days, I can tell you.");
+  const result = run({ joinSentences: true });
+  result.data.forEach((cue) => {
+    expect(cue.speakers).toHaveLength(cue.text.replace(/\n+$/, "").split("\n").length);
+  });
+  const bySpeaker = (name) => result.data.filter((c) => c.speakers.every((s) => s === name)).map((c) => flat(c.text)).join(" ");
+  expect(bySpeaker("Ann")).toBe("This sentence is far too long to sit on one line of a caption. Yes.");
+  expect(bySpeaker("Bob")).toBe("No. Never. It was not like that at all in those days, I can tell you.");
+  // label styles: "[Ann] " and "Ann: " both read as Ann; no label at all is ''
+  document.body.innerHTML = '<div id="transcript"><p><span class="speaker" data-m="0">Ann: </span><span data-m="0" data-d="300">Yes. </span></p></div>';
+  expect(run().data[0].speakers).toEqual(["Ann"]);
+  buildFromText("Yes.");
+  expect(run().data[0].speakers).toEqual([""]);
+});
+
+test("dualSpeakers: two speakers share a caption, one per line, each line with a hyphen", () => {
+  buildFromText(INTERVIEW);
+  const result = run({ joinSentences: true, dualSpeakers: true, detectAbbreviations: true, abbreviations: ["Dr."] });
+  expect(lineTexts(result)[0]).toEqual(["-to start?", "-Sure."]);
+  expect(result.data[0].speakers).toEqual(["speaker-A", "speaker-B"]);
+  // the caption runs from the first speaker's first word to the second's last
+  const cues = parseVtt(result.vtt);
+  expect(cues[0].start).toBe("00:00:00.000");
+  expect(cues[0].stop).toBe("00:00:01.150"); // "Sure." starts at 0.8s, lasts 350ms
+  // the long sentence after it is one speaker's: no hyphen anywhere else
+  result.data.slice(1).forEach((cue) => {
+    expect(cue.text).not.toMatch(/(^|\n)-/);
+    expect(new Set(cue.speakers)).toEqual(new Set(["speaker-B"]));
+  });
+  expect(result.srt).toMatch(/-to start\? ?\n-Sure\./);
+});
+
+test("dualSpeakers: a hyphen marks a shared caption, never just a change of speaker", () => {
+  // the pause is too long to share: two captions, neither with a hyphen
+  buildTranscript([
+    ["speaker", "[Ann] "], [0, 300, "Ready?"],
+    ["speaker", "[Bob] "], [3000, 300, "Sure."],
+  ]);
+  const apart = run({ dualSpeakers: true });
+  expect(cueTexts(apart)).toEqual(["Ready?", "Sure."]);
+  expect(apart.data.map((c) => c.speakers)).toEqual([["Ann"], ["Bob"]]);
+});
+
+test("dualSpeakers: two speakers at most, whole sentences only, and the hyphen counts", () => {
+  // a third turn starts a new caption
+  buildFromText("@Ann Ready? @Bob Sure. @Ann Good.");
+  expect(lineTexts(run({ dualSpeakers: true }))).toEqual([["-Ready?", "-Sure."], ["Good."]]);
+
+  // several short sentences from one speaker can make up a side
+  buildFromText("@Ann Yes. No. @Bob Fine. Go on.");
+  const sides = run({ joinSentences: true, dualSpeakers: true });
+  expect(lineTexts(sides)).toEqual([["-Yes. No.", "-Fine. Go on."]]);
+  expect(sides.data[0].speakers).toEqual(["Ann", "Bob"]);
+
+  // the tail of a long sentence is not a whole sentence: nothing shares with it
+  buildFromText("@Ann This sentence is far too long to sit on one line. @Bob Sure.");
+  const tail = lineTexts(run({ dualSpeakers: true }));
+  expect(tail[tail.length - 1]).toEqual(["Sure."]);
+
+  // a line filled to the limit by joining has no room left for its hyphen
+  buildFromText("@Ann Yes. No. @Bob Fine.");
+  expect(lineTexts(run({ joinSentences: true, dualSpeakers: true }, 9, 5))).toEqual([["Yes. No."], ["Fine."]]);
+  buildFromText("@Ann Yes. No. @Bob Fine.");
+  expect(lineTexts(run({ joinSentences: true, dualSpeakers: true }, 10, 5))).toEqual([["-Yes. No.", "-Fine."]]);
+});
+
+test("dualSpeakers: paragraphBreaks wins, and a transcript with no labels is untouched", () => {
+  buildFromText("@Ann Ready? | @Bob Sure.");
+  expect(lineTexts(run({ dualSpeakers: true, paragraphBreaks: true }))).toEqual([["Ready?"], ["Sure."]]);
+  buildFromText("Yes. No. Maybe. I see.");
+  const withOption = run({ joinSentences: true, dualSpeakers: true, leadSentences: true });
+  buildFromText("Yes. No. Maybe. I see.");
+  expect(withOption.vtt).toBe(run({ joinSentences: true }).vtt);
+});
+
+test("leadSentences: a short sentence leads the long sentence after it", () => {
+  buildFromText("@speaker-B Sure. So to start off, Dr. Ashby, can you just introduce yourself and give us a little insight into your background?");
+  const options = { joinSentences: true, detectAbbreviations: true, abbreviations: ["Dr."] };
+  expect(cueTexts(run(options))[0]).toBe("Sure.");
+  const led = run({ ...options, leadSentences: true });
+  expect(cueTexts(led)[0].startsWith("Sure. So to start off,")).toBe(true);
+  // nothing lost, nothing over the line length, one speaker throughout
+  expect(cueTexts(led).join(" ")).toBe("Sure. So to start off, Dr. Ashby, can you just introduce yourself and give us a little insight into your background?");
+  lineTexts(led)[0].forEach((line) => expect(line.length).toBeLessThanOrEqual(32));
+  led.data.forEach((cue) => expect(new Set(cue.speakers)).toEqual(new Set(["speaker-B"])));
+  expect(parseVtt(led.vtt)[0].start).toBe("00:00:00.000");
+});
+
+test("leadSentences: not across a speaker, a long pause, or a paragraph that must break; joining back comes first", () => {
+  const LONG = "This sentence is far too long to sit on one line of a caption.";
+  buildFromText(`Sure. @Bob ${LONG}`);
+  expect(cueTexts(run({ leadSentences: true }))[0]).toBe("Sure.");
+
+  buildFromText(`Sure. | ${LONG}`);
+  expect(cueTexts(run({ leadSentences: true, paragraphBreaks: true }))[0]).toBe("Sure.");
+  expect(cueTexts(run({ leadSentences: true }))[0]).not.toBe("Sure.");
+
+  buildTranscript([[0, 300, "Sure."], ...LONG.split(" ").map((w, i) => [5000 + i * 400, 350, w])]);
+  expect(cueTexts(run({ leadSentences: true }))[0]).toBe("Sure.");
+
+  // "Yes." has a caption behind it to join: it goes back, not forward
+  buildFromText(`No. Yes. ${LONG}`);
+  expect(cueTexts(run({ joinSentences: true, leadSentences: true }))[0]).toBe("No. Yes.");
+});
+
+test("together: the interview opening, as the pause allows", () => {
+  const options = { joinSentences: true, dualSpeakers: true, leadSentences: true, detectAbbreviations: true, abbreviations: ["Dr."] };
+  buildFromText(INTERVIEW);
+  expect(lineTexts(run(options))[0]).toEqual(["-to start?", "-Sure."]);
+
+  // a long pause before the answer: no shared caption, and "Sure." leads instead
+  const answer = "Sure. So to start off, Dr. Ashby, can you just introduce yourself?".split(" ");
+  buildTranscript([
+    ["speaker", "[speaker-A] "], [0, 300, "to"], [350, 300, "start?"],
+    ["speaker", "[speaker-B] "], ...answer.map((w, i) => [4000 + i * 400, 350, w]),
+  ]);
+  const apart = run(options);
+  expect(cueTexts(apart)[0]).toBe("to start?");
+  expect(cueTexts(apart)[1].startsWith("Sure. So to start off,")).toBe(true);
+  expect(apart.vtt).not.toMatch(/\n-/);
+  // and no cue overlaps the next
+  const cues = parseVtt(apart.vtt);
+  for (let i = 1; i < cues.length; i += 1) expect(cues[i].start >= cues[i - 1].stop).toBe(true);
+});

js/caption.d.ts

@@ -7,6 +7,8 @@ export interface CaptionCue {
   /** "HH:MM:SS.mmm" */
   stop: string;
   text: string;
+  /** The speaker of each line of `text`, '' where the transcript names none. */
+  speakers: string[];
 }
 
 export interface CaptionsResult {
@@ -38,6 +40,17 @@ export interface CaptionOptions {
   maxJoinGap?: number;
   /** A new paragraph always starts a new caption. */
   paragraphBreaks?: boolean;
+  /**
+   * Two short sentences from different speakers may share a caption: two
+   * speakers at most, one per line, each line opening with a hyphen. A
+   * caption with one speaker never has a hyphen. `paragraphBreaks` wins.
+   */
+  dualSpeakers?: boolean;
+  /**
+   * A short sentence that could not join the caption before it leads the long
+   * sentence after it, when both are one speaker's and within maxJoinGap.
+   */
+  leadSentences?: boolean;
 }
 
 export interface CaptionInstance {

js/caption.js

@@ -1,5 +1,5 @@
 /*! (C) The Hyperaudio Project. MIT @license: en.wikipedia.org/wiki/MIT_License. */
-/*! Version 2.2.0 */
+/*! Version 2.3.0 */
 'use strict';
 
 const caption = function () {
@@ -47,6 +47,18 @@ const caption = function () {
   //                        joined across a speaker label or a long pause.
   //   maxJoinGap           seconds of silence that still allow a join (default 1)
   //   paragraphBreaks      true: a new paragraph always starts a new caption
+  //   dualSpeakers         true: two short sentences from different speakers may
+  //                        share a caption, in the Netflix form - two speakers
+  //                        at most, one per line, each line opening with a
+  //                        hyphen. The hyphen marks two speakers SHARING a
+  //                        caption, not a change of speaker: a caption with
+  //                        one speaker never has one. paragraphBreaks wins.
+  //   leadSentences        true: a short sentence that could not join the
+  //                        caption before it leads the long sentence after
+  //                        it, when both are one speaker's and close in time.
+  //
+  // Every cue in the result's data carries `speakers`: the speaker of each of
+  // its lines, '' where the transcript names none.
   cap.init = function (transcriptId, playerId, maxLength, minLength, label, srclang, parent, options) {
 
     const opts = options || {};
@@ -58,6 +70,8 @@ const caption = function () {
     const joinSentences = opts.joinSentences === true;
     const paragraphBreaks = opts.paragraphBreaks === true;
     const maxJoinGap = typeof opts.maxJoinGap === 'number' && opts.maxJoinGap >= 0 ? opts.maxJoinGap : 1;
+    const dualSpeakers = opts.dualSpeakers === true;
+    const leadSentences = opts.leadSentences === true;
 
     let transcript = document.getElementById(transcriptId);
 
@@ -159,11 +173,18 @@ const caption = function () {
       minLineLength = minLength;
     }
 
+    // A label as the transcript writes it ("[Ann] ", "Ann: ") to a name.
+    const speakerName = (label) => String(label).trim().replace(/^\[(.*)\]$/, '$1').replace(/:$/, '').trim();
+    // Only the sentence a label opens has `speaker` set; the name is carried
+    // through the rest of the turn here, so every caption knows whose it is.
+    let currentSpeaker = '';
+
     words.forEach((word, i) => {
       if (thisSegmentMeta === null) {
         // create segment meta object
         thisSegmentMeta = new segmentMeta('', null, 0, 0, 0);
       }
+      thisSegmentMeta.speakerName = currentSpeaker;
 
       if (word.classList.contains('speaker')) {
         // checking that this is not a new segment AND a new empty segment wasn't already created
@@ -175,6 +196,8 @@ const caption = function () {
         // textContent, not innerText: identical for plain transcript spans,
         // doesn't force a layout pass, and works in jsdom for tests.
         thisSegmentMeta.speaker = word.textContent;
+        currentSpeaker = speakerName(word.textContent);
+        thisSegmentMeta.speakerName = currentSpeaker;
       } else {
         // A new paragraph (paragraphBreaks): close the sentence in hand, so no
         // caption runs across the break even where the paragraph ended without
@@ -186,6 +209,7 @@ const caption = function () {
           if (thisSegmentMeta.start !== null) {
             data.segments.push(thisSegmentMeta);
             thisSegmentMeta = new segmentMeta('', null, 0, 0, 0);
+            thisSegmentMeta.speakerName = currentSpeaker;
           }
           thisSegmentMeta.paragraphStart = true;
         }
@@ -252,6 +276,7 @@ const caption = function () {
       this.start = start;
       this.stop = stop;
       this.text = text;
+      this.speakers = []; // the speaker of each line; filled in once the lines are final
     }
 
     const captions = [];
@@ -262,16 +287,52 @@ const caption = function () {
     // timing safeguards may later extend.
     const speechEnd = new Map();
 
+    // Whose each caption is, and which captions hold nothing but whole
+    // sentences - only those may become one side of a dual-speaker caption.
+    const owner = new Map();
+    const lineSpeakers = new Map(); // set only where the lines differ
+    const wholeSentences = new Set();
+    const claim = (segment, from) => {
+      for (let c = from; c < captions.length; c++) owner.set(captions[c], segment.speakerName);
+    };
+
+    // dualSpeakers: a short sentence that opens another speaker's turn shares
+    // the one-line caption before it - "-to start?" / "-Sure." Each line is
+    // one speaker's whole sentences, and the hyphens count towards its length.
+    function joinAsSecondSpeaker(prev, segment, text, segmentStop) {
+      if (!dualSpeakers || paragraphBreaks) return false;
+      if (!wholeSentences.has(prev) || lineSpeakers.has(prev)) return false;
+      const first = prev.text.replace(/\n+$/, '');
+      if (first.includes('\n')) return false;
+      if (('-' + first).length > maxLineLength || ('-' + text).length > maxLineLength) return false;
+      lineSpeakers.set(prev, [owner.get(prev) || '', segment.speakerName]);
+      prev.text = '-' + first + '\n-' + text;
+      prev.stop = formatSeconds(segmentStop);
+      speechEnd.set(prev, segmentStop);
+      return true;
+    }
+
     // joinSentences: put a whole short sentence into the caption before it.
     // On the same line when it fits there, else as the second line of a
-    // one-line caption. Never across a speaker label, a paragraph that must
-    // break, or a pause longer than maxJoinGap.
+    // one-line caption. Never across a paragraph that must break or a pause
+    // longer than maxJoinGap, and across a speaker label only as the second
+    // speaker of a dual-speaker caption.
     function joinToPrevious(segment, text, segmentStop) {
       const prev = captions[captions.length - 1];
-      if (!joinSentences || prev === undefined) return false;
-      if (segment.speaker !== '' || segment.paragraphStart === true) return false;
+      if (prev === undefined || segment.paragraphStart === true) return false;
       const prevEnd = speechEnd.get(prev);
       if (prevEnd === undefined || segment.start - prevEnd > maxJoinGap) return false;
+      // A label always ends ordinary joining, as it did in 2.2.0. What it may
+      // do now is open the second line of a dual-speaker caption - when it
+      // really is another speaker, not the same name written again.
+      if (segment.speaker !== '') {
+        return segment.speakerName !== (owner.get(prev) || '')
+          && joinAsSecondSpeaker(prev, segment, text, segmentStop);
+      }
+      if (!joinSentences) return false;
+      // further sentences of a dual-speaker caption's second speaker go on
+      // their line; the first speaker's line is closed
+      if (lineSpeakers.has(prev) && lineSpeakers.get(prev)[1] !== segment.speakerName) return false;
 
       const lines = prev.text.replace(/\n+$/, '').split('\n');
       const last = lines[lines.length - 1];
@@ -317,6 +378,23 @@ const caption = function () {
           return; // the sentence went into the caption before it
         }
 
+        // leadSentences: nothing behind to join, and the same speaker's next
+        // sentence is too long for one line - lay the two out as one run, so
+        // "Sure." opens that sentence's first caption instead of standing
+        // alone. The long sentence is divided across captions either way.
+        const next = arr[i + 1];
+        if (leadSentences && next !== undefined && segment.chars < minLineLength
+            && next.speaker === '' && next.paragraphStart !== true
+            && next.chars >= maxLineLength && next.start - segmentStop <= maxJoinGap) {
+          next.words = segment.words.concat(next.words);
+          next.chars += segment.chars;
+          next.duration += segment.duration;
+          next.start = segment.start;
+          next.speaker = segment.speaker;               // it opens the turn now, if this did
+          next.paragraphStart = segment.paragraphStart;
+          return;
+        }
+
         thisCaption = new captionMeta(
           formatSeconds(segment.start),
           formatSeconds(segmentStop),
@@ -328,6 +406,8 @@ const caption = function () {
         //console.log(thisCaption);
         captions.push(thisCaption);
         speechEnd.set(thisCaption, segmentStop);
+        wholeSentences.add(thisCaption);
+        claim(segment, segmentStartCount);
         thisCaption = null;
       } else {
         // The number of chars in this segment is longer than our single line maximum
@@ -480,9 +560,16 @@ const caption = function () {
         if (captions.length > segmentStartCount && lastOutTime !== undefined) {
           speechEnd.set(captions[captions.length - 1], lastOutTime);
         }
+        claim(segment, segmentStartCount);
       }
     });
 
+    // the speaker of each line, now that the lines are final
+    captions.forEach((caption) => {
+      const lineCount = caption.text.replace(/\n+$/, '').split('\n').length;
+      caption.speakers = lineSpeakers.get(caption) || new Array(lineCount).fill(owner.get(caption) || '');
+    });
+
     // Enforce a comfortable minimum on-screen time for each cue. A cue is
     // extended (never shortened) so it lasts at least minCaptionDuration and
     // long enough to read its text at readingSpeedCps, but only as far as the