diff --git a/CHANGELOG.md b/CHANGELOG.md index 5416e55..2e3cd94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,8 @@ dialer's prospect is marked Do Not Contact. - The decision engine reads the calls in their history. Someone who opted out is always skipped; "not interested" on a recent call turns reach out - now into later; a wrong or bad number is no longer a way to reach them; + now into later; a wrong or bad number is no longer a way to reach them + for 90 days, unless their number has changed since; and skipping someone who was interested goes to you to check. - Quiz leads say how their last call went. - **Calls at their hour.** A call sheet for a prospect, in the app and from @@ -50,11 +51,12 @@ phone through ntfy within a minute of each qualified lead (several at once come as one), tapping it opens the lead; it looks once a minute across every server on the database, never pushes a lead twice, retries one that - failed, and never pushes leads from before it was set up. Settings, Lead - alerts sends a test. A lead who opted out anywhere, "do not call" on the - dialer included, is Do Not Contact: last on Calls, with no Call button, - and never pushed. Their answers read whether offer writes the JSON on one - line or several. + failed, never pushes leads from before it was set up, and takes more than + one look's 500 new leads over the next looks, oldest first, skipping + none. Settings, Lead alerts sends a test. A lead who opted out anywhere, + "do not call" on the dialer included, is Do Not Contact: last on Calls, + with no Call button, and never pushed. Their answers read whether offer + writes the JSON on one line or several. - **Picking a dialer prospect fills the script in again.** Prospects from `twenty:agencyProspects` and `twenty:agencyPeople` were never found by their key in a call sheet or a research run, so the script kept its @@ -65,8 +67,9 @@ delivered, and each watch is a prospect source, `listen:`: the trigger line is the post itself, quoted, and a post on X carries its author's handle for an X DM. **Save as a list** in Settings writes a - watch's matches to one of your own lists to fill in and research; saving - again only adds what is new. Prospect sources are defined once, so + watch's matches to one of your own lists to fill in and research, one + list per watch even when two share a name; saving again only adds what is + new. Prospect sources are defined once, so research and campaigns accept the same ones. - **X DMs go out from COLD.** Add your own X developer app in Settings (`X_CLIENT_ID`, `X_CLIENT_SECRET`) and press **Connect X** for an account: diff --git a/docs/decision-engine.md b/docs/decision-engine.md index 6e2d3fa..9d5211c 100644 --- a/docs/decision-engine.md +++ b/docs/decision-engine.md @@ -173,7 +173,8 @@ about them. The dialer's calls are part of what the model reads: each one in the history with its outcome, the dialer's AI score and tone, and its summary. A wrong or bad number stops counting the phone as a way to reach them, which lowers -their reach score. +their reach score: for 90 days, and only while their phone is still the number +that was dialed. Anyone marked Do Not Contact, or who ever opted out, is never researched or scored. diff --git a/src/domains/config/helpers/parseConfig.test.ts b/src/domains/config/helpers/parseConfig.test.ts index ec87eb2..bc3cf31 100644 --- a/src/domains/config/helpers/parseConfig.test.ts +++ b/src/domains/config/helpers/parseConfig.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import path from "node:path"; import { parse } from "yaml"; +import { DEFAULT_CALL_HOURS } from "../types"; import { DEFAULT_FOOTERS, parseConfig } from "./parseConfig"; const example = parse(readFileSync(path.resolve(import.meta.dir, "../../../../cold.example.yaml"), "utf8")); @@ -31,6 +32,8 @@ describe("parseConfig", () => { test("the calls section is optional: 8am to 9pm every day, in the prospect's time", () => { expect(parseConfig(example, "x").config!.calls).toEqual({ start: "08:00", end: "21:00", days: ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] }); + // Each config has its own copy of the default days: editing one never edits the defaults. + expect(parseConfig(example, "x").config!.calls.days).not.toBe(DEFAULT_CALL_HOURS.days); const tuned = parseConfig({ ...example, calls: { start: "09:00", end: "18:30", days: ["Mon", "tue", "wed", "thu", "fri"] } }, "x"); expect(tuned.config!.calls).toEqual({ start: "09:00", end: "18:30", days: ["mon", "tue", "wed", "thu", "fri"] }); expect(parseConfig({ ...example, calls: { end: "20:00" } }, "x").config!.calls.start).toBe("08:00"); diff --git a/src/domains/config/helpers/parseConfig.ts b/src/domains/config/helpers/parseConfig.ts index 11f1d38..b9e15d0 100644 --- a/src/domains/config/helpers/parseConfig.ts +++ b/src/domains/config/helpers/parseConfig.ts @@ -168,12 +168,15 @@ function isTimezone(name: string): boolean { } } +/** The default calling hours as a config's own copy, so nothing that edits one config's days edits the defaults. */ +const defaultCallHours = (): CallHours => ({ ...DEFAULT_CALL_HOURS, days: [...DEFAULT_CALL_HOURS.days] }); + /** The optional calls: section: when to call, in each prospect's own time. Every key has a default. */ function parseCalls(raw: unknown, errors: string[]): CallHours { - if (raw === undefined || raw === null) return { ...DEFAULT_CALL_HOURS, days: [...DEFAULT_CALL_HOURS.days] }; + if (raw === undefined || raw === null) return defaultCallHours(); if (!isObject(raw)) { errors.push("calls: must be a mapping (start, end, days)"); - return { ...DEFAULT_CALL_HOURS }; + return defaultCallHours(); } for (const key of Object.keys(raw)) if (!["start", "end", "days"].includes(key)) errors.push(`calls.${key}: not a calls setting (start, end, days)`); const start = text(raw.start) || DEFAULT_CALL_HOURS.start; diff --git a/src/domains/db/index.ts b/src/domains/db/index.ts index 695cff5..36e148b 100644 --- a/src/domains/db/index.ts +++ b/src/domains/db/index.ts @@ -367,7 +367,6 @@ export function databaseFromEnv(env: Record = proces return undefined; } -/** A value stored once and kept: the first call makes it, later calls read it. */ /** * Claim a job that runs now and then (a look for new leads, logging calls) * for every server on this database: true when it last ran at least @@ -384,6 +383,7 @@ export async function claimRun(db: Db, key: string, now: Date, everyMs: number): return changes > 0; } +/** A value stored once and kept: the first call makes it, later calls read it. */ export async function metaValue(db: Db, key: string, make: () => string): Promise { const found = await db.get<{ value: string }>("SELECT value FROM meta WHERE key = ?", key); if (found) return found.value; diff --git a/src/domains/decide/model.test.ts b/src/domains/decide/model.test.ts index 58477ec..439cd4d 100644 --- a/src/domains/decide/model.test.ts +++ b/src/domains/decide/model.test.ts @@ -5,6 +5,8 @@ import type { Dossier } from "./diligence"; import { allowedAngles, combine, decisionState, outboundQuestions, reachableOn, scoreOf, tierOf, type DecisionInput } from "./model"; import type { Answer, DecisionResult } from "./types"; +const NOW = new Date("2026-10-31T12:00:00Z"); + const prospect = (extra: Partial = {}): Prospect => ({ key: "csv:1", identities: ["csv:1", "email:dana@clearview.example"], @@ -93,7 +95,7 @@ describe("the questions", () => { }); test("an email that bounced is not a way to reach them", () => { - expect(reachableOn(prospect(), [{ kind: "BOUNCED", channel: "EMAIL", prospectKey: "csv:1", occurredAt: "2026-10-01T10:00:00Z" } as Touch])).toEqual(["phone"]); + expect(reachableOn(prospect(), [{ kind: "BOUNCED", channel: "EMAIL", prospectKey: "csv:1", occurredAt: "2026-10-01T10:00:00Z" } as Touch], NOW)).toEqual(["phone"]); }); }); @@ -128,12 +130,20 @@ describe("the decision", () => { expect(notInterested.action).toBe("nurture"); expect(notInterested.reviewReasons[0]).toContain("they said they weren't interested on a call on 2026-10-20"); expect(combine(input({ history: [call("NOT_INTERESTED", "2026-06-01T15:00:00Z")] }), answers()).action).toBe("contact_now"); - expect(reachableOn(prospect(), [call("WRONG_NUMBER", "2026-10-30T15:00:00Z")])).toEqual(["email"]); - expect(reachableOn(prospect(), [call("WRONG_NUMBER", "2026-10-28T15:00:00Z"), call("GOOD_NUMBER", "2026-10-30T15:00:00Z")])).toEqual(["email", "phone"]); + expect(reachableOn(prospect(), [call("WRONG_NUMBER", "2026-10-30T15:00:00Z")], NOW)).toEqual(["email"]); + expect(reachableOn(prospect(), [call("WRONG_NUMBER", "2026-10-28T15:00:00Z"), call("GOOD_NUMBER", "2026-10-30T15:00:00Z")], NOW)).toEqual(["email", "phone"]); const interested = combine(input({ history: [call("INTERESTED", "2026-10-30T15:00:00Z")] }), answers({ action: choice("skip", 0.9) })); expect(interested.reviewReasons).toContain('It says skip, but the dialer logged "interested" on a call on 2026-10-30.'); }); + test("a wrong or bad number holds for 90 days, and only for the number that was dialed", () => { + const wrong = (at: string, toAddress?: string): Touch => ({ kind: "CALLED", channel: "CALL", templateId: "dialer:WRONG_NUMBER", prospectKey: "csv:1", occurredAt: at, toAddress }); + expect(reachableOn(prospect(), [wrong("2026-10-30T15:00:00Z", "(512) 555-0101")], NOW)).toEqual(["email"]); + expect(reachableOn(prospect(), [wrong("2026-06-01T15:00:00Z", "(512) 555-0101")], NOW)).toEqual(["email", "phone"]); + expect(reachableOn(prospect(), [wrong("2026-10-30T15:00:00Z", "+1 512 555 0199")], NOW)).toEqual(["email", "phone"]); + expect(reachableOn(prospect(), [{ ...wrong("2026-10-30T15:00:00Z"), templateId: "dialer:BAD_NUMBER" }], NOW)).toEqual(["email"]); + }); + test("an unsure call, a contradiction, or a decision on the list alone goes to you to check", () => { expect(combine(input(), answers({ action: choice("contact_now", 0.5) })).reviewReasons).toEqual(["It isn't sure about the next step (confidence 50%)."]); const contradiction = combine(input(), answers({ need: { type: "noul", noul: 0.05 }, timing: { type: "noul", noul: 0.05 }, authority: { type: "noul", noul: 0.1 }, fit: { type: "noul", noul: 0.4 } })); diff --git a/src/domains/decide/model.ts b/src/domains/decide/model.ts index 69f7d94..31624dd 100644 --- a/src/domains/decide/model.ts +++ b/src/domains/decide/model.ts @@ -82,11 +82,26 @@ const day = (d: Date) => d.toISOString().slice(0, 10); const pct = (x: number) => `${Math.round(x * 100)}%`; const SHARED_INBOX = /^(info|hello|hi|contact|office|admin|sales|support|team|bookings?|enquiries|inquiries|service|mail|frontdesk|reception)@/i; -/** The channels a prospect can be reached on: an email that hasn't bounced, their profiles, a phone the dialer didn't find wrong. */ -export function reachableOn(prospect: Prospect, history: Touch[]): Channel[] { +/** A dialed number and a phone, by their last ten digits: "(512) 555-0147" and "+1 512 555 0147" are one number. */ +function sameNumber(dialed: string, phone: string | undefined): boolean { + const digits = (s: string | undefined) => (s ?? "").replace(/\D/g, "").slice(-10); + return !!digits(dialed) && digits(dialed) === digits(phone); +} + +/** + * The channels a prospect can be reached on: an email that hasn't bounced, + * their profiles, and a phone, unless the dialer's latest call found that + * same number wrong or dead in the last 90 days. + */ +export function reachableOn(prospect: Prospect, history: Touch[], now: Date): Channel[] { const bounced = history.some((t) => t.channel === "EMAIL" && (t.kind === "BOUNCED" || t.kind === "UNDELIVERABLE")); const lastCall = latestCall(history); - const deadNumber = !!lastCall && DEAD_NUMBERS.has(lastCall.disposition); + const deadNumber = + !!lastCall && + DEAD_NUMBERS.has(lastCall.disposition) && + now.getTime() - Date.parse(lastCall.touch.occurredAt) < CALL_MEMORY_DAYS * 86_400_000 && + // A call to a number they no longer have says nothing about the one they have now. + (!lastCall.touch.toAddress || sameNumber(lastCall.touch.toAddress, prospect.phone)); const found: Array<[Channel, unknown]> = [ ["email", prospect.email && !bounced], ["linkedin", prospect.linkedinUrl], @@ -134,7 +149,7 @@ export function decisionState(input: DecisionInput): Record { today: day(input.now), seller: { sells_to: offer.audience, offers: offer.offer, outcome: offer.outcome, pain_it_removes: offer.pain, proof: offer.proof, writes_as: input.sender.voice === "company" ? "the company (we)" : "one person (I)" }, prospect: listFacts(input.prospect), - can_reach_them_by: reachableOn(input.prospect, input.history), + can_reach_them_by: reachableOn(input.prospect, input.history, input.now), diligence: dossier && { summary: dossier.summary, offerings: short(dossier.offerings, 8), @@ -157,7 +172,7 @@ const describeAngle = (a: Angle) => `${ANGLE_GUIDE[a].label}: ${ANGLE_GUIDE[a].w /** The questions asked about every prospect: ten at most, so Jev answers them in one request. */ export function outboundQuestions(input: DecisionInput): Record { - const channels = reachableOn(input.prospect, input.history); + const channels = reachableOn(input.prospect, input.history, input.now); const questions: Record = { fit: { type: "noul", @@ -218,7 +233,7 @@ export function scoreOf(f: Record): number { */ export function combine(input: DecisionInput, result: DecisionResult & { provider: "jev" | "claude" }, reviewBelow = REVIEW_BELOW): Decision { const a = result.answers; - const channels = reachableOn(input.prospect, input.history); + const channels = reachableOn(input.prospect, input.history, input.now); const budget = a.budget?.type === "score" ? a.budget.score / 3 : 0.5; const factors: Decision["factors"] = { fit: { value: noul(a.fit), reason: a.fit?.reason }, diff --git a/src/domains/leads/index.ts b/src/domains/leads/index.ts index eb55eaa..942b65a 100644 --- a/src/domains/leads/index.ts +++ b/src/domains/leads/index.ts @@ -1,8 +1,8 @@ import { DEFAULT_CALL_HOURS, type CallHours } from "../config"; import { claimRun, type Db } from "../db"; -import { loadAgencyLeads } from "../prospects"; +import { loadAgencyLeads, type Prospect } from "../prospects"; import type { Touch } from "../touches"; -import { filterValue, type TwentyClient } from "../twenty"; +import { filterAnd, filterValue, type TwentyClient, type TwentyRecord } from "../twenty"; import { countPush, lastCall, leadPush, quizLead, rankLeads, readNtfy, sendNtfy } from "./helpers"; import { LEAD_CHECK_MS, LEAD_WINDOW_DAYS, MAX_LEAD_PUSHES, MAX_LEADS_PER_CHECK, type LeadCheck, type NtfyTarget, type QuizLead } from "./types"; @@ -36,7 +36,9 @@ export interface LeadWatchOptions { * they get a call while the quiz is fresh. It looks at most once a minute * across every server on the database (the look is claimed in it), and only * at leads newer than the last one it saw. The first look only marks the - * time: leads from before alerts were set up are never pushed. + * time: leads from before alerts were set up are never pushed. More new + * leads than one look reads come over the next looks, oldest first, and none + * is skipped. */ export class LeadWatch { private nextAt = 0; @@ -66,7 +68,7 @@ export class LeadWatch { let found = 0; try { const hours = (await this.options.hours?.()) ?? DEFAULT_CALL_HOURS; - const rows = await loadAgencyLeads(client, { where: `createdAt[gt]:${filterValue(seen.value)}`, limit: MAX_LEADS_PER_CHECK }); + const { rows, until } = await this.newLeads(client, seen.value); // A log that can't be opened only leaves the opt-outs out, as on the Calls page. const suppressions = rows.length ? ((await this.options.suppressions?.().catch(() => [])) ?? []) : []; const leads = rows @@ -76,9 +78,10 @@ export class LeadWatch { found = leads.length; const due = leads.filter((l) => l.qualified && !l.doNotContact); const newest = leads.at(-1)?.createdAt; - const lots = rows.length >= MAX_LEADS_PER_CHECK; - if (due.length > MAX_LEAD_PUSHES || (lots && due.length)) { - await sendNtfy(target, countPush(due, this.options.appUrl, lots), this.options.fetch); + // More came in than one look reads: this look took the oldest of them, and the next look carries on. + const more = until !== undefined || rows.length >= MAX_LEADS_PER_CHECK; + if (due.length > MAX_LEAD_PUSHES || (more && due.length)) { + await sendNtfy(target, countPush(due, this.options.appUrl, more), this.options.fetch); alerted = 1; if (newest) await this.setSeen(newest); return { looked: true, alerted, found }; @@ -97,6 +100,30 @@ export class LeadWatch { } } + /** + * The leads that came in after `after`: every one of them up to a time, so + * marking the newest seen never passes one that wasn't read. Twenty lists + * by id, not by time, so when more came in than one look reads, the window + * closes at the median time of what was read, again until it fits; `until` + * is where it closed, and the next look carries on from there. + */ + private async newLeads(client: TwentyClient, after: string): Promise<{ rows: Array<{ record: TwentyRecord; prospect: Prospect }>; until?: string }> { + let until: string | undefined; + for (;;) { + const where = filterAnd(`createdAt[gt]:${filterValue(after)}`, until ? `createdAt[lte]:${filterValue(until)}` : undefined); + const rows = await loadAgencyLeads(client, { where, limit: MAX_LEADS_PER_CHECK }); + if (rows.length < MAX_LEADS_PER_CHECK) return { rows, until }; + const times = rows + .map((r) => r.record.createdAt) + .filter((t): t is string => typeof t === "string" && t > after) + .sort(); + const median = times[Math.floor((times.length - 1) / 2)]; + // Hundreds of leads in one millisecond can't be told apart by time: take them as they are. + if (!median || (until !== undefined && median >= until)) return { rows, until }; + until = median; + } + } + private async setSeen(at: string): Promise { await this.options.db.run("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value", SEEN_KEY, at); } diff --git a/src/domains/leads/leads.test.ts b/src/domains/leads/leads.test.ts index 305a677..868e94b 100644 --- a/src/domains/leads/leads.test.ts +++ b/src/domains/leads/leads.test.ts @@ -16,12 +16,22 @@ function fakeTwenty() { const url = new URL(String(input)); const filter = url.searchParams.get("filter") ?? ""; const after = /createdAt\[(gt|gte)\]:"([^"]+)"/.exec(filter); + const upTo = /createdAt\[lte\]:"([^"]+)"/.exec(filter)?.[1]; + const cursor = /(?:^|[(,])id\[gt\]:"([^"]+)"/.exec(filter)?.[1]; const ids = /id\[in\]:\[([^\]]*)\]/.exec(filter)?.[1]?.split(",").map((s) => s.replace(/"/g, "")); - const rows = (url.pathname === "/rest/agencyLeads" ? state.leads : state.prospects).filter((r) => { - if (ids && !ids.includes(r.id as string)) return false; - if (after) return after[1] === "gt" ? (r.createdAt as string) > after[2]! : (r.createdAt as string) >= after[2]!; - return true; - }); + const limit = Number(url.searchParams.get("limit")) || Number.POSITIVE_INFINITY; + const rows = (url.pathname === "/rest/agencyLeads" ? state.leads : state.prospects) + .filter((r) => { + const at = r.createdAt as string; + if (ids && !ids.includes(r.id as string)) return false; + if (cursor && (r.id as string) <= cursor) return false; + if (upTo && at > upTo) return false; + if (after) return after[1] === "gt" ? at > after[2]! : at >= after[2]!; + return true; + }) + // Like Twenty: in id order, a page at a time, as cold asks (order_by id, limit, id[gt] for the next page). + .sort((a, b) => ((a.id as string) < (b.id as string) ? -1 : (a.id as string) > (b.id as string) ? 1 : 0)) + .slice(0, limit); return Response.json({ data: { [url.pathname.slice(6)]: rows } }); }) as typeof fetch; return { state, client: new TwentyClient({ baseUrl: "https://crm.example", apiKey: "k" }, fetchImpl) }; @@ -265,6 +275,27 @@ describe("LeadWatch", () => { later(60); expect((await w.tick()).alerted).toBe(0); }); + + test("more new leads than one look reads come over the next looks, oldest first, and none is skipped", async () => { + const w = watch(); + await w.tick(); + later(90); + // 1,200 leads in the last minute whose ids sort in another order than their times, as Twenty's do. + const start = clock.getTime() - 60_000; + for (let i = 0; i < 1200; i++) { + twenty.state.leads.push({ id: `lead-${String((i * 7919) % 1200).padStart(4, "0")}`, contactName: `Lead ${i}`, note: note(`Lead ${i}`), qualificationStatus: "QUALIFIED", createdAt: new Date(start + i * 10).toISOString() }); + } + const newest = new Date(start + 1199 * 10).toISOString(); + let found = 0; + for (let look = 0; look < 10 && (await w.state()).seenUntil !== newest; look++) { + found += (await w.tick()).found; + later(90); + } + expect((await w.state()).seenUntil).toBe(newest); + expect(found).toBe(1200); + expect(ntfy.state.sent.length).toBeGreaterThan(1); + expect(ntfy.state.sent.every((s) => /^\d+\+? new quiz leads$/.test(s.body.title))).toBe(true); + }); }); describe("recentLeads", () => { diff --git a/src/domains/leads/types.ts b/src/domains/leads/types.ts index 27d3a23..064805c 100644 --- a/src/domains/leads/types.ts +++ b/src/domains/leads/types.ts @@ -68,5 +68,5 @@ export const LEAD_CHECK_MS = 60_000; export const LEAD_WINDOW_DAYS = 14; /** More new leads than this in one look come as one push that counts them. */ export const MAX_LEAD_PUSHES = 5; -/** A look reads at most this many new leads. */ +/** A look reads at most this many new leads; more come over the next looks, oldest first. */ export const MAX_LEADS_PER_CHECK = 500; diff --git a/src/domains/listen/index.ts b/src/domains/listen/index.ts index 4b1f843..9624e1a 100644 --- a/src/domains/listen/index.ts +++ b/src/domains/listen/index.ts @@ -12,23 +12,25 @@ export interface ListenWatch { latest: string; } +/** Matches written by one INSERT: seven values each, far under SQLite's limit on values in a statement. */ +const ROWS_PER_INSERT = 100; + /** The matches OpenMagpie sent, in the app's database. Each key is kept once. */ export class ListenStore { constructor(private readonly db: Db) {} - /** Keep new matches; one already kept (a redelivery) is left as it is. */ + /** + * Keep new matches, a hundred to a statement, so a 500-match digest is five + * writes rather than five hundred round trips to the database. One already + * kept (a redelivery, or twice in one delivery) is left as it is. + */ async receive(items: ListenItem[]): Promise<{ added: number }> { let added = 0; - for (const item of items) { + for (let i = 0; i < items.length; i += ROWS_PER_INSERT) { + const rows = items.slice(i, i + ROWS_PER_INSERT); const { changes } = await this.db.run( - "INSERT INTO listen_items (key, watch_id, watch_name, source_kind, occurred_at, received_at, item) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (key) DO NOTHING", - item.key, - item.watchId, - item.watchName, - item.sourceKind, - item.occurredAt ?? item.receivedAt, - item.receivedAt, - JSON.stringify(item), + `INSERT INTO listen_items (key, watch_id, watch_name, source_kind, occurred_at, received_at, item) VALUES ${rows.map(() => "(?, ?, ?, ?, ?, ?, ?)").join(", ")} ON CONFLICT (key) DO NOTHING`, + ...rows.flatMap((item) => [item.key, item.watchId, item.watchName, item.sourceKind, item.occurredAt ?? item.receivedAt, item.receivedAt, JSON.stringify(item)]), ); added += changes; } diff --git a/src/domains/listen/listen.test.ts b/src/domains/listen/listen.test.ts index ac73475..1ea188b 100644 --- a/src/domains/listen/listen.test.ts +++ b/src/domains/listen/listen.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { openDatabase } from "../db"; +import { openDatabase, type Db } from "../db"; import { ListenStore, listenProspect, quoteOf, readMagpie, triggerOf, type ListenItem } from "."; const NOW = new Date("2026-11-02T15:00:00Z"); @@ -93,4 +93,17 @@ describe("the matches kept", () => { expect((await store.items({ watch: "nope" })).length).toBe(0); expect(await store.watches()).toEqual([{ id: "01JWATCH", name: "Needs booking software", count: 4, latest: NOW.toISOString() }]); }); + + test("a delivery is kept a hundred matches to a statement, a match twice in one delivery once", async () => { + const db = await openDatabase(":memory:"); + let statements = 0; + const counted: Db = { get: db.get.bind(db), all: db.all.bind(db), batch: db.batch.bind(db), close: db.close.bind(db), run: (sql, ...params) => (statements++, db.run(sql, ...params)) }; + const store = new ListenStore(counted); + const [first] = readMagpie(digest, NOW).items; + const many = Array.from({ length: 250 }, (_, i) => ({ ...first!, key: `reddit_subreddit:t3_${i}` })); + expect(await store.receive([...many, many[0]!])).toEqual({ added: 250 }); + expect(statements).toBe(3); + expect(await store.receive(many)).toEqual({ added: 0 }); + expect((await store.items({ limit: 5000 })).length).toBe(250); + }); }); diff --git a/src/domains/web/listen/index.ts b/src/domains/web/listen/index.ts index 9b73cbb..415e5a8 100644 --- a/src/domains/web/listen/index.ts +++ b/src/domains/web/listen/index.ts @@ -13,6 +13,35 @@ function sameSecret(given: string | null, expected: string): boolean { return timingSafeEqual(a, b); } +/** + * The body as text, read until it passes `max` bytes: undefined then, and the + * rest is never read, so a body without a Content-Length (sent in chunks) is + * refused without holding all of it in memory first. + */ +async function readCapped(request: Request, max: number): Promise { + if (!request.body) return ""; + const reader = request.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > max) { + await reader.cancel().catch(() => undefined); + return undefined; + } + chunks.push(value); + } + const bytes = new Uint8Array(size); + let at = 0; + for (const chunk of chunks) { + bytes.set(chunk, at); + at += chunk.byteLength; + } + return new TextDecoder().decode(bytes); +} + /** * OpenMagpie's webhook action delivers matches here (POST, instant or * digest). It proves itself with `Authorization: Bearer `, @@ -25,9 +54,10 @@ export function listenWebhook(ctx: WebContext): (request: Request) => Promise MAX_LISTEN_BYTES) return json({ error: "That delivery is too large: use a shorter digest interval." }, 413); - const text = await request.text(); - if (text.length > MAX_LISTEN_BYTES) return json({ error: "That delivery is too large: use a shorter digest interval." }, 413); + const tooLarge = () => json({ error: "That delivery is too large: use a shorter digest interval." }, 413); + if (Number(request.headers.get("content-length") ?? 0) > MAX_LISTEN_BYTES) return tooLarge(); + const text = await readCapped(request, MAX_LISTEN_BYTES); + if (text === undefined) return tooLarge(); let body: unknown; try { body = JSON.parse(text); @@ -69,7 +99,9 @@ export function listenRoutes(ctx: WebContext): Route[] { const name = watch === "all" ? "all" : (await ctx.listening.watches()).find((w) => w.id === watch)?.name; if (!name) throw new HttpError(404, "No matches have come in from that watch yet."); const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60) || "matches"; - const file = `prospects-listen-${slug}.csv`; + // One list per watch, named for it and its id: two watches with the same name never share a list. + const id = watch.toLowerCase().replace(/[^a-z0-9-]+/g, "-").slice(-40).replace(/^-+|-+$/g, "") || "watch"; + const file = watch === "all" ? "prospects-listen-all.csv" : `prospects-listen-${slug}-${id}.csv`; const existing = await ctx.files.read(file).catch(() => undefined); const { text, added, rows } = listenCsv(await ctx.listening.items({ watch: watch === "all" ? undefined : watch, limit: 5000 }), existing); await ctx.files.write(file, text); diff --git a/src/domains/web/listen/listen.test.ts b/src/domains/web/listen/listen.test.ts index b057c71..b0e4094 100644 --- a/src/domains/web/listen/listen.test.ts +++ b/src/domains/web/listen/listen.test.ts @@ -3,6 +3,7 @@ import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:f import { tmpdir } from "node:os"; import path from "node:path"; import { openDatabase } from "../../db"; +import { MAX_LISTEN_BYTES } from "../../listen"; import { createApi } from "../api"; import { WebContext } from "../context"; import type { Campaign, ListenStatusResponse, PlanResponse, ProspectSummary, SourceOption } from "../types"; @@ -65,6 +66,23 @@ describe("OpenMagpie's webhook", () => { expect(await ctx.listening!.items()).toEqual([]); }); + test("a delivery over the size limit is refused while it is read, sent in chunks with no Content-Length too", async () => { + process.env.COLD_LISTEN_SECRET = "a-long-listening-secret"; + const chunk = new Uint8Array(64 * 1024).fill(32); + let pulled = 0; + const body = new ReadableStream({ + pull(controller) { + if (++pulled > 100) controller.close(); + else controller.enqueue(chunk); + }, + }); + const request = new Request(`${ORIGIN}/api/listen/magpie`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer a-long-listening-secret" }, body, duplex: "half" } as RequestInit); + expect(request.headers.get("content-length")).toBeNull(); + const response = await webhook(request); + expect(response.status).toBe(413); + expect(pulled * chunk.byteLength).toBeLessThan(MAX_LISTEN_BYTES + 4 * chunk.byteLength); + }); + test("a delivery is kept, and the same delivery again adds nothing", async () => { const auth = { authorization: "Bearer a-long-listening-secret" }; const first = await post(delivery, auth); @@ -104,8 +122,8 @@ describe("matches as prospects", () => { test("saved as a list, a match can be filled in, and saving again only adds what is new", async () => { const { data } = await call<{ source: string; added: number; rows: number; message: string }>("POST", "/api/listen/export", { watch: "01JWATCH" }); - expect(data).toMatchObject({ source: "csv:prospects-listen-needs-booking-software.csv", added: 2, rows: 2 }); - const file = path.join(root, "prospects-listen-needs-booking-software.csv"); + expect(data).toMatchObject({ source: "csv:prospects-listen-needs-booking-software-01jwatch.csv", added: 2, rows: 2 }); + const file = path.join(root, "prospects-listen-needs-booking-software-01jwatch.csv"); const text = readFileSync(file, "utf8"); expect(text.split("\n")[0]).toBe("id,first_name,full_name,company,email,website,x_handle,trigger,post_url,post_title,post_text,posted_at,listen_source,listen_watch"); writeFileSync(file, text.replace("twitter_search:1850,Dana,Dana Reyes,,", "twitter_search:1850,Dana,Dana Reyes,Clearview Tint,")); @@ -117,4 +135,12 @@ describe("matches as prospects", () => { expect(plan.due[0]!.text).toContain('Saw your post on X: "Is there a booking page that turns Google reviews into appointments?".'); expect(plan.due[0]!.text).toContain("for Clearview Tint?"); }); + + test("two watches with the same name keep a list each", async () => { + const twin = { ...delivery, watch: { id: "01JTWIN", name: "Needs booking software" }, items: [{ key: "hackernews:991", source: { label: "Hacker News", kind: "hackernews" }, item: { occurred_at: "2026-11-02T14:40:00Z", title: "Ask HN: booking software for a small shop?", url: "https://news.ycombinator.com/item?id=991", author: "pg_fan" } }] }; + expect((await post(twin, { authorization: "Bearer a-long-listening-secret" })).status).toBe(200); + const { data } = await call<{ source: string; added: number; rows: number }>("POST", "/api/listen/export", { watch: "01JTWIN" }); + expect(data).toMatchObject({ source: "csv:prospects-listen-needs-booking-software-01jtwin.csv", added: 1, rows: 1 }); + expect(readFileSync(path.join(root, "prospects-listen-needs-booking-software-01jwatch.csv"), "utf8")).not.toContain("hackernews:991"); + }); });