From f38b686f3510d26576d9a8129f4c37d0e1c6197f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 09:25:01 +0000 Subject: [PATCH 1/8] fix: lead alerts never skip leads past one look's 500 Twenty lists records by id, not by time. When more than 500 leads came in between two looks, a look read 500 of them in id order and marked seen up to the newest of those, past older leads it had never read. With no qualified lead among the 500, nothing was pushed for any of them. A look now reads every lead up to a time: when more came in than it reads, it closes the window at the median time of what it read, again until the window fits, and the next look carries on from there. Seen never passes a lead that wasn't read. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuVXF5k9RXQpSdqYkdV --- CHANGELOG.md | 3 ++- src/domains/leads/index.ts | 41 +++++++++++++++++++++++++++------ src/domains/leads/leads.test.ts | 41 +++++++++++++++++++++++++++++---- src/domains/leads/types.ts | 2 +- 4 files changed, 73 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5416e55..3e1f7ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,7 +50,8 @@ phone through ntfy within a minute of each qualified lead (several at once come as one), tapping it opens the lead; it looks once a minute across every server on the database, never pushes a lead twice, retries one that - failed, and never pushes leads from before it was set up. Settings, Lead + failed, never pushes leads from before it was set up, and takes more than + one look's 500 new leads over the next looks, oldest first, skipping none. Settings, Lead alerts sends a test. A lead who opted out anywhere, "do not call" on the dialer included, is Do Not Contact: last on Calls, with no Call button, and never pushed. Their answers read whether offer writes the JSON on one diff --git a/src/domains/leads/index.ts b/src/domains/leads/index.ts index eb55eaa..942b65a 100644 --- a/src/domains/leads/index.ts +++ b/src/domains/leads/index.ts @@ -1,8 +1,8 @@ import { DEFAULT_CALL_HOURS, type CallHours } from "../config"; import { claimRun, type Db } from "../db"; -import { loadAgencyLeads } from "../prospects"; +import { loadAgencyLeads, type Prospect } from "../prospects"; import type { Touch } from "../touches"; -import { filterValue, type TwentyClient } from "../twenty"; +import { filterAnd, filterValue, type TwentyClient, type TwentyRecord } from "../twenty"; import { countPush, lastCall, leadPush, quizLead, rankLeads, readNtfy, sendNtfy } from "./helpers"; import { LEAD_CHECK_MS, LEAD_WINDOW_DAYS, MAX_LEAD_PUSHES, MAX_LEADS_PER_CHECK, type LeadCheck, type NtfyTarget, type QuizLead } from "./types"; @@ -36,7 +36,9 @@ export interface LeadWatchOptions { * they get a call while the quiz is fresh. It looks at most once a minute * across every server on the database (the look is claimed in it), and only * at leads newer than the last one it saw. The first look only marks the - * time: leads from before alerts were set up are never pushed. + * time: leads from before alerts were set up are never pushed. More new + * leads than one look reads come over the next looks, oldest first, and none + * is skipped. */ export class LeadWatch { private nextAt = 0; @@ -66,7 +68,7 @@ export class LeadWatch { let found = 0; try { const hours = (await this.options.hours?.()) ?? DEFAULT_CALL_HOURS; - const rows = await loadAgencyLeads(client, { where: `createdAt[gt]:${filterValue(seen.value)}`, limit: MAX_LEADS_PER_CHECK }); + const { rows, until } = await this.newLeads(client, seen.value); // A log that can't be opened only leaves the opt-outs out, as on the Calls page. const suppressions = rows.length ? ((await this.options.suppressions?.().catch(() => [])) ?? []) : []; const leads = rows @@ -76,9 +78,10 @@ export class LeadWatch { found = leads.length; const due = leads.filter((l) => l.qualified && !l.doNotContact); const newest = leads.at(-1)?.createdAt; - const lots = rows.length >= MAX_LEADS_PER_CHECK; - if (due.length > MAX_LEAD_PUSHES || (lots && due.length)) { - await sendNtfy(target, countPush(due, this.options.appUrl, lots), this.options.fetch); + // More came in than one look reads: this look took the oldest of them, and the next look carries on. + const more = until !== undefined || rows.length >= MAX_LEADS_PER_CHECK; + if (due.length > MAX_LEAD_PUSHES || (more && due.length)) { + await sendNtfy(target, countPush(due, this.options.appUrl, more), this.options.fetch); alerted = 1; if (newest) await this.setSeen(newest); return { looked: true, alerted, found }; @@ -97,6 +100,30 @@ export class LeadWatch { } } + /** + * The leads that came in after `after`: every one of them up to a time, so + * marking the newest seen never passes one that wasn't read. Twenty lists + * by id, not by time, so when more came in than one look reads, the window + * closes at the median time of what was read, again until it fits; `until` + * is where it closed, and the next look carries on from there. + */ + private async newLeads(client: TwentyClient, after: string): Promise<{ rows: Array<{ record: TwentyRecord; prospect: Prospect }>; until?: string }> { + let until: string | undefined; + for (;;) { + const where = filterAnd(`createdAt[gt]:${filterValue(after)}`, until ? `createdAt[lte]:${filterValue(until)}` : undefined); + const rows = await loadAgencyLeads(client, { where, limit: MAX_LEADS_PER_CHECK }); + if (rows.length < MAX_LEADS_PER_CHECK) return { rows, until }; + const times = rows + .map((r) => r.record.createdAt) + .filter((t): t is string => typeof t === "string" && t > after) + .sort(); + const median = times[Math.floor((times.length - 1) / 2)]; + // Hundreds of leads in one millisecond can't be told apart by time: take them as they are. + if (!median || (until !== undefined && median >= until)) return { rows, until }; + until = median; + } + } + private async setSeen(at: string): Promise { await this.options.db.run("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value", SEEN_KEY, at); } diff --git a/src/domains/leads/leads.test.ts b/src/domains/leads/leads.test.ts index 305a677..868e94b 100644 --- a/src/domains/leads/leads.test.ts +++ b/src/domains/leads/leads.test.ts @@ -16,12 +16,22 @@ function fakeTwenty() { const url = new URL(String(input)); const filter = url.searchParams.get("filter") ?? ""; const after = /createdAt\[(gt|gte)\]:"([^"]+)"/.exec(filter); + const upTo = /createdAt\[lte\]:"([^"]+)"/.exec(filter)?.[1]; + const cursor = /(?:^|[(,])id\[gt\]:"([^"]+)"/.exec(filter)?.[1]; const ids = /id\[in\]:\[([^\]]*)\]/.exec(filter)?.[1]?.split(",").map((s) => s.replace(/"/g, "")); - const rows = (url.pathname === "/rest/agencyLeads" ? state.leads : state.prospects).filter((r) => { - if (ids && !ids.includes(r.id as string)) return false; - if (after) return after[1] === "gt" ? (r.createdAt as string) > after[2]! : (r.createdAt as string) >= after[2]!; - return true; - }); + const limit = Number(url.searchParams.get("limit")) || Number.POSITIVE_INFINITY; + const rows = (url.pathname === "/rest/agencyLeads" ? state.leads : state.prospects) + .filter((r) => { + const at = r.createdAt as string; + if (ids && !ids.includes(r.id as string)) return false; + if (cursor && (r.id as string) <= cursor) return false; + if (upTo && at > upTo) return false; + if (after) return after[1] === "gt" ? at > after[2]! : at >= after[2]!; + return true; + }) + // Like Twenty: in id order, a page at a time, as cold asks (order_by id, limit, id[gt] for the next page). + .sort((a, b) => ((a.id as string) < (b.id as string) ? -1 : (a.id as string) > (b.id as string) ? 1 : 0)) + .slice(0, limit); return Response.json({ data: { [url.pathname.slice(6)]: rows } }); }) as typeof fetch; return { state, client: new TwentyClient({ baseUrl: "https://crm.example", apiKey: "k" }, fetchImpl) }; @@ -265,6 +275,27 @@ describe("LeadWatch", () => { later(60); expect((await w.tick()).alerted).toBe(0); }); + + test("more new leads than one look reads come over the next looks, oldest first, and none is skipped", async () => { + const w = watch(); + await w.tick(); + later(90); + // 1,200 leads in the last minute whose ids sort in another order than their times, as Twenty's do. + const start = clock.getTime() - 60_000; + for (let i = 0; i < 1200; i++) { + twenty.state.leads.push({ id: `lead-${String((i * 7919) % 1200).padStart(4, "0")}`, contactName: `Lead ${i}`, note: note(`Lead ${i}`), qualificationStatus: "QUALIFIED", createdAt: new Date(start + i * 10).toISOString() }); + } + const newest = new Date(start + 1199 * 10).toISOString(); + let found = 0; + for (let look = 0; look < 10 && (await w.state()).seenUntil !== newest; look++) { + found += (await w.tick()).found; + later(90); + } + expect((await w.state()).seenUntil).toBe(newest); + expect(found).toBe(1200); + expect(ntfy.state.sent.length).toBeGreaterThan(1); + expect(ntfy.state.sent.every((s) => /^\d+\+? new quiz leads$/.test(s.body.title))).toBe(true); + }); }); describe("recentLeads", () => { diff --git a/src/domains/leads/types.ts b/src/domains/leads/types.ts index 27d3a23..064805c 100644 --- a/src/domains/leads/types.ts +++ b/src/domains/leads/types.ts @@ -68,5 +68,5 @@ export const LEAD_CHECK_MS = 60_000; export const LEAD_WINDOW_DAYS = 14; /** More new leads than this in one look come as one push that counts them. */ export const MAX_LEAD_PUSHES = 5; -/** A look reads at most this many new leads. */ +/** A look reads at most this many new leads; more come over the next looks, oldest first. */ export const MAX_LEADS_PER_CHECK = 500; From 6d3e0aecb861fba437708c78597f0b8f099cc0d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 09:25:54 +0000 Subject: [PATCH 2/8] fix: a wrong or bad number holds for 90 days, and only for that number reachableOn dropped the phone whenever the dialer's latest call was a wrong or bad number, with no end and no look at which number was dialed: a prospect whose number was corrected in Twenty, or whose wrong number was logged two years ago, was never reachable by phone again. The phone now stays off only for 90 days (the window "not interested" already uses) and only while the prospect's phone is the number that was dialed, compared by its last ten digits. reachableOn takes the decision's clock. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuVXF5k9RXQpSdqYkdV --- CHANGELOG.md | 3 ++- docs/decision-engine.md | 3 ++- src/domains/decide/model.test.ts | 16 +++++++++++++--- src/domains/decide/model.ts | 27 +++++++++++++++++++++------ 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e1f7ea..b0e6c9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,8 @@ dialer's prospect is marked Do Not Contact. - The decision engine reads the calls in their history. Someone who opted out is always skipped; "not interested" on a recent call turns reach out - now into later; a wrong or bad number is no longer a way to reach them; + now into later; a wrong or bad number is no longer a way to reach them + for 90 days, unless their number has changed since; and skipping someone who was interested goes to you to check. - Quiz leads say how their last call went. - **Calls at their hour.** A call sheet for a prospect, in the app and from diff --git a/docs/decision-engine.md b/docs/decision-engine.md index 6e2d3fa..9d5211c 100644 --- a/docs/decision-engine.md +++ b/docs/decision-engine.md @@ -173,7 +173,8 @@ about them. The dialer's calls are part of what the model reads: each one in the history with its outcome, the dialer's AI score and tone, and its summary. A wrong or bad number stops counting the phone as a way to reach them, which lowers -their reach score. +their reach score: for 90 days, and only while their phone is still the number +that was dialed. Anyone marked Do Not Contact, or who ever opted out, is never researched or scored. diff --git a/src/domains/decide/model.test.ts b/src/domains/decide/model.test.ts index 58477ec..439cd4d 100644 --- a/src/domains/decide/model.test.ts +++ b/src/domains/decide/model.test.ts @@ -5,6 +5,8 @@ import type { Dossier } from "./diligence"; import { allowedAngles, combine, decisionState, outboundQuestions, reachableOn, scoreOf, tierOf, type DecisionInput } from "./model"; import type { Answer, DecisionResult } from "./types"; +const NOW = new Date("2026-10-31T12:00:00Z"); + const prospect = (extra: Partial = {}): Prospect => ({ key: "csv:1", identities: ["csv:1", "email:dana@clearview.example"], @@ -93,7 +95,7 @@ describe("the questions", () => { }); test("an email that bounced is not a way to reach them", () => { - expect(reachableOn(prospect(), [{ kind: "BOUNCED", channel: "EMAIL", prospectKey: "csv:1", occurredAt: "2026-10-01T10:00:00Z" } as Touch])).toEqual(["phone"]); + expect(reachableOn(prospect(), [{ kind: "BOUNCED", channel: "EMAIL", prospectKey: "csv:1", occurredAt: "2026-10-01T10:00:00Z" } as Touch], NOW)).toEqual(["phone"]); }); }); @@ -128,12 +130,20 @@ describe("the decision", () => { expect(notInterested.action).toBe("nurture"); expect(notInterested.reviewReasons[0]).toContain("they said they weren't interested on a call on 2026-10-20"); expect(combine(input({ history: [call("NOT_INTERESTED", "2026-06-01T15:00:00Z")] }), answers()).action).toBe("contact_now"); - expect(reachableOn(prospect(), [call("WRONG_NUMBER", "2026-10-30T15:00:00Z")])).toEqual(["email"]); - expect(reachableOn(prospect(), [call("WRONG_NUMBER", "2026-10-28T15:00:00Z"), call("GOOD_NUMBER", "2026-10-30T15:00:00Z")])).toEqual(["email", "phone"]); + expect(reachableOn(prospect(), [call("WRONG_NUMBER", "2026-10-30T15:00:00Z")], NOW)).toEqual(["email"]); + expect(reachableOn(prospect(), [call("WRONG_NUMBER", "2026-10-28T15:00:00Z"), call("GOOD_NUMBER", "2026-10-30T15:00:00Z")], NOW)).toEqual(["email", "phone"]); const interested = combine(input({ history: [call("INTERESTED", "2026-10-30T15:00:00Z")] }), answers({ action: choice("skip", 0.9) })); expect(interested.reviewReasons).toContain('It says skip, but the dialer logged "interested" on a call on 2026-10-30.'); }); + test("a wrong or bad number holds for 90 days, and only for the number that was dialed", () => { + const wrong = (at: string, toAddress?: string): Touch => ({ kind: "CALLED", channel: "CALL", templateId: "dialer:WRONG_NUMBER", prospectKey: "csv:1", occurredAt: at, toAddress }); + expect(reachableOn(prospect(), [wrong("2026-10-30T15:00:00Z", "(512) 555-0101")], NOW)).toEqual(["email"]); + expect(reachableOn(prospect(), [wrong("2026-06-01T15:00:00Z", "(512) 555-0101")], NOW)).toEqual(["email", "phone"]); + expect(reachableOn(prospect(), [wrong("2026-10-30T15:00:00Z", "+1 512 555 0199")], NOW)).toEqual(["email", "phone"]); + expect(reachableOn(prospect(), [{ ...wrong("2026-10-30T15:00:00Z"), templateId: "dialer:BAD_NUMBER" }], NOW)).toEqual(["email"]); + }); + test("an unsure call, a contradiction, or a decision on the list alone goes to you to check", () => { expect(combine(input(), answers({ action: choice("contact_now", 0.5) })).reviewReasons).toEqual(["It isn't sure about the next step (confidence 50%)."]); const contradiction = combine(input(), answers({ need: { type: "noul", noul: 0.05 }, timing: { type: "noul", noul: 0.05 }, authority: { type: "noul", noul: 0.1 }, fit: { type: "noul", noul: 0.4 } })); diff --git a/src/domains/decide/model.ts b/src/domains/decide/model.ts index 69f7d94..31624dd 100644 --- a/src/domains/decide/model.ts +++ b/src/domains/decide/model.ts @@ -82,11 +82,26 @@ const day = (d: Date) => d.toISOString().slice(0, 10); const pct = (x: number) => `${Math.round(x * 100)}%`; const SHARED_INBOX = /^(info|hello|hi|contact|office|admin|sales|support|team|bookings?|enquiries|inquiries|service|mail|frontdesk|reception)@/i; -/** The channels a prospect can be reached on: an email that hasn't bounced, their profiles, a phone the dialer didn't find wrong. */ -export function reachableOn(prospect: Prospect, history: Touch[]): Channel[] { +/** A dialed number and a phone, by their last ten digits: "(512) 555-0147" and "+1 512 555 0147" are one number. */ +function sameNumber(dialed: string, phone: string | undefined): boolean { + const digits = (s: string | undefined) => (s ?? "").replace(/\D/g, "").slice(-10); + return !!digits(dialed) && digits(dialed) === digits(phone); +} + +/** + * The channels a prospect can be reached on: an email that hasn't bounced, + * their profiles, and a phone, unless the dialer's latest call found that + * same number wrong or dead in the last 90 days. + */ +export function reachableOn(prospect: Prospect, history: Touch[], now: Date): Channel[] { const bounced = history.some((t) => t.channel === "EMAIL" && (t.kind === "BOUNCED" || t.kind === "UNDELIVERABLE")); const lastCall = latestCall(history); - const deadNumber = !!lastCall && DEAD_NUMBERS.has(lastCall.disposition); + const deadNumber = + !!lastCall && + DEAD_NUMBERS.has(lastCall.disposition) && + now.getTime() - Date.parse(lastCall.touch.occurredAt) < CALL_MEMORY_DAYS * 86_400_000 && + // A call to a number they no longer have says nothing about the one they have now. + (!lastCall.touch.toAddress || sameNumber(lastCall.touch.toAddress, prospect.phone)); const found: Array<[Channel, unknown]> = [ ["email", prospect.email && !bounced], ["linkedin", prospect.linkedinUrl], @@ -134,7 +149,7 @@ export function decisionState(input: DecisionInput): Record { today: day(input.now), seller: { sells_to: offer.audience, offers: offer.offer, outcome: offer.outcome, pain_it_removes: offer.pain, proof: offer.proof, writes_as: input.sender.voice === "company" ? "the company (we)" : "one person (I)" }, prospect: listFacts(input.prospect), - can_reach_them_by: reachableOn(input.prospect, input.history), + can_reach_them_by: reachableOn(input.prospect, input.history, input.now), diligence: dossier && { summary: dossier.summary, offerings: short(dossier.offerings, 8), @@ -157,7 +172,7 @@ const describeAngle = (a: Angle) => `${ANGLE_GUIDE[a].label}: ${ANGLE_GUIDE[a].w /** The questions asked about every prospect: ten at most, so Jev answers them in one request. */ export function outboundQuestions(input: DecisionInput): Record { - const channels = reachableOn(input.prospect, input.history); + const channels = reachableOn(input.prospect, input.history, input.now); const questions: Record = { fit: { type: "noul", @@ -218,7 +233,7 @@ export function scoreOf(f: Record): number { */ export function combine(input: DecisionInput, result: DecisionResult & { provider: "jev" | "claude" }, reviewBelow = REVIEW_BELOW): Decision { const a = result.answers; - const channels = reachableOn(input.prospect, input.history); + const channels = reachableOn(input.prospect, input.history, input.now); const budget = a.budget?.type === "score" ? a.budget.score / 3 : 0.5; const factors: Decision["factors"] = { fit: { value: noul(a.fit), reason: a.fit?.reason }, From 76a0ee7bb88f6ae5b7632be9cbaba2bb026e0cb7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 09:26:34 +0000 Subject: [PATCH 3/8] fix: one saved list per listening watch, even when two share a name Save as a list named the file after the watch's name alone, so two watches with the same name wrote into one prospects-listen-.csv and a campaign on it reached both watches' matches. The file is now named for the watch and its id, prospects-listen--.csv; every watch together stays prospects-listen-all.csv. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuVXF5k9RXQpSdqYkdV --- CHANGELOG.md | 5 +++-- src/domains/web/listen/index.ts | 4 +++- src/domains/web/listen/listen.test.ts | 12 ++++++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0e6c9b..25b5750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,8 +67,9 @@ delivered, and each watch is a prospect source, `listen:`: the trigger line is the post itself, quoted, and a post on X carries its author's handle for an X DM. **Save as a list** in Settings writes a - watch's matches to one of your own lists to fill in and research; saving - again only adds what is new. Prospect sources are defined once, so + watch's matches to one of your own lists to fill in and research, one + list per watch even when two share a name; saving again only adds what is + new. Prospect sources are defined once, so research and campaigns accept the same ones. - **X DMs go out from COLD.** Add your own X developer app in Settings (`X_CLIENT_ID`, `X_CLIENT_SECRET`) and press **Connect X** for an account: diff --git a/src/domains/web/listen/index.ts b/src/domains/web/listen/index.ts index 9b73cbb..8ac2718 100644 --- a/src/domains/web/listen/index.ts +++ b/src/domains/web/listen/index.ts @@ -69,7 +69,9 @@ export function listenRoutes(ctx: WebContext): Route[] { const name = watch === "all" ? "all" : (await ctx.listening.watches()).find((w) => w.id === watch)?.name; if (!name) throw new HttpError(404, "No matches have come in from that watch yet."); const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60) || "matches"; - const file = `prospects-listen-${slug}.csv`; + // One list per watch, named for it and its id: two watches with the same name never share a list. + const id = watch.toLowerCase().replace(/[^a-z0-9-]+/g, "-").slice(-40).replace(/^-+|-+$/g, "") || "watch"; + const file = watch === "all" ? "prospects-listen-all.csv" : `prospects-listen-${slug}-${id}.csv`; const existing = await ctx.files.read(file).catch(() => undefined); const { text, added, rows } = listenCsv(await ctx.listening.items({ watch: watch === "all" ? undefined : watch, limit: 5000 }), existing); await ctx.files.write(file, text); diff --git a/src/domains/web/listen/listen.test.ts b/src/domains/web/listen/listen.test.ts index b057c71..4424502 100644 --- a/src/domains/web/listen/listen.test.ts +++ b/src/domains/web/listen/listen.test.ts @@ -104,8 +104,8 @@ describe("matches as prospects", () => { test("saved as a list, a match can be filled in, and saving again only adds what is new", async () => { const { data } = await call<{ source: string; added: number; rows: number; message: string }>("POST", "/api/listen/export", { watch: "01JWATCH" }); - expect(data).toMatchObject({ source: "csv:prospects-listen-needs-booking-software.csv", added: 2, rows: 2 }); - const file = path.join(root, "prospects-listen-needs-booking-software.csv"); + expect(data).toMatchObject({ source: "csv:prospects-listen-needs-booking-software-01jwatch.csv", added: 2, rows: 2 }); + const file = path.join(root, "prospects-listen-needs-booking-software-01jwatch.csv"); const text = readFileSync(file, "utf8"); expect(text.split("\n")[0]).toBe("id,first_name,full_name,company,email,website,x_handle,trigger,post_url,post_title,post_text,posted_at,listen_source,listen_watch"); writeFileSync(file, text.replace("twitter_search:1850,Dana,Dana Reyes,,", "twitter_search:1850,Dana,Dana Reyes,Clearview Tint,")); @@ -117,4 +117,12 @@ describe("matches as prospects", () => { expect(plan.due[0]!.text).toContain('Saw your post on X: "Is there a booking page that turns Google reviews into appointments?".'); expect(plan.due[0]!.text).toContain("for Clearview Tint?"); }); + + test("two watches with the same name keep a list each", async () => { + const twin = { ...delivery, watch: { id: "01JTWIN", name: "Needs booking software" }, items: [{ key: "hackernews:991", source: { label: "Hacker News", kind: "hackernews" }, item: { occurred_at: "2026-11-02T14:40:00Z", title: "Ask HN: booking software for a small shop?", url: "https://news.ycombinator.com/item?id=991", author: "pg_fan" } }] }; + expect((await post(twin, { authorization: "Bearer a-long-listening-secret" })).status).toBe(200); + const { data } = await call<{ source: string; added: number; rows: number }>("POST", "/api/listen/export", { watch: "01JTWIN" }); + expect(data).toMatchObject({ source: "csv:prospects-listen-needs-booking-software-01jtwin.csv", added: 1, rows: 1 }); + expect(readFileSync(path.join(root, "prospects-listen-needs-booking-software-01jwatch.csv"), "utf8")).not.toContain("hackernews:991"); + }); }); From 8f6813577eb24315f9a9db2644fb7f4adaa3b4ee Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 09:27:10 +0000 Subject: [PATCH 4/8] fix: the listening webhook stops reading at its size limit A delivery sent in chunks has no Content-Length, so the webhook read the whole body with request.text() before it compared it with the 2 MB limit: a large or endless body was held in memory first and refused only after. The body is now read a chunk at a time and the request is cancelled as soon as it passes the limit, in bytes rather than characters. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuVXF5k9RXQpSdqYkdV --- src/domains/web/listen/index.ts | 36 ++++++++++++++++++++++++--- src/domains/web/listen/listen.test.ts | 18 ++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/domains/web/listen/index.ts b/src/domains/web/listen/index.ts index 8ac2718..415e5a8 100644 --- a/src/domains/web/listen/index.ts +++ b/src/domains/web/listen/index.ts @@ -13,6 +13,35 @@ function sameSecret(given: string | null, expected: string): boolean { return timingSafeEqual(a, b); } +/** + * The body as text, read until it passes `max` bytes: undefined then, and the + * rest is never read, so a body without a Content-Length (sent in chunks) is + * refused without holding all of it in memory first. + */ +async function readCapped(request: Request, max: number): Promise { + if (!request.body) return ""; + const reader = request.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > max) { + await reader.cancel().catch(() => undefined); + return undefined; + } + chunks.push(value); + } + const bytes = new Uint8Array(size); + let at = 0; + for (const chunk of chunks) { + bytes.set(chunk, at); + at += chunk.byteLength; + } + return new TextDecoder().decode(bytes); +} + /** * OpenMagpie's webhook action delivers matches here (POST, instant or * digest). It proves itself with `Authorization: Bearer `, @@ -25,9 +54,10 @@ export function listenWebhook(ctx: WebContext): (request: Request) => Promise MAX_LISTEN_BYTES) return json({ error: "That delivery is too large: use a shorter digest interval." }, 413); - const text = await request.text(); - if (text.length > MAX_LISTEN_BYTES) return json({ error: "That delivery is too large: use a shorter digest interval." }, 413); + const tooLarge = () => json({ error: "That delivery is too large: use a shorter digest interval." }, 413); + if (Number(request.headers.get("content-length") ?? 0) > MAX_LISTEN_BYTES) return tooLarge(); + const text = await readCapped(request, MAX_LISTEN_BYTES); + if (text === undefined) return tooLarge(); let body: unknown; try { body = JSON.parse(text); diff --git a/src/domains/web/listen/listen.test.ts b/src/domains/web/listen/listen.test.ts index 4424502..b0e4094 100644 --- a/src/domains/web/listen/listen.test.ts +++ b/src/domains/web/listen/listen.test.ts @@ -3,6 +3,7 @@ import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:f import { tmpdir } from "node:os"; import path from "node:path"; import { openDatabase } from "../../db"; +import { MAX_LISTEN_BYTES } from "../../listen"; import { createApi } from "../api"; import { WebContext } from "../context"; import type { Campaign, ListenStatusResponse, PlanResponse, ProspectSummary, SourceOption } from "../types"; @@ -65,6 +66,23 @@ describe("OpenMagpie's webhook", () => { expect(await ctx.listening!.items()).toEqual([]); }); + test("a delivery over the size limit is refused while it is read, sent in chunks with no Content-Length too", async () => { + process.env.COLD_LISTEN_SECRET = "a-long-listening-secret"; + const chunk = new Uint8Array(64 * 1024).fill(32); + let pulled = 0; + const body = new ReadableStream({ + pull(controller) { + if (++pulled > 100) controller.close(); + else controller.enqueue(chunk); + }, + }); + const request = new Request(`${ORIGIN}/api/listen/magpie`, { method: "POST", headers: { "content-type": "application/json", authorization: "Bearer a-long-listening-secret" }, body, duplex: "half" } as RequestInit); + expect(request.headers.get("content-length")).toBeNull(); + const response = await webhook(request); + expect(response.status).toBe(413); + expect(pulled * chunk.byteLength).toBeLessThan(MAX_LISTEN_BYTES + 4 * chunk.byteLength); + }); + test("a delivery is kept, and the same delivery again adds nothing", async () => { const auth = { authorization: "Bearer a-long-listening-secret" }; const first = await post(delivery, auth); From cefbb193bdfed0ed103b32a13635d77588433a10 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 09:27:52 +0000 Subject: [PATCH 5/8] perf: keep a listening delivery's matches a hundred to a statement ListenStore.receive wrote each match with its own awaited INSERT, so a 500-match digest made 500 round trips to the database inside one webhook request: on Vercel with Turso, enough to risk the function's time limit. Matches now go in one multi-row INSERT per hundred (seven values each, far under SQLite's limit), and ON CONFLICT still keeps each key once, a match repeated within one delivery included. The count of matches added is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuVXF5k9RXQpSdqYkdV --- src/domains/listen/index.ts | 22 ++++++++++++---------- src/domains/listen/listen.test.ts | 15 ++++++++++++++- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/domains/listen/index.ts b/src/domains/listen/index.ts index 4b1f843..9624e1a 100644 --- a/src/domains/listen/index.ts +++ b/src/domains/listen/index.ts @@ -12,23 +12,25 @@ export interface ListenWatch { latest: string; } +/** Matches written by one INSERT: seven values each, far under SQLite's limit on values in a statement. */ +const ROWS_PER_INSERT = 100; + /** The matches OpenMagpie sent, in the app's database. Each key is kept once. */ export class ListenStore { constructor(private readonly db: Db) {} - /** Keep new matches; one already kept (a redelivery) is left as it is. */ + /** + * Keep new matches, a hundred to a statement, so a 500-match digest is five + * writes rather than five hundred round trips to the database. One already + * kept (a redelivery, or twice in one delivery) is left as it is. + */ async receive(items: ListenItem[]): Promise<{ added: number }> { let added = 0; - for (const item of items) { + for (let i = 0; i < items.length; i += ROWS_PER_INSERT) { + const rows = items.slice(i, i + ROWS_PER_INSERT); const { changes } = await this.db.run( - "INSERT INTO listen_items (key, watch_id, watch_name, source_kind, occurred_at, received_at, item) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (key) DO NOTHING", - item.key, - item.watchId, - item.watchName, - item.sourceKind, - item.occurredAt ?? item.receivedAt, - item.receivedAt, - JSON.stringify(item), + `INSERT INTO listen_items (key, watch_id, watch_name, source_kind, occurred_at, received_at, item) VALUES ${rows.map(() => "(?, ?, ?, ?, ?, ?, ?)").join(", ")} ON CONFLICT (key) DO NOTHING`, + ...rows.flatMap((item) => [item.key, item.watchId, item.watchName, item.sourceKind, item.occurredAt ?? item.receivedAt, item.receivedAt, JSON.stringify(item)]), ); added += changes; } diff --git a/src/domains/listen/listen.test.ts b/src/domains/listen/listen.test.ts index ac73475..1ea188b 100644 --- a/src/domains/listen/listen.test.ts +++ b/src/domains/listen/listen.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { openDatabase } from "../db"; +import { openDatabase, type Db } from "../db"; import { ListenStore, listenProspect, quoteOf, readMagpie, triggerOf, type ListenItem } from "."; const NOW = new Date("2026-11-02T15:00:00Z"); @@ -93,4 +93,17 @@ describe("the matches kept", () => { expect((await store.items({ watch: "nope" })).length).toBe(0); expect(await store.watches()).toEqual([{ id: "01JWATCH", name: "Needs booking software", count: 4, latest: NOW.toISOString() }]); }); + + test("a delivery is kept a hundred matches to a statement, a match twice in one delivery once", async () => { + const db = await openDatabase(":memory:"); + let statements = 0; + const counted: Db = { get: db.get.bind(db), all: db.all.bind(db), batch: db.batch.bind(db), close: db.close.bind(db), run: (sql, ...params) => (statements++, db.run(sql, ...params)) }; + const store = new ListenStore(counted); + const [first] = readMagpie(digest, NOW).items; + const many = Array.from({ length: 250 }, (_, i) => ({ ...first!, key: `reddit_subreddit:t3_${i}` })); + expect(await store.receive([...many, many[0]!])).toEqual({ added: 250 }); + expect(statements).toBe(3); + expect(await store.receive(many)).toEqual({ added: 0 }); + expect((await store.items({ limit: 5000 })).length).toBe(250); + }); }); From 744bf60ec300aec859f7186ee75dc8c8424c51fc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 09:28:25 +0000 Subject: [PATCH 6/8] docs: claimRun and metaValue each keep their own comment claimRun was added between metaValue's doc comment and metaValue, so claimRun carried two comments, the first describing metaValue, and metaValue had none. Each now sits above its own function. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuVXF5k9RXQpSdqYkdV --- src/domains/db/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domains/db/index.ts b/src/domains/db/index.ts index 695cff5..36e148b 100644 --- a/src/domains/db/index.ts +++ b/src/domains/db/index.ts @@ -367,7 +367,6 @@ export function databaseFromEnv(env: Record = proces return undefined; } -/** A value stored once and kept: the first call makes it, later calls read it. */ /** * Claim a job that runs now and then (a look for new leads, logging calls) * for every server on this database: true when it last ran at least @@ -384,6 +383,7 @@ export async function claimRun(db: Db, key: string, now: Date, everyMs: number): return changes > 0; } +/** A value stored once and kept: the first call makes it, later calls read it. */ export async function metaValue(db: Db, key: string, make: () => string): Promise { const found = await db.get<{ value: string }>("SELECT value FROM meta WHERE key = ?", key); if (found) return found.value; From dab0abfd27b796928b457e6c61d03998fc2afd78 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 09:28:45 +0000 Subject: [PATCH 7/8] refactor: copy the default calling hours in one place parseCalls copied the default days in one branch and returned a shallow copy that shared DEFAULT_CALL_HOURS.days in the other. That copy never left parseConfig (a calls: section that isn't a mapping fails the whole config), but both branches now go through one helper that copies the days, and a test checks a parsed config has its own. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuVXF5k9RXQpSdqYkdV --- src/domains/config/helpers/parseConfig.test.ts | 3 +++ src/domains/config/helpers/parseConfig.ts | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/domains/config/helpers/parseConfig.test.ts b/src/domains/config/helpers/parseConfig.test.ts index ec87eb2..bc3cf31 100644 --- a/src/domains/config/helpers/parseConfig.test.ts +++ b/src/domains/config/helpers/parseConfig.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import path from "node:path"; import { parse } from "yaml"; +import { DEFAULT_CALL_HOURS } from "../types"; import { DEFAULT_FOOTERS, parseConfig } from "./parseConfig"; const example = parse(readFileSync(path.resolve(import.meta.dir, "../../../../cold.example.yaml"), "utf8")); @@ -31,6 +32,8 @@ describe("parseConfig", () => { test("the calls section is optional: 8am to 9pm every day, in the prospect's time", () => { expect(parseConfig(example, "x").config!.calls).toEqual({ start: "08:00", end: "21:00", days: ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] }); + // Each config has its own copy of the default days: editing one never edits the defaults. + expect(parseConfig(example, "x").config!.calls.days).not.toBe(DEFAULT_CALL_HOURS.days); const tuned = parseConfig({ ...example, calls: { start: "09:00", end: "18:30", days: ["Mon", "tue", "wed", "thu", "fri"] } }, "x"); expect(tuned.config!.calls).toEqual({ start: "09:00", end: "18:30", days: ["mon", "tue", "wed", "thu", "fri"] }); expect(parseConfig({ ...example, calls: { end: "20:00" } }, "x").config!.calls.start).toBe("08:00"); diff --git a/src/domains/config/helpers/parseConfig.ts b/src/domains/config/helpers/parseConfig.ts index 11f1d38..b9e15d0 100644 --- a/src/domains/config/helpers/parseConfig.ts +++ b/src/domains/config/helpers/parseConfig.ts @@ -168,12 +168,15 @@ function isTimezone(name: string): boolean { } } +/** The default calling hours as a config's own copy, so nothing that edits one config's days edits the defaults. */ +const defaultCallHours = (): CallHours => ({ ...DEFAULT_CALL_HOURS, days: [...DEFAULT_CALL_HOURS.days] }); + /** The optional calls: section: when to call, in each prospect's own time. Every key has a default. */ function parseCalls(raw: unknown, errors: string[]): CallHours { - if (raw === undefined || raw === null) return { ...DEFAULT_CALL_HOURS, days: [...DEFAULT_CALL_HOURS.days] }; + if (raw === undefined || raw === null) return defaultCallHours(); if (!isObject(raw)) { errors.push("calls: must be a mapping (start, end, days)"); - return { ...DEFAULT_CALL_HOURS }; + return defaultCallHours(); } for (const key of Object.keys(raw)) if (!["start", "end", "days"].includes(key)) errors.push(`calls.${key}: not a calls setting (start, end, days)`); const start = text(raw.start) || DEFAULT_CALL_HOURS.start; From e840adb60222ad83c8d55b0bb2f328f927fa9139 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 09:29:15 +0000 Subject: [PATCH 8/8] docs: rewrap the quiz leads entry in the changelog Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuVXF5k9RXQpSdqYkdV --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25b5750..2e3cd94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,11 +52,11 @@ come as one), tapping it opens the lead; it looks once a minute across every server on the database, never pushes a lead twice, retries one that failed, never pushes leads from before it was set up, and takes more than - one look's 500 new leads over the next looks, oldest first, skipping none. Settings, Lead - alerts sends a test. A lead who opted out anywhere, "do not call" on the - dialer included, is Do Not Contact: last on Calls, with no Call button, - and never pushed. Their answers read whether offer writes the JSON on one - line or several. + one look's 500 new leads over the next looks, oldest first, skipping + none. Settings, Lead alerts sends a test. A lead who opted out anywhere, + "do not call" on the dialer included, is Do Not Contact: last on Calls, + with no Call button, and never pushed. Their answers read whether offer + writes the JSON on one line or several. - **Picking a dialer prospect fills the script in again.** Prospects from `twenty:agencyProspects` and `twenty:agencyPeople` were never found by their key in a call sheet or a research run, so the script kept its