From ee1ea3b2298c3b40aee0e5f5f1d170dfd6f945fe Mon Sep 17 00:00:00 2001 From: operatoruplift Date: Sun, 1 Nov 2026 15:26:55 +0000 Subject: [PATCH 1/6] Send X DMs through X's official API, one approved message per click - Connect X in Settings: your own X developer app (X_CLIENT_ID, X_CLIENT_SECRET), OAuth 2.0 with PKCE and a one-time state. X sends you back to /api/x/callback, which is trusted through that state rather than the session cookie (a strict cookie doesn't come back from x.com). - Send on X in a campaign's X queue: the DM you approved, sent from your account through X's API, within the daily cap, logged with X's message id. Each step is claimed in the database first, so two clicks or two servers send it once. - A follow-up is checked against the conversation first: when they have answered, the reply is logged and nothing is sent. - Check X DMs on the Inbox logs answers to earlier DMs, which stops those sequences. - X sign-ins live in the database, sealed with COLD_SECRET_KEY (required on a hosted server). An X sign-in problem answers 409, never 401, so the app doesn't sign you out of cold. - LinkedIn, Instagram and Facebook stay copy and open. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuUQ7RbYzBy4coULGCu --- .env.example | 8 + CHANGELOG.md | 11 + README.md | 26 +- docs/compliance.md | 19 +- docs/deploy.md | 6 +- src/domains/db/db.test.ts | 6 +- src/domains/db/index.ts | 29 +++ src/domains/dm/index.ts | 15 +- src/domains/storage/index.ts | 4 +- src/domains/web/api/index.ts | 93 +++++++ src/domains/web/context/index.ts | 23 +- src/domains/web/guard/guard.test.ts | 7 + src/domains/web/guard/index.ts | 6 +- src/domains/web/server/handler.ts | 5 + src/domains/web/server/index.ts | 4 +- src/domains/web/types.ts | 27 +++ src/domains/web/vercel/index.ts | 4 +- src/domains/web/x/index.ts | 188 ++++++++++++++ src/domains/web/x/x.test.ts | 200 +++++++++++++++ src/domains/xApi/helpers/handleOf.ts | 8 + src/domains/xApi/helpers/helpers.test.ts | 107 ++++++++ src/domains/xApi/helpers/index.ts | 4 + src/domains/xApi/helpers/pkce.ts | 31 +++ src/domains/xApi/helpers/repliesIn.ts | 23 ++ src/domains/xApi/helpers/xFailure.ts | 48 ++++ src/domains/xApi/index.ts | 296 +++++++++++++++++++++++ src/domains/xApi/types.ts | 83 +++++++ src/domains/xApi/xApi.test.ts | 185 ++++++++++++++ src/web/pages/campaign.tsx | 34 ++- src/web/pages/replies.tsx | 31 ++- src/web/pages/settings.tsx | 100 +++++++- 31 files changed, 1595 insertions(+), 36 deletions(-) create mode 100644 src/domains/web/x/index.ts create mode 100644 src/domains/web/x/x.test.ts create mode 100644 src/domains/xApi/helpers/handleOf.ts create mode 100644 src/domains/xApi/helpers/helpers.test.ts create mode 100644 src/domains/xApi/helpers/index.ts create mode 100644 src/domains/xApi/helpers/pkce.ts create mode 100644 src/domains/xApi/helpers/repliesIn.ts create mode 100644 src/domains/xApi/helpers/xFailure.ts create mode 100644 src/domains/xApi/index.ts create mode 100644 src/domains/xApi/types.ts create mode 100644 src/domains/xApi/xApi.test.ts diff --git a/.env.example b/.env.example index 150563d..20be7a7 100644 --- a/.env.example +++ b/.env.example @@ -42,6 +42,14 @@ SUBQ_API_KEY= # SUBQ_BASE_URL=https://api.subq.ai/v1 # SUBQ_MODEL=subq-preview +# X DMs (optional): your own X developer app at developer.x.com, with OAuth +# 2.0 on (type Web App) and the callback URL Settings shows, e.g. +# http://localhost:4100/api/x/callback on your computer. Then Connect X in +# Settings. The app's X API plan has to include Direct Messages. cold sends +# one approved DM per click, from your own account, never on its own. +X_CLIENT_ID= +X_CLIENT_SECRET= + # Hosting (docs/deploy.md). These belong in the server's real environment, # not in this file, since .env.local lives inside COLD_HOME. # COLD_PUBLIC_URL=https://cold.example.com diff --git a/CHANGELOG.md b/CHANGELOG.md index dffac30..04e63b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +- **X DMs go out from COLD.** Add your own X developer app in Settings + (`X_CLIENT_ID`, `X_CLIENT_SECRET`) and press **Connect X** for an account: + a campaign's X queue then has **Send on X**, one approved DM per click, + sent from your own account through X's API and logged with X's message + id. A step goes out once, even after two clicks or on two servers. A + follow-up is never sent to someone who has answered since: their reply is + logged instead and the sequence stops for them. **Check X DMs** on the + Inbox logs answers to earlier DMs. X sign-ins are sealed with + `COLD_SECRET_KEY` when it is set, which a hosted server requires. + LinkedIn, Instagram and Facebook stay copy and open. + - **The logs say which database each deployment is on**: at start, a fingerprint of its address (never the address or the token), the variable it came from, and when COLD first used it. If the fingerprint changes with diff --git a/README.md b/README.md index 558aad4..eea23d1 100644 --- a/README.md +++ b/README.md @@ -30,14 +30,15 @@ things in one small server: - **The engine underneath**: pick a sequence, point it at a list, and COLD works out who is due for which step today. It writes each message with the prospect's details, sends the emails from your own mailbox, lines up the DMs - for you to send, and prepares your calls. It stops for anyone who replied, - bounced or asked to be left alone, and logs every touch in Twenty. + for you to send (on X, with one click through X's API), and prepares your + calls. It stops for anyone who replied, bounced or asked to be left alone, + and logs every touch in Twenty. Claude does the research and the first drafts: it finds businesses that fit your offer, reads their public footprint for one true reason to reach out, personalizes each message, preps each call and reads each reply. **You approve -everything.** Nothing is sent by AI on its own, DMs are always sent by you, and -calls are always made by you. +everything.** Nothing is sent by AI on its own, DMs are always sent by you +(on X, by your click on Send on X), and calls are always made by you. Every message comes in two voices: **personal**, sent from your own name and profiles, and **company**, sent from the brand or a rep. It works with any @@ -51,7 +52,7 @@ call scripts and the Do Not Contact list. - **An app that's yours alone.** `bun run ui` opens it on your computer, or host it and install it on your phone. Campaigns that show who is due today, a Send button with live progress you can stop, a DM queue with Copy and Open - buttons, call scripts, the message library, the inbox, analytics, your + buttons (and Send on X), call scripts, the message library, the inbox, analytics, your waitlist and settings, behind a login only you have. Your mailbox passwords and API keys stay on your machine or your server. - **AI that shows its work.** Claude finds prospects and researches them with @@ -83,9 +84,12 @@ call scripts and the Do Not Contact list. and the touch is kept locally, then saved to Twenty before the next run plans anything. - **DMs that keep your account.** LinkedIn, X and Instagram restrict accounts - that send DMs from scripts, so `cold` never does. It writes the next DM due, - copies it, opens the profile, and logs what happened when you say. It keeps - a daily cap per platform. + that send DMs from scripts, so `cold` never scripts one. It writes the next + DM due, copies it, opens the profile, and logs what happened when you say. + On X you can connect your account instead (your own X developer app, in + Settings): **Send on X** sends the DM you approved through X's own API, + one per click, checks a follow-up against the conversation first, and + **Check X DMs** logs the answers. It keeps a daily cap per platform. - **Calls in the dialer.** One command writes the call scripts and objection answers into the dialer's `agencyScripts`, so they sit next to every live call. For a handset, `cold call sheet` prints one filled in for a prospect. @@ -149,13 +153,13 @@ bun run cold preview founder-direct --source csv:examples/prospects.csv --prospe | Home | What's been sent, replies and opt-outs this week, and what's due in each campaign. | | Prospects | Find new prospects with Claude and web search, save them as a list; research a list and save a trigger line per person. | | Campaigns | A sequence, the account that sends it, an offer and a prospect list (upload a CSV, use an AI list, or use Twenty). | -| Campaign | Due now, waiting, skipped and held back, with reasons. Personalize any due message with AI and approve it. Emails: Send, with live progress. DMs: the queue, plus a checklist download for a VA. | +| Campaign | Due now, waiting, skipped and held back, with reasons. Personalize any due message with AI and approve it. Emails: Send, with live progress. DMs: the queue (on X, Send on X), plus a checklist download for a VA. | | Sequences | Every message, previewed with real data, editable in place. | | Calls | Every script and objection answer, filled in for a prospect, and AI call prep. Sync scripts to the dialer. | -| Inbox | Check an inbox for replies, opt-outs and bounces. Claude reads each reply, sorts it (interested, not now, wants out...) and drafts your answer. | +| Inbox | Check an inbox for replies, opt-outs and bounces, and your X DMs for answers. Claude reads each reply, sorts it (interested, not now, wants out...) and drafts your answer. | | Analytics | Reply rate per sequence and per message. | | Waitlist | Everyone who joined on your site, with their role and channels. Mark them invited, export a CSV. | -| Settings | Your login and devices, the AI connection, `cold.yaml` (checked before saving), mailbox URLs, the Twenty key, and a test for each. | +| Settings | Your login and devices, the AI connection, `cold.yaml` (checked before saving), mailbox URLs, the Twenty key, Connect X, and a test for each. | ### The site and the waitlist diff --git a/docs/compliance.md b/docs/compliance.md index 0b27418..3b6612c 100644 --- a/docs/compliance.md +++ b/docs/compliance.md @@ -23,7 +23,8 @@ the stakes are high. address and is flagged, so you can find who it forwards for. - **Keeps subjects honest.** Lint rejects a subject that pretends to be a reply or a forward. Only real follow-ups in a thread start with "Re:". -- **Stops on bounces and invalid addresses**, and never automates DMs. +- **Stops on bounces and invalid addresses**, and never automates DMs: an X + DM goes out only when you press Send on X, one approved message at a time. - **Keeps a record.** Every send, reply, bounce and opt-out is a `coldTouch` with a timestamp: what you sent, to whom, and when they said no. @@ -85,9 +86,19 @@ within 5 business days. LinkedIn's User Agreement prohibits bots and other automated ways of sending messages and invitations. X's rules prohibit bulk, aggressive or unsolicited automated DMs. Meta's terms prohibit automated access to Instagram and -Facebook. `cold` never sends a DM: it writes it, you send it. Keep the daily -caps human, and when someone says no in a DM, log it (`o` in `cold dm run`, or -`cold log --kind OPTED_OUT`). That stops every channel. +Facebook. `cold` never automates a DM: + +- **X.** With an X account connected in Settings (your own X developer app, + through X's official API), **Send on X** sends the DM you approved, from + your account, when you press it: one message per click, within the daily + cap, never in a batch or on a timer. A follow-up is checked against the + conversation first and is not sent when they have answered. Without a + connection, X works like the others. +- **LinkedIn, Instagram and Facebook.** `cold` writes the message, copies it + and opens the profile; you send it yourself. + +Keep the daily caps human, and when someone says no in a DM, log it (`o` in +`cold dm run`, or `cold log --kind OPTED_OUT`). That stops every channel. ## Data and records diff --git a/docs/deploy.md b/docs/deploy.md index e36fc9a..1bebdf9 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -31,8 +31,10 @@ and AI drafts live in Turso, which is SQLite over the network. 3. **Settings, Environment Variables**, as Sensitive, for Production and Preview: - `COLD_SECRET_KEY`: a long random string (`openssl rand -base64 36`). It - encrypts the keys and passwords you save in COLD's Settings; change it - and the saved ones can no longer be read. + encrypts the keys and passwords you save in COLD's Settings, and the X + sign-ins of Connect X; change it and the saved ones can no longer be + read. To send X DMs, register `https:///api/x/callback` as + your X developer app's callback URL (Settings, X shows the exact one). - `CRON_SECRET`: another random string. Vercel's cron sends it to `/api/cron/tick`, and COLD refuses the tick without it. - `COLD_PUBLIC_URL`, for Production only, if people should use a custom diff --git a/src/domains/db/db.test.ts b/src/domains/db/db.test.ts index 66faef0..e4a6e9b 100644 --- a/src/domains/db/db.test.ts +++ b/src/domains/db/db.test.ts @@ -46,10 +46,10 @@ describe("the database", () => { test("a new database gets every table, and migrating again changes nothing", async () => { const db = await openDatabase(":memory:"); const tables = (await db.all<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")).map((t) => t.name); - expect(tables).toEqual(["ai_drafts", "ai_runs", "decisions", "files", "meta", "owner", "rate_limits", "score_job_items", "score_jobs", "secrets", "send_events", "send_jobs", "sessions", "style_assignments", "touches", "waitlist"]); - expect((await db.get<{ value: string }>("SELECT value FROM meta WHERE key = 'schema_version'"))?.value).toBe("5"); + expect(tables).toEqual(["ai_drafts", "ai_runs", "decisions", "files", "meta", "owner", "rate_limits", "score_job_items", "score_jobs", "secrets", "send_events", "send_jobs", "sessions", "style_assignments", "touches", "waitlist", "x_connections", "x_oauth_states", "x_sends", "x_users"]); + expect((await db.get<{ value: string }>("SELECT value FROM meta WHERE key = 'schema_version'"))?.value).toBe("6"); await migrate(db); - expect((await db.get<{ value: string }>("SELECT value FROM meta WHERE key = 'schema_version'"))?.value).toBe("5"); + expect((await db.get<{ value: string }>("SELECT value FROM meta WHERE key = 'schema_version'"))?.value).toBe("6"); }); test("a file from before the version moved into meta keeps its data and gains the new tables", async () => { diff --git a/src/domains/db/index.ts b/src/domains/db/index.ts index 5e989e4..1b4a521 100644 --- a/src/domains/db/index.ts +++ b/src/domains/db/index.ts @@ -249,6 +249,35 @@ const MIGRATIONS: string[] = [ PRIMARY KEY (job_id, seq) ); CREATE INDEX score_job_items_status ON score_job_items (job_id, status);`, + // X DMs through the official API: who each account is connected as (tokens + // sealed with COLD_SECRET_KEY when set), Connect X attempts, handles looked + // up, and each step sent once. + `CREATE TABLE x_connections ( + account TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + username TEXT NOT NULL, + tokens TEXT NOT NULL, + connected_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE x_oauth_states ( + state TEXT PRIMARY KEY, + account TEXT NOT NULL, + verifier TEXT NOT NULL, + redirect_uri TEXT NOT NULL, + created_at TEXT NOT NULL + ); + CREATE TABLE x_users (handle TEXT PRIMARY KEY, user_id TEXT NOT NULL, updated_at TEXT NOT NULL); + CREATE TABLE x_sends ( + account TEXT NOT NULL, + prospect_key TEXT NOT NULL, + sequence TEXT NOT NULL, + step INTEGER NOT NULL, + status TEXT NOT NULL, + event_id TEXT, + created_at TEXT NOT NULL, + PRIMARY KEY (account, prospect_key, sequence, step) + );`, ]; const statements = (script: string): Statement[] => diff --git a/src/domains/dm/index.ts b/src/domains/dm/index.ts index bb194b2..0d9e0e9 100644 --- a/src/domains/dm/index.ts +++ b/src/domains/dm/index.ts @@ -146,9 +146,17 @@ export async function prepareDmRun(input: { sequence: Sequence; account: Account /** * Log what happened to one DM: sent (this step), or a reply or opt-out * (against the step they answered). An opt-out also marks the source record - * Do Not Contact. Throws when the store cannot save it. + * Do Not Contact. `extra` carries what the platform's API said: the message's + * id, and a reply's words. Throws when the store cannot save it. */ -export async function recordDmOutcome(store: TouchStore, prepared: Pick, item: DmItem, outcome: DmOutcome, now: Date = new Date()): Promise { +export async function recordDmOutcome( + store: TouchStore, + prepared: Pick, + item: DmItem, + outcome: DmOutcome, + now: Date = new Date(), + extra: { messageId?: string; body?: string } = {}, +): Promise { const sent = outcome === "SENT"; await store.record({ kind: outcome, @@ -161,7 +169,8 @@ export async function recordDmOutcome(store: TouchStore, prepared: Pick(request: Request): Promise { } } +/** + * X's refusal as an HTTP answer that says what to do next. Never 401: to the + * app a 401 means you were signed out of cold, not that X needs reconnecting. + */ +export function xHttpError(err: unknown): unknown { + if (!(err instanceof XApiError)) return err; + const status = err.kind === "rateLimited" ? 429 : err.kind === "notFound" ? 404 : err.kind === "other" ? 502 : 409; + return new HttpError(status, err.message); +} + +/** The X store, X's API and cold.yaml, or a refusal that says which is missing. */ +export async function xTools(ctx: WebContext): Promise<{ store: XStore; client: XClient; config: ColdConfig }> { + if (!ctx.x) throw new HttpError(409, "Sending X DMs needs the app's database, which this server doesn't have."); + const client = ctx.xClient(); + if (!client) throw new HttpError(409, "Set up X first: add your X developer app's X_CLIENT_ID (and X_CLIENT_SECRET) in Settings."); + const { config, error, isExample } = await ctx.config(); + if (!config) throw new HttpError(400, error ?? "cold.yaml has problems."); + if (isExample) throw new HttpError(409, "cold.yaml does not exist yet: create it in Settings, so X is connected to your own account."); + return { store: ctx.x, client, config }; +} + +/** They answered on X since cold's last DM, so the next one is not sent. */ +class RepliedOnX extends Error { + constructor(readonly reply: XDmEvent) { + super("They replied on X."); + } +} + /** One API route: method, path pattern (its groups become params), handler. */ export type Route = [string, RegExp, (request: Request, params: string[], url: URL) => Promise | Response]; @@ -80,6 +111,8 @@ function envFields(config: ColdConfig | undefined): SettingsResponse["env"] { fields.set("SUBQ_API_KEY", "Optional: SubQ, Subquadratic's long-context model, reads a prospect's whole site for diligence. Without it Claude does (subq.ai)"); fields.set("REACHER_URL", "Optional: a self-hosted Reacher, e.g. http://localhost:8080"); fields.set("REACHER_SECRET", "Optional: Reacher's header secret"); + fields.set("X_CLIENT_ID", "Optional: your X developer app's OAuth 2.0 Client ID, so X DMs go out from cold through X's API (developer.x.com). Then Connect X below"); + fields.set("X_CLIENT_SECRET", "Optional: that app's OAuth 2.0 Client Secret (a Web App; leave it empty for a Native App)"); return [...fields.entries()].map(([name, hint]) => ({ name, set: !!process.env[name]?.trim(), hint })); } @@ -191,9 +224,18 @@ export function createApi(ctx: WebContext, extraRoutes: Route[] = []): (request: sentToday: prepared.sentToday, sendable: prepared.batch.length, blockers, + x: prepared.platform === "x" ? await xSendState(account.key) : undefined, }; } + /** Whether an account can send X DMs through X's API, and if not, why. */ + async function xSendState(account: string): Promise { + if (!ctx.x) return { ready: false, why: "This server has no database to keep an X sign-in." }; + if (!ctx.xClient()) return { ready: false, why: "Add your X developer app in Settings to send from here." }; + const connected = (await ctx.x.accounts()).find((a) => a.account === account); + return connected ? { ready: true, username: connected.username } : { ready: false, why: `Connect X for ${account} in Settings to send from here.` }; + } + /** Approved AI drafts replace the template's words for that prospect's step. */ const emailOverride = async (campaign: Campaign): Promise => { if (!ctx.ai) return undefined; @@ -493,6 +535,57 @@ export function createApi(ctx: WebContext, extraRoutes: Route[] = []): (request: return json({ ok: true }); }, ], + [ + "POST", + /^\/api\/campaigns\/([\w-]+)\/dm\/x$/, + async (request, [id]) => { + // One approved X DM, sent by the owner's click through X's API from their own account. + const campaign = await campaignOr404(id!); + const { prospect } = await readBody<{ prospect: string }>(request); + const { store: x, client, config } = await xTools(ctx); + const { sequence, account, offer } = resolveCampaign(campaign, (await ctx.library()).library, config); + if (sequence.channel !== "dm" || sequence.platform !== "x") throw new HttpError(400, "Only X DMs go out through X's API. On other platforms, copy the message and send it yourself."); + const store = await ctx.store(config); + const prepared = await withDmDrafts(await prepareDmRun({ sequence, account, offer, prospects: await ctx.prospects(campaign.source, campaign.where), store, now: () => ctx.now() }), campaign); + const item = prepared.queue.find((q) => q.prospect.key === prospect); + if (!item) throw new HttpError(409, "That DM is no longer due. Refresh the page."); + if (!prepared.batch.includes(item)) throw new HttpError(409, `The daily cap of ${prepared.cap} is used up for today.`); + const handle = handleOf(item.prospect.xHandle ?? item.url); + if (!handle) throw new HttpError(400, `${item.prospect.label} has no X handle cold can read: fix it in the list.`); + + const key = { account: account.key, prospectKey: item.prospect.key, sequence: sequence.id, step: item.step.step ?? 1 }; + const held = await x.claim(key); + if (held) throw new HttpError(409, held.status === "SENT" ? `This step already went out on X at ${held.at.slice(0, 16).replace("T", " ")} UTC. If it isn't logged yet, press "I sent it".` : "This DM is being sent right now: wait a moment, then refresh."); + + let eventId: string; + try { + eventId = await withXToken(x, client, account.key, () => ctx.now(), async (token, connection) => { + const userId = await xUserIdOf(x, client, token, handle); + if (item.previous) { + // A follow-up: never send it to someone who has answered since. + const events = await client.dmEvents(token, { participantId: userId, limit: 100 }); + const reply = repliesIn(events, { me: connection.userId, awaiting: new Map([[userId, { userId, sentAt: item.previous.occurredAt }]]), logged: new Set() })[0]; + if (reply) throw new RepliedOnX(reply); + } + return (await client.sendDm(token, userId, item.text)).eventId; + }); + } catch (err) { + await x.release(key); + if (err instanceof RepliedOnX) { + await recordDmOutcome(store, prepared, item, "REPLIED", new Date(err.reply.createdAt), { messageId: err.reply.id, body: err.reply.text }); + return json({ ok: true, outcome: "REPLIED", message: `${item.prospect.label} replied on X, so nothing was sent: the reply is logged and their sequence stops.` } satisfies XSendResponse); + } + throw xHttpError(err); + } + await x.markSent(key, eventId); + try { + await recordDmOutcome(store, prepared, item, "SENT", ctx.now(), { messageId: eventId }); + } catch (err) { + throw new HttpError(502, `Sent on X to @${handle}, but logging it failed (${(err as Error).message}). Press "I sent it" to log it.`); + } + return json({ ok: true, outcome: "SENT", message: `Sent on X to @${handle}.` } satisfies XSendResponse); + }, + ], [ "GET", /^\/api\/campaigns\/([\w-]+)\/checklist$/, diff --git a/src/domains/web/context/index.ts b/src/domains/web/context/index.ts index 5b0e6c2..16a63d9 100644 --- a/src/domains/web/context/index.ts +++ b/src/domains/web/context/index.ts @@ -7,8 +7,9 @@ import { claudeDecider, DecideStore, jevConfigured, jevProvider, subqConfigured, import { reacherFromEnv, smtpTransport, type MailTransport } from "../../email"; import { loadLibraryFrom } from "../../messages"; import { loadProspects, prospectsFromCsv, type Prospect } from "../../prospects"; -import { DbFiles, DiskFiles, EnvFileSecrets, type Files, type Secrets } from "../../storage"; +import { DbFiles, DiskFiles, EnvFileSecrets, secretKey, type Files, type Secrets } from "../../storage"; import { DbStore, filesJournal, openStore, twentyClient, TwentyStore, type TouchStore } from "../../touches"; +import { xAppFromEnv, XClient, XStore, type XFetch } from "../../xApi"; import { CampaignStore } from "../campaigns"; import { processScoreItem } from "../decide/engine"; import { ScoreJobs } from "../decide/jobs"; @@ -30,10 +31,21 @@ export interface WebContextOptions { /** How diligence reads prospects' websites. */ pages?: Pick; transport?: (account: Account) => MailTransport; + /** How cold reaches X's API (tests hand it a fake X). */ + xFetch?: XFetch; now?: () => Date; random?: () => number; } +/** COLD_SECRET_KEY as a key, or undefined when it is unset or unusable (the X store then says what to do). */ +function usableSecretKey(): Buffer | undefined { + try { + return secretKey(); + } catch { + return undefined; + } +} + const PROSPECT_TTL_MS = 5 * 60 * 1000; const STORE_TTL_MS = 5 * 60 * 1000; const CONFIG = "cold.yaml"; @@ -65,6 +77,8 @@ export class WebContext { readonly decisions?: DecideStore; /** Whole lists scored on the server, kept in the database; only with the database. */ readonly scoring?: ScoreJobs; + /** X connections, sign-ins and sends through X's API; only with the database. */ + readonly x?: XStore; private readonly prospectCache = new Map(); private storeCache?: { at: number; key: string; store: Promise }; private kept: { config?: Promise; library?: ReturnType } = {}; @@ -89,9 +103,16 @@ export class WebContext { now: () => this.now(), random: options.random, }); + this.x = new XStore(options.db, { key: usableSecretKey(), requireKey: this.storage === "database", now: () => this.now() }); } } + /** X's API through the owner's developer app, once X_CLIENT_ID is set. */ + xClient(): XClient | undefined { + const app = xAppFromEnv(); + return app ? new XClient(app, this.options.xFetch, () => this.now()) : undefined; + } + /** Where your files are: a folder on this computer, or the database. */ get storage(): "disk" | "database" { return this.files instanceof DbFiles ? "database" : "disk"; diff --git a/src/domains/web/guard/guard.test.ts b/src/domains/web/guard/guard.test.ts index 5f17b5d..7c5e735 100644 --- a/src/domains/web/guard/guard.test.ts +++ b/src/domains/web/guard/guard.test.ts @@ -26,6 +26,13 @@ describe("refuseRequest", () => { expect(refuseRequest(req("/api/health", { host: "10.0.0.7:8080" }), access)).toBeUndefined(); }); + test("lets X send the owner back to /api/x/callback, a plain page load, on this server's hosts only", () => { + expect(refuseRequest(req("/api/x/callback?state=s&code=c", { host: "127.0.0.1:4100" }), access)).toBeUndefined(); + expect(refuseRequest(req("/api/x/callback?state=s&code=c", { host: "evil.example" }), access)).toBe("cold does not answer on evil.example"); + expect(refuseRequest(new Request("http://127.0.0.1:4100/api/x/callback", { method: "POST", headers: { host: "127.0.0.1:4100" } }), access)).toBe("missing the x-cold header"); + expect(refuseRequest(req("/api/x/connect", { host: "127.0.0.1:4100" }), access)).toBe("missing the x-cold header"); + }); + test("a hosted server answers on its public name and origin", () => { const hosted = { hosts: ["cold.example.com"], origins: ["https://cold.example.com"] }; expect(refuseRequest(req("/api/session", { host: "cold.example.com", "x-cold": "1", origin: "https://cold.example.com" }), hosted)).toBeUndefined(); diff --git a/src/domains/web/guard/index.ts b/src/domains/web/guard/index.ts index a80f50c..4622c86 100644 --- a/src/domains/web/guard/index.ts +++ b/src/domains/web/guard/index.ts @@ -15,7 +15,10 @@ export interface Access { * - An Origin, when the browser sends one, must be this server's. * * /api/health is open to anyone, for a host's uptime checks; /api/cron/ - * carries its own secret instead (Vercel's cron calls it). + * carries its own secret instead (Vercel's cron calls it). /api/x/callback is + * where X sends the owner back after Connect X: a plain page load, so no + * header, answered on this server's own hosts only and trusted only through + * its one-time state. */ export function refuseRequest(request: Request, access: Access): string | undefined { const { pathname } = new URL(request.url); @@ -23,6 +26,7 @@ export function refuseRequest(request: Request, access: Access): string | undefi const host = request.headers.get("host") ?? ""; if (!access.hosts.includes(host)) return `cold does not answer on ${host || "this address"}`; if (!pathname.startsWith("/api/")) return undefined; + if (pathname === "/api/x/callback" && request.method === "GET") return undefined; if (request.headers.get("x-cold") !== "1") return "missing the x-cold header"; const origin = request.headers.get("origin"); if (origin && !access.origins.includes(origin)) return "cross-origin requests are not allowed"; diff --git a/src/domains/web/server/handler.ts b/src/domains/web/server/handler.ts index e751161..e5c1a91 100644 --- a/src/domains/web/server/handler.ts +++ b/src/domains/web/server/handler.ts @@ -161,6 +161,8 @@ export interface HandlerDeps { limiter?: (limit: number, windowMs: number) => Limiter; /** What Vercel's cron job runs every minute (the next emails of a send), behind CRON_SECRET. */ cron?: () => Promise; + /** Where X sends the owner back after Connect X. Trusted through its one-time state, not the session cookie (a strict cookie doesn't come back from x.com). */ + xCallback?: (url: URL) => Promise; now?: () => Date; } @@ -239,6 +241,9 @@ export function createHandler(deps: HandlerDeps): (request: Request) => Promise< switch (route) { case "GET /api/health": return json({ ok: true, version }); + case "GET /api/x/callback": + if (!deps.xCallback) throw new HttpError(404, "X is not set up on this server."); + return deps.xCallback(url); case "GET /api/cron/tick": { const secret = process.env.CRON_SECRET; if (!deps.cron || !secret || request.headers.get("authorization") !== `Bearer ${secret}`) throw new HttpError(401, "Not for you."); diff --git a/src/domains/web/server/index.ts b/src/domains/web/server/index.ts index 9f91d26..54b05e1 100644 --- a/src/domains/web/server/index.ts +++ b/src/domains/web/server/index.ts @@ -22,6 +22,7 @@ import { aiRoutes } from "../ai"; import { decideRoutes } from "../decide"; import { createApi } from "../api"; import { WebContext } from "../context"; +import { xCallback, xRoutes } from "../x"; import { buildAssets, bundle, DIST, readBundle } from "./assets"; import { clientAddress, createHandler, SetupCode } from "./handler"; import { serverSettings } from "./settings"; @@ -60,9 +61,10 @@ const handler = createHandler({ version: assets.version, auth, waitlist, - privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx)]), + privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx), ...xRoutes(ctx)]), setup, clientIp: clientAddress(settings.proxyHops, (request) => server.requestIP(request)?.address), + xCallback: xCallback(ctx), }); try { diff --git a/src/domains/web/types.ts b/src/domains/web/types.ts index d1c030a..7b61107 100644 --- a/src/domains/web/types.ts +++ b/src/domains/web/types.ts @@ -157,6 +157,33 @@ export interface PlanResponse { pace?: { min: number; max: number }; /** Why sending is not possible right now, in words. Empty when it is. */ blockers: string[]; + /** X DMs: whether the campaign's account can send through X's API, and as whom. */ + x?: XSendState; +} + +/** After Send on X: it went out, or they had replied, so nothing did. */ +export interface XSendResponse { + ok: boolean; + outcome: "SENT" | "REPLIED"; + message: string; +} + +/** Can cold send an X DM itself for this account? When not, why, in words. */ +export interface XSendState { + ready: boolean; + username?: string; + why?: string; +} + +/** Settings, X: the developer app, the callback URL to register, and each account's connection. */ +export interface XStatusResponse { + /** X_CLIENT_ID is set. */ + configured: boolean; + /** The app's database is there to keep sign-ins. */ + available: boolean; + /** Register this as the callback URL of the app at developer.x.com. */ + callbackUrl: string; + accounts: Array<{ key: string; voice: Voice; username?: string; connectedAt?: string }>; } /** What a send job reports, before it is timestamped. */ diff --git a/src/domains/web/vercel/index.ts b/src/domains/web/vercel/index.ts index 0d64ff2..236eb02 100644 --- a/src/domains/web/vercel/index.ts +++ b/src/domains/web/vercel/index.ts @@ -11,6 +11,7 @@ import { invocationDeadline } from "../background"; import { WebContext } from "../context"; import { clientAddress, createHandler, DbSetupCode } from "../server/handler"; import { serverSettings } from "../server/settings"; +import { xCallback, xRoutes } from "../x"; /** Things that only need doing now and then: expired rate-limit windows and sessions. */ async function tidy(db: Db): Promise { @@ -43,10 +44,11 @@ export async function createVercelApp(db: Db, options: { now?: () => Date } = {} version: (process.env.VERCEL_GIT_COMMIT_SHA ?? process.env.VERCEL_DEPLOYMENT_ID ?? "dev").slice(0, 12), auth, waitlist: new Waitlist(db), - privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx)]), + privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx), ...xRoutes(ctx)]), setup, // Vercel sets X-Forwarded-For to the visitor's address itself. clientIp: clientAddress(settings.proxyHops, () => undefined), + xCallback: xCallback(ctx), limiter: (limit, windowMs) => new DbRateLimiter(db, limit, windowMs), cron: async () => { await ctx.freshen(); diff --git a/src/domains/web/x/index.ts b/src/domains/web/x/index.ts new file mode 100644 index 0000000..9831c7f --- /dev/null +++ b/src/domains/web/x/index.ts @@ -0,0 +1,188 @@ +import type { Touch, TouchStore } from "../../touches"; +import { connectUrl, finishConnect, handleOf, xReplies, xUserIdOf, withXToken, type AwaitingOnX, type XClient, type XStore } from "../../xApi"; +import { HttpError, json, readBody, xHttpError, xTools, type Route } from "../api"; +import type { WebContext } from "../context"; +import type { ActionResponse, XStatusResponse } from "../types"; + +/** X keeps DMs for 30 days, so that is as far back as replies can be read. */ +const REPLY_WINDOW_MS = 30 * 24 * 60 * 60_000; +/** New handles looked up per reply check, so one check can't spend the app's lookup limit. */ +const LOOKUPS_PER_CHECK = 25; + +export const X_CALLBACK_PATH = "/api/x/callback"; + +const callbackUrl = (request: Request) => new URL(X_CALLBACK_PATH, new URL(request.url).origin).href; + +/** + * Every X touch an account made in the last 30 days, newest last: who is + * still waiting for an answer (their latest X touch is a send), and which + * messages are already logged as replies. + */ +function awaitingOf(touches: Touch[]): { sends: Map; logged: Set } { + const latest = new Map(); + const logged = new Set(); + for (const t of [...touches].sort((a, b) => a.occurredAt.localeCompare(b.occurredAt))) { + latest.set(t.prospectKey, t); + if (t.kind === "REPLIED" && t.messageId) logged.add(t.messageId); + } + return { sends: new Map([...latest].filter(([, t]) => t.kind === "SENT")), logged }; +} + +/** + * Read an account's X DMs and log a reply for everyone who answered after + * cold's last DM to them. A reply stops their sequences, like a reply to an + * email. Returns one line per reply. + */ +export async function logXReplies(input: { store: XStore; client: XClient; touches: TouchStore; account: string; voice: "personal" | "company"; now: () => Date }): Promise<{ lines: string[]; skipped: number }> { + const since = new Date(input.now().getTime() - REPLY_WINDOW_MS).toISOString(); + const { sends, logged } = awaitingOf(await input.touches.since(since, { account: input.account, channel: "X" })); + if (!sends.size) return { lines: [], skipped: 0 }; + + const byUser = new Map(); + const awaiting = new Map(); + let lookups = 0; + let skipped = 0; + await withXToken(input.store, input.client, input.account, input.now, async (token) => { + for (const touch of sends.values()) { + const handle = handleOf(touch.toAddress); + if (!handle) continue; + let userId = await input.store.cachedUserId(handle); + if (!userId) { + if (lookups >= LOOKUPS_PER_CHECK) { + skipped++; + continue; + } + lookups++; + userId = await xUserIdOf(input.store, input.client, token, handle).catch(() => undefined); + } + if (!userId) continue; + byUser.set(userId, touch); + awaiting.set(userId, { userId, sentAt: touch.occurredAt }); + } + }); + + const replies = await xReplies({ store: input.store, client: input.client, account: input.account, now: input.now, awaiting, logged }); + const lines: string[] = []; + for (const reply of replies) { + const sent = byUser.get(reply.senderId)!; + await input.touches.record({ + kind: "REPLIED", + channel: "X", + voice: input.voice === "personal" ? "PERSONAL" : "COMPANY", + account: input.account, + sequence: sent.sequence, + step: sent.step, + templateId: sent.templateId, + prospectKey: sent.prospectKey, + prospectLabel: sent.prospectLabel, + toAddress: sent.toAddress, + body: reply.text, + messageId: reply.id, + occurredAt: reply.createdAt, + links: sent.links, + }); + lines.push(`${sent.prospectLabel ?? sent.prospectKey}: ${reply.text.replace(/\s+/g, " ").slice(0, 120)}`); + } + return { lines, skipped }; +} + +/** Settings, X: connect, disconnect and read replies. All behind the login. */ +export function xRoutes(ctx: WebContext): Route[] { + return [ + [ + "GET", + /^\/api\/x$/, + async (request) => { + const { config } = await ctx.config(); + const connected = new Map((await ctx.x?.accounts())?.map((a) => [a.account, a]) ?? []); + const response: XStatusResponse = { + configured: !!ctx.xClient(), + available: !!ctx.x, + callbackUrl: callbackUrl(request), + accounts: Object.values(config?.accounts ?? {}).map((a) => ({ key: a.key, voice: a.voice, username: connected.get(a.key)?.username, connectedAt: connected.get(a.key)?.connectedAt })), + }; + return json(response); + }, + ], + [ + "POST", + /^\/api\/x\/connect$/, + async (request) => { + const { account } = await readBody<{ account: string }>(request); + const { store, client, config } = await xTools(ctx); + if (!config.accounts[account]) throw new HttpError(400, `There is no account "${account}" in cold.yaml.`); + try { + return json({ url: await connectUrl(store, client.app, account, callbackUrl(request)) }); + } catch (err) { + throw xHttpError(err); + } + }, + ], + [ + "POST", + /^\/api\/x\/disconnect$/, + async (request) => { + const { account } = await readBody<{ account: string }>(request); + if (!ctx.x) throw new HttpError(409, "There is no X connection on this server."); + const removed = await ctx.x.remove(account); + if (removed) await ctx.xClient()?.revoke(removed.tokens.accessToken); + return json({ ok: true, message: removed ? `Disconnected @${removed.username} from ${account}. cold no longer sends or reads its DMs.` : `${account} was not connected to X.` } satisfies ActionResponse); + }, + ], + [ + "POST", + /^\/api\/x\/replies$/, + async () => { + const { store, client, config } = await xTools(ctx); + const touches = await ctx.store(config); + const lines: string[] = []; + const notes: string[] = []; + for (const { account } of await store.accounts()) { + const a = config.accounts[account]; + if (!a) continue; + try { + const found = await logXReplies({ store, client, touches, account, voice: a.voice, now: () => ctx.now() }); + lines.push(...found.lines); + if (found.skipped) notes.push(`${found.skipped} people on ${account} wait for the next check (X lookups are limited).`); + } catch (err) { + notes.push(`${account}: ${(err as Error).message}`); + } + } + const message = lines.length ? `${lines.length} new ${lines.length === 1 ? "reply" : "replies"} on X, logged: ${lines.length === 1 ? "that sequence stops" : "those sequences stop"}.` : "No new replies on X."; + return json({ ok: true, message: [message, ...notes].join(" "), lines } satisfies ActionResponse); + }, + ], + ]; +} + +const escape = (text: string) => text.replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c]!); + +/** A small page after Connect X: what happened, and the way back to Settings. */ +function page(title: string, text: string, ok: boolean): Response { + const back = "/app/settings"; + const html = `${ok ? `` : ""}${escape(title)}

${escape(title)}

${escape(text)}

Back to Settings

`; + return new Response(html, { status: ok ? 200 : 400, headers: { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" } }); +} + +/** Where X sends the owner back: trade the one-time state and code for a connection. */ +export function xCallback(ctx: WebContext): (url: URL) => Promise { + return async (url) => { + const state = url.searchParams.get("state") ?? ""; + const code = url.searchParams.get("code") ?? ""; + const denied = url.searchParams.get("error"); + try { + if (denied) { + if (state) await ctx.x?.take(state); + throw new Error(denied === "access_denied" ? "You didn't allow cold on X, so nothing changed." : `X said: ${denied}`); + } + if (!ctx.x) throw new Error("This server has no database to keep the X sign-in."); + const client = ctx.xClient(); + if (!client) throw new Error("X_CLIENT_ID is not set on this server: add it in Settings and press Connect X again."); + if (!state || !code) throw new Error("X sent you back without a sign-in code: press Connect X again."); + const connection = await finishConnect(ctx.x, client, { state, code }, () => ctx.now()); + return page("X connected", `${connection.account} sends X DMs as @${connection.username} now. Taking you back to Settings.`, true); + } catch (err) { + return page("X is not connected", (err as Error).message, false); + } + }; +} diff --git a/src/domains/web/x/x.test.ts b/src/domains/web/x/x.test.ts new file mode 100644 index 0000000..4a3fff2 --- /dev/null +++ b/src/domains/web/x/x.test.ts @@ -0,0 +1,200 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { openDatabase } from "../../db"; +import type { Touch } from "../../touches"; +import type { XFetch } from "../../xApi"; +import { createApi } from "../api"; +import { WebContext } from "../context"; +import type { ActionResponse, Campaign, PlanResponse, PlanRow, XSendResponse, XStatusResponse } from "../types"; +import { xCallback, xRoutes } from "."; + +const REPO = path.resolve(import.meta.dir, "../../../.."); +const ORIGIN = "http://127.0.0.1:4100"; +let now = new Date("2026-11-02T15:00:00Z"); +let root: string; +let ctx: WebContext; +let api: (request: Request) => Promise; +let callback: (url: URL) => Promise; +const before = { id: process.env.X_CLIENT_ID, secret: process.env.X_CLIENT_SECRET }; + +type RawEvent = { id: string; text: string; sender_id: string; created_at: string; dm_conversation_id: string }; + +/** A fake X: who exists, whose inbox is closed, what was sent, and every conversation. The owner is user 999. */ +const x = { + users: new Map([ + ["clearviewtint", "101"], + ["shinedetailco", "103"], + ]), + closed: new Set(), + sent: [] as Array<{ to: string; text: string }>, + events: [] as RawEvent[], + requests: [] as string[], +}; + +const fakeX: XFetch = async (url, init = {}) => { + const u = new URL(url); + const method = init.method ?? "GET"; + x.requests.push(`${method} ${u.pathname}`); + const reply = (body: unknown, status = 200) => Response.json(body, { status }); + if (u.pathname === "/2/oauth2/token") return reply({ access_token: "at1", refresh_token: "rt1", expires_in: 7200, scope: "dm.read dm.write tweet.read users.read offline.access" }); + if (u.pathname === "/2/oauth2/revoke") return reply({ revoked: true }); + if (u.pathname === "/2/users/me") return reply({ data: { id: "999", username: "alexsells", name: "Alex" } }); + const byName = /^\/2\/users\/by\/username\/(.+)$/.exec(u.pathname); + if (byName) { + const id = x.users.get(decodeURIComponent(byName[1]!).toLowerCase()); + return id ? reply({ data: { id, username: byName[1] } }) : reply({ errors: [{ title: "Not Found Error" }] }); + } + const send = /^\/2\/dm_conversations\/with\/(\d+)\/messages$/.exec(u.pathname); + if (send && method === "POST") { + if (x.closed.has(send[1]!)) return reply({ title: "Forbidden", detail: "You are not permitted to send Direct Messages to this user.", status: 403 }, 403); + const text = (JSON.parse(String(init.body)) as { text: string }).text; + x.sent.push({ to: send[1]!, text }); + const id = `ev${x.sent.length}`; + x.events.push({ id, text, sender_id: "999", created_at: now.toISOString(), dm_conversation_id: `999-${send[1]}` }); + return reply({ data: { dm_event_id: id, dm_conversation_id: `999-${send[1]}` } }, 201); + } + const conversation = /^\/2\/dm_conversations\/with\/(\d+)\/dm_events$/.exec(u.pathname); + if (conversation) return reply({ data: x.events.filter((e) => e.dm_conversation_id === `999-${conversation[1]}`).reverse(), meta: {} }); + if (u.pathname === "/2/dm_events") return reply({ data: [...x.events].reverse(), meta: {} }); + return reply({ title: "Not Found" }, 404); +}; + +async function call(method: string, url: string, body?: unknown): Promise<{ status: number; data: T }> { + const response = await api(new Request(`${ORIGIN}${url}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) })); + return { status: response.status, data: (await response.json()) as T }; +} + +async function touchesOf(key: string): Promise { + return (await ctx.store(await ctx.requireConfig())).forProspects([key]); +} + +beforeAll(async () => { + delete process.env.COLD_CONFIG; + process.env.X_CLIENT_ID = "app1"; + process.env.X_CLIENT_SECRET = "secret1"; + root = mkdtempSync(path.join(tmpdir(), "cold-x-")); + cpSync(path.join(REPO, "messages"), path.join(root, "messages"), { recursive: true }); + cpSync(path.join(REPO, "examples"), path.join(root, "examples"), { recursive: true }); + writeFileSync(path.join(root, "cold.yaml"), readFileSync(path.join(REPO, "cold.example.yaml"), "utf8").replace("store: auto", "store: file")); + ctx = new WebContext({ root, db: await openDatabase(":memory:"), xFetch: fakeX, now: () => now }); + api = createApi(ctx, xRoutes(ctx)); + callback = xCallback(ctx); +}); + +afterAll(() => { + rmSync(root, { recursive: true, force: true }); + for (const [name, value] of [ + ["X_CLIENT_ID", before.id], + ["X_CLIENT_SECRET", before.secret], + ] as const) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +}); + +describe("X DMs through X's API, one approved message per click", () => { + let campaign: Campaign; + let dana: PlanRow; + let priya: PlanRow; + + test("Settings shows the app, the callback URL to register at developer.x.com, and the accounts to connect", async () => { + const { data } = await call("GET", "/api/x"); + expect(data).toMatchObject({ configured: true, available: true, callbackUrl: `${ORIGIN}/api/x/callback` }); + expect(data.accounts.map((a) => [a.key, a.username])).toEqual([ + ["personal", undefined], + ["company", undefined], + ]); + }); + + test("before X is connected the plan says how to send from here, and a send is refused in words with a 409, not a 401", async () => { + campaign = (await call("POST", "/api/campaigns", { name: "X openers", sequence: "dm/personal/x", account: "personal", source: "csv:examples/prospects.csv" })).data; + const { data: plan } = await call("GET", `/api/campaigns/${campaign.id}/plan`); + expect(plan.x).toEqual({ ready: false, why: "Connect X for personal in Settings to send from here." }); + dana = plan.due.find((r) => r.label.includes("Dana"))!; + priya = plan.due.find((r) => r.label.includes("Priya"))!; + expect(dana && priya).toBeTruthy(); + const refused = await call<{ error: string }>("POST", `/api/campaigns/${campaign.id}/dm/x`, { prospect: dana.key }); + expect(refused.status).toBe(409); + expect(refused.data.error).toBe("Account personal is not connected to X: connect it in Settings."); + expect(x.sent).toEqual([]); + }); + + test("Connect X sends the owner to X with a one-time state, and the callback keeps the sign-in", async () => { + const { data } = await call<{ url: string }>("POST", "/api/x/connect", { account: "personal" }); + const url = new URL(data.url); + expect(url.searchParams.get("redirect_uri")).toBe(`${ORIGIN}/api/x/callback`); + const page = await callback(new URL(`${ORIGIN}/api/x/callback?state=${url.searchParams.get("state")}&code=c1`)); + expect(page.status).toBe(200); + expect(await page.text()).toContain("personal sends X DMs as @alexsells now"); + expect((await call("GET", "/api/x")).data.accounts[0]).toMatchObject({ key: "personal", username: "alexsells" }); + const replay = await callback(new URL(`${ORIGIN}/api/x/callback?state=${url.searchParams.get("state")}&code=c1`)); + expect(replay.status).toBe(400); + expect(await replay.text()).toContain("press Connect X again"); + }); + + test("Send on X posts one DM, the approved text, and logs it with X's message id; a second click sends nothing", async () => { + const { data: plan } = await call("GET", `/api/campaigns/${campaign.id}/plan`); + expect(plan.x).toEqual({ ready: true, username: "alexsells" }); + const { status, data } = await call("POST", `/api/campaigns/${campaign.id}/dm/x`, { prospect: dana.key }); + expect(status).toBe(200); + expect(data).toEqual({ ok: true, outcome: "SENT", message: "Sent on X to @clearviewtint." }); + expect(x.sent).toEqual([{ to: "101", text: dana.text! }]); + expect(dana.text).toContain("second bay in Round Rock"); + expect(await touchesOf(dana.key)).toEqual([expect.objectContaining({ kind: "SENT", channel: "X", step: 1, messageId: "ev1", toAddress: "https://x.com/clearviewtint" })]); + const again = await call<{ error: string }>("POST", `/api/campaigns/${campaign.id}/dm/x`, { prospect: dana.key }); + expect(again.status).toBe(409); + expect(x.sent).toHaveLength(1); + }); + + test("a closed inbox is refused in words, nothing is held, and the step can be tried again", async () => { + x.closed.add("103"); + const refused = await call<{ error: string }>("POST", `/api/campaigns/${campaign.id}/dm/x`, { prospect: priya.key }); + expect(refused.status).toBe(409); + expect(refused.data.error).toContain("their messages are closed to you"); + expect(await touchesOf(priya.key)).toEqual([]); + x.closed.delete("103"); + const sent = await call("POST", `/api/campaigns/${campaign.id}/dm/x`, { prospect: priya.key }); + expect(sent.data.outcome).toBe("SENT"); + expect(x.sent.map((s) => s.to)).toEqual(["101", "103"]); + }); + + test("Check X DMs logs an answer to an earlier send, and stops that sequence", async () => { + x.events.push({ id: "r2", text: "Not for us right now, thanks", sender_id: "103", created_at: new Date(now.getTime() + 60_000).toISOString(), dm_conversation_id: "999-103" }); + const { data } = await call("POST", "/api/x/replies", {}); + expect(data.message).toBe("1 new reply on X, logged: that sequence stops."); + expect(data.lines).toEqual([expect.stringContaining("Not for us right now, thanks")]); + expect((await touchesOf(priya.key)).find((t) => t.kind === "REPLIED")).toMatchObject({ channel: "X", messageId: "r2", step: 1 }); + expect((await call("POST", "/api/x/replies", {})).data.message).toBe("No new replies on X."); + }); + + test("on the follow-up's day, someone who answered since gets nothing: their reply is logged instead", async () => { + x.events.push({ id: "r1", text: "Sounds good, send the video", sender_id: "101", created_at: "2026-11-03T09:00:00Z", dm_conversation_id: "999-101" }); + now = new Date("2026-11-09T15:00:00Z"); + const { data: plan } = await call("GET", `/api/campaigns/${campaign.id}/plan?refresh=1`); + expect(plan.due.find((r) => r.key === dana.key)?.step).toBe(2); + const { data } = await call("POST", `/api/campaigns/${campaign.id}/dm/x`, { prospect: dana.key }); + expect(data).toEqual({ ok: true, outcome: "REPLIED", message: `${dana.label} replied on X, so nothing was sent: the reply is logged and their sequence stops.` }); + expect(x.sent).toHaveLength(2); + expect((await touchesOf(dana.key)).find((t) => t.kind === "REPLIED")).toMatchObject({ messageId: "r1", body: "Sounds good, send the video", step: 1 }); + const after = (await call("GET", `/api/campaigns/${campaign.id}/plan?refresh=1`)).data; + expect(after.due.some((r) => r.key === dana.key)).toBe(false); + }); + + test("Disconnect forgets the sign-in and asks X to revoke it", async () => { + const { data } = await call("POST", "/api/x/disconnect", { account: "personal" }); + expect(data.message).toBe("Disconnected @alexsells from personal. cold no longer sends or reads its DMs."); + expect(x.requests).toContain("POST /2/oauth2/revoke"); + expect((await call("GET", "/api/x")).data.accounts[0]!.username).toBeUndefined(); + }); + + test("a sign-in the owner refused on X explains itself and keeps nothing", async () => { + const { data } = await call<{ url: string }>("POST", "/api/x/connect", { account: "company" }); + const state = new URL(data.url).searchParams.get("state"); + const page = await callback(new URL(`${ORIGIN}/api/x/callback?error=access_denied&state=${state}`)); + expect(page.status).toBe(400); + expect(await page.text()).toContain("You didn't allow cold on X, so nothing changed."); + expect((await call("GET", "/api/x")).data.accounts.every((a) => !a.username)).toBe(true); + }); +}); diff --git a/src/domains/xApi/helpers/handleOf.ts b/src/domains/xApi/helpers/handleOf.ts new file mode 100644 index 0000000..bd39b50 --- /dev/null +++ b/src/domains/xApi/helpers/handleOf.ts @@ -0,0 +1,8 @@ +/** An X handle from what a list or a touch holds: "@name", "name" or a profile link. Undefined when it isn't one. Pure. */ +export function handleOf(value: string | undefined): string | undefined { + if (!value) return undefined; + const trimmed = value.trim(); + const fromUrl = /^(?:https?:\/\/)?(?:www\.|mobile\.)?(?:x|twitter)\.com\/@?([A-Za-z0-9_]{1,15})(?:[/?#]|$)/i.exec(trimmed)?.[1]; + const handle = (fromUrl ?? trimmed).replace(/^@/, ""); + return /^[A-Za-z0-9_]{1,15}$/.test(handle) ? handle : undefined; +} diff --git a/src/domains/xApi/helpers/helpers.test.ts b/src/domains/xApi/helpers/helpers.test.ts new file mode 100644 index 0000000..3d83ddd --- /dev/null +++ b/src/domains/xApi/helpers/helpers.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, test } from "bun:test"; +import { XApiError } from "../types"; +import { authorizeUrl, base64Url, codeChallenge, handleOf, randomToken, repliesIn, xErrorText, xFailure } from "."; + +describe("PKCE and the X sign-in page", () => { + test("the S256 challenge matches RFC 7636's example", async () => { + expect(await codeChallenge("dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk")).toBe("E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM"); + }); + + test("random tokens are URL-safe and 43 characters for 32 bytes", () => { + const token = randomToken((n) => new Uint8Array(n).fill(251)); + expect(token).toHaveLength(43); + expect(token).toMatch(/^[A-Za-z0-9_-]+$/); + expect(base64Url(new Uint8Array([251, 255]))).toBe("-_8"); + }); + + test("the authorize URL asks for DMs, profiles and a refresh token, with the challenge", () => { + const url = new URL(authorizeUrl({ clientId: "app1", redirectUri: "http://127.0.0.1:4100/api/x/callback", state: "s1", challenge: "c1" })); + expect(url.origin + url.pathname).toBe("https://x.com/i/oauth2/authorize"); + expect(Object.fromEntries(url.searchParams)).toEqual({ + response_type: "code", + client_id: "app1", + redirect_uri: "http://127.0.0.1:4100/api/x/callback", + scope: "dm.read dm.write tweet.read users.read offline.access", + state: "s1", + code_challenge: "c1", + code_challenge_method: "S256", + }); + }); +}); + +describe("what X's errors mean", () => { + const kind = (status: number, body: unknown, doing: Parameters[0]["doing"], reset?: string) => xFailure({ status, body, doing, resetHeader: reset }); + + test("a closed inbox, a plan without DMs, and a dead token are told apart", () => { + expect(kind(403, { detail: "You are not permitted to send Direct Messages to this user." }, "send").kind).toBe("cannotDm"); + expect(kind(403, { detail: "You currently have access to a subset of X API V2 endpoints and limited v1.1 endpoints only." }, "send").kind).toBe("access"); + expect(kind(403, { errors: [{ message: "Your client app is not configured with the appropriate oauth1 app permissions for this endpoint." }] }, "send").kind).toBe("access"); + expect(kind(401, { title: "Unauthorized" }, "send").kind).toBe("auth"); + expect(kind(404, {}, "lookup").kind).toBe("notFound"); + expect(kind(503, "Service Unavailable", "read").kind).toBe("other"); + }); + + test("the token endpoint: a bad app key, an expired code, a refused renewal", () => { + expect(kind(401, { error: "invalid_client" }, "connect").kind).toBe("access"); + expect(kind(400, { error: "invalid_request", error_description: "Value passed for the authorization code was invalid." }, "connect").kind).toBe("auth"); + expect(kind(400, { error: "invalid_grant" }, "refresh").message).toContain("connect X again"); + }); + + test("a rate limit says when to try again", () => { + const err = kind(429, { title: "Too Many Requests" }, "send", "1793632800"); + expect(err).toBeInstanceOf(XApiError); + expect(err.kind).toBe("rateLimited"); + expect(err.retryAt?.toISOString()).toBe("2026-11-02T15:20:00.000Z"); + expect(err.message).toContain("after 15:20 UTC"); + expect(kind(429, {}, "send").message).toContain("in 15 minutes"); + }); + + test("X's own words are quoted, whatever shape they come in", () => { + expect(xErrorText({ detail: "d" })).toBe("d"); + expect(xErrorText({ errors: [{ message: "m" }] })).toBe("m"); + expect(xErrorText({ error: "invalid_grant", error_description: "gone" })).toBe("gone"); + expect(xErrorText(null)).toBe(""); + expect(kind(403, { detail: "You cannot send messages to this user." }, "send").message).toContain('X said: "You cannot send messages to this user."'); + }); +}); + +describe("X handles", () => { + test("a handle from @name, name or a profile link", () => { + expect(handleOf("@ClearviewTint")).toBe("ClearviewTint"); + expect(handleOf("clearviewtint")).toBe("clearviewtint"); + expect(handleOf("https://x.com/clearviewtint")).toBe("clearviewtint"); + expect(handleOf("https://twitter.com/clearviewtint/status/123")).toBe("clearviewtint"); + expect(handleOf("x.com/@clearviewtint?ref=1")).toBe("clearviewtint"); + }); + + test("anything else is not a handle", () => { + expect(handleOf(undefined)).toBeUndefined(); + expect(handleOf("not a handle")).toBeUndefined(); + expect(handleOf("waytoolongforahandle1")).toBeUndefined(); + expect(handleOf("https://linkedin.com/in/dana")).toBeUndefined(); + }); +}); + +describe("replies on X", () => { + const event = (id: string, senderId: string, createdAt: string) => ({ id, senderId, createdAt, text: `message ${id}` }); + const awaiting = new Map([ + ["dana", { userId: "dana", sentAt: "2026-11-02T10:00:00Z" }], + ["marcus", { userId: "marcus", sentAt: "2026-11-02T12:00:00Z" }], + ]); + + test("the first answer from each person after cold's last DM, oldest first", () => { + const events = [ + event("e5", "dana", "2026-11-02T11:30:00Z"), + event("e4", "dana", "2026-11-02T11:00:00Z"), + event("e3", "me", "2026-11-02T10:30:00Z"), + event("e2", "marcus", "2026-11-02T11:59:00Z"), + event("e1", "stranger", "2026-11-02T11:00:00Z"), + ]; + expect(repliesIn(events, { me: "me", awaiting, logged: new Set() }).map((e) => e.id)).toEqual(["e4"]); + }); + + test("a message already logged is not a new reply", () => { + const events = [event("e4", "dana", "2026-11-02T11:00:00Z"), event("e6", "marcus", "2026-11-02T13:00:00Z")]; + expect(repliesIn(events, { me: "me", awaiting, logged: new Set(["e4"]) }).map((e) => e.id)).toEqual(["e6"]); + }); +}); diff --git a/src/domains/xApi/helpers/index.ts b/src/domains/xApi/helpers/index.ts new file mode 100644 index 0000000..cb58663 --- /dev/null +++ b/src/domains/xApi/helpers/index.ts @@ -0,0 +1,4 @@ +export * from "./handleOf"; +export * from "./pkce"; +export * from "./repliesIn"; +export * from "./xFailure"; diff --git a/src/domains/xApi/helpers/pkce.ts b/src/domains/xApi/helpers/pkce.ts new file mode 100644 index 0000000..32de4ce --- /dev/null +++ b/src/domains/xApi/helpers/pkce.ts @@ -0,0 +1,31 @@ +import { X_AUTHORIZE_URL, X_SCOPES } from "../types"; + +/** Bytes as base64url without padding (RFC 4648, section 5). */ +export function base64Url(bytes: Uint8Array): string { + return Buffer.from(bytes).toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +/** A random URL-safe value for the PKCE code verifier or the state. 32 bytes give 43 characters. */ +export function randomToken(random: (bytes: number) => Uint8Array = (n) => crypto.getRandomValues(new Uint8Array(n)), bytes = 32): string { + return base64Url(random(bytes)); +} + +/** The S256 code challenge of a verifier (RFC 7636). */ +export async function codeChallenge(verifier: string): Promise { + return base64Url(new Uint8Array(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(verifier)))); +} + +/** The page on X where the owner approves cold for their account. Pure. */ +export function authorizeUrl(input: { clientId: string; redirectUri: string; state: string; challenge: string; scopes?: readonly string[] }): string { + const url = new URL(X_AUTHORIZE_URL); + url.search = new URLSearchParams({ + response_type: "code", + client_id: input.clientId, + redirect_uri: input.redirectUri, + scope: (input.scopes ?? X_SCOPES).join(" "), + state: input.state, + code_challenge: input.challenge, + code_challenge_method: "S256", + }).toString(); + return url.href; +} diff --git a/src/domains/xApi/helpers/repliesIn.ts b/src/domains/xApi/helpers/repliesIn.ts new file mode 100644 index 0000000..a4fa2e5 --- /dev/null +++ b/src/domains/xApi/helpers/repliesIn.ts @@ -0,0 +1,23 @@ +import type { XDmEvent } from "../types"; + +/** Someone cold has DMed on X and not heard back from since. */ +export interface AwaitingOnX { + /** Their X user id. */ + userId: string; + /** When cold's last DM to them went out (ISO). */ + sentAt: string; +} + +/** + * The first answer from each person cold is waiting on: a message they sent + * after cold's last one, that isn't logged yet. Oldest first. Pure. + */ +export function repliesIn(events: readonly XDmEvent[], input: { me: string; awaiting: ReadonlyMap; logged: ReadonlySet }): XDmEvent[] { + const first = new Map(); + for (const event of [...events].sort((a, b) => a.createdAt.localeCompare(b.createdAt))) { + if (event.senderId === input.me || input.logged.has(event.id) || first.has(event.senderId)) continue; + const waiting = input.awaiting.get(event.senderId); + if (waiting && Date.parse(event.createdAt) > Date.parse(waiting.sentAt)) first.set(event.senderId, event); + } + return [...first.values()]; +} diff --git a/src/domains/xApi/helpers/xFailure.ts b/src/domains/xApi/helpers/xFailure.ts new file mode 100644 index 0000000..059cb1a --- /dev/null +++ b/src/domains/xApi/helpers/xFailure.ts @@ -0,0 +1,48 @@ +import { XApiError } from "../types"; + +/** What cold was doing when X said no, for the words of the error. */ +export type XDoing = "connect" | "refresh" | "send" | "lookup" | "read"; + +/** What X said went wrong: v2 problem details, OAuth errors, or the older errors list. Pure. */ +export function xErrorText(body: unknown): string { + if (!body || typeof body !== "object") return typeof body === "string" ? body.slice(0, 200).trim() : ""; + const b = body as { detail?: unknown; title?: unknown; error_description?: unknown; error?: unknown; errors?: Array<{ message?: unknown; detail?: unknown }> }; + const first = b.errors?.[0]; + const text = b.detail ?? b.error_description ?? first?.detail ?? first?.message ?? (typeof b.error === "string" ? b.error : undefined) ?? b.title; + return typeof text === "string" ? text.trim() : ""; +} + +/** The app's plan or setup, not the person: X's words for these. */ +const APP_PROBLEM = /subset of|not enrolled|client-not-enrolled|attached to a project|app permissions|access level|not configured|upgrade|453/i; + +/** + * X's answer as an XApiError that says what to do next. `resetHeader` is + * x-rate-limit-reset (epoch seconds) on a 429. Pure. + */ +export function xFailure(input: { status: number; body: unknown; doing: XDoing; resetHeader?: string | null }): XApiError { + const { status, doing } = input; + const said = xErrorText(input.body); + const quoted = said ? ` X said: "${said}"` : ""; + + if (status === 429) { + const reset = Number(input.resetHeader); + const retryAt = input.resetHeader && Number.isFinite(reset) && reset > 0 ? new Date(reset * 1000) : undefined; + const when = retryAt ? `after ${retryAt.toISOString().slice(11, 16)} UTC` : "in 15 minutes"; + return new XApiError("rateLimited", `X's limit for this is used up for now: try again ${when}.`, status, retryAt); + } + if (doing === "connect" || doing === "refresh") { + if (/invalid_client|unauthorized_client/i.test(said) || status === 401) + return new XApiError("access", `X does not accept the app's keys: check X_CLIENT_ID and X_CLIENT_SECRET in Settings against your app at developer.x.com.${quoted}`, status); + if (doing === "refresh") return new XApiError("auth", `X would not renew the sign-in: connect X again in Settings.${quoted}`, status); + return new XApiError("auth", `X refused the sign-in (it may have taken too long): press Connect X again.${quoted}`, status); + } + if (status === 401) return new XApiError("auth", `X no longer accepts this sign-in: connect X again in Settings.${quoted}`, status); + if (status === 403 && APP_PROBLEM.test(said)) + return new XApiError("access", `Your X developer app can't do this yet: its plan or permissions don't include it. Check the app at developer.x.com.${quoted}`, status); + if (status === 403 && doing === "send") + return new XApiError("cannotDm", `X won't deliver a DM from you to them: their messages are closed to you. Skip them, or reach them on another channel.${quoted}`, status); + if (status === 403) return new XApiError("access", `X refused this for your account or app.${quoted}`, status); + if (status === 404) return new XApiError("notFound", `X has no such account.${quoted}`, status); + if (status >= 500) return new XApiError("other", `X had a problem on its side (HTTP ${status}): try again in a minute.`, status); + return new XApiError("other", `X answered HTTP ${status}.${quoted}`, status); +} diff --git a/src/domains/xApi/index.ts b/src/domains/xApi/index.ts new file mode 100644 index 0000000..1335854 --- /dev/null +++ b/src/domains/xApi/index.ts @@ -0,0 +1,296 @@ +import type { Db } from "../db"; +import { seal, unseal } from "../storage"; +import { authorizeUrl, codeChallenge, randomToken, repliesIn, xFailure, type AwaitingOnX, type XDoing } from "./helpers"; +import { X_API, X_REFRESH_EARLY_MS, X_REVOKE_URL, X_STATE_TTL_MS, X_TOKEN_URL, XApiError, type XApp, type XConnection, type XDmEvent, type XFetch, type XTokens, type XUser } from "./types"; + +export * from "./types"; +export * from "./helpers"; + +/** The owner's X developer app, from X_CLIENT_ID (and X_CLIENT_SECRET for a confidential client). Undefined until it is set. */ +export function xAppFromEnv(env: Record = process.env): XApp | undefined { + const clientId = env.X_CLIENT_ID?.trim(); + if (!clientId) return undefined; + return { clientId, clientSecret: env.X_CLIENT_SECRET?.trim() || undefined }; +} + +async function bodyOf(response: Response): Promise { + const text = await response.text(); + if (!text) return {}; + try { + return JSON.parse(text); + } catch { + return text; + } +} + +type RawEvent = { id: string; text?: string; sender_id?: string; created_at?: string; dm_conversation_id?: string }; + +/** X's API as cold uses it. Each call takes the access token it acts with. */ +export class XClient { + constructor( + readonly app: XApp, + private readonly fetchImpl: XFetch = (input, init) => fetch(input, init), + private readonly now: () => Date = () => new Date(), + ) {} + + private async token(params: Record, doing: XDoing): Promise { + const headers: Record = { "content-type": "application/x-www-form-urlencoded" }; + if (this.app.clientSecret) headers.authorization = `Basic ${Buffer.from(`${encodeURIComponent(this.app.clientId)}:${encodeURIComponent(this.app.clientSecret)}`).toString("base64")}`; + const response = await this.fetchImpl(X_TOKEN_URL, { method: "POST", headers, body: new URLSearchParams({ ...params, client_id: this.app.clientId }).toString() }); + const body = await bodyOf(response); + if (!response.ok) throw xFailure({ status: response.status, body, doing, resetHeader: response.headers.get("x-rate-limit-reset") }); + const t = body as { access_token?: string; refresh_token?: string; expires_in?: number; scope?: string }; + if (!t.access_token) throw new XApiError("other", "X answered without an access token: press Connect X again.", response.status); + return { accessToken: t.access_token, refreshToken: t.refresh_token, expiresAt: new Date(this.now().getTime() + (t.expires_in ?? 7200) * 1000).toISOString(), scope: t.scope ?? "" }; + } + + /** The code X sent back, traded for tokens (PKCE: with the verifier its challenge came from). */ + exchangeCode(code: string, verifier: string, redirectUri: string): Promise { + return this.token({ grant_type: "authorization_code", code, redirect_uri: redirectUri, code_verifier: verifier }, "connect"); + } + + refresh(refreshToken: string): Promise { + return this.token({ grant_type: "refresh_token", refresh_token: refreshToken }, "refresh"); + } + + /** Tell X to forget a token. Best effort: disconnecting goes ahead either way. */ + async revoke(token: string): Promise { + const headers: Record = { "content-type": "application/x-www-form-urlencoded" }; + if (this.app.clientSecret) headers.authorization = `Basic ${Buffer.from(`${encodeURIComponent(this.app.clientId)}:${encodeURIComponent(this.app.clientSecret)}`).toString("base64")}`; + await this.fetchImpl(X_REVOKE_URL, { method: "POST", headers, body: new URLSearchParams({ token, token_type_hint: "access_token", client_id: this.app.clientId }).toString() }).catch(() => undefined); + } + + private async call(accessToken: string, path: string, doing: XDoing, init: RequestInit = {}): Promise { + const headers: Record = { authorization: `Bearer ${accessToken}` }; + if (init.body) headers["content-type"] = "application/json"; + const response = await this.fetchImpl(`${X_API}${path}`, { ...init, headers }); + const body = await bodyOf(response); + if (!response.ok) throw xFailure({ status: response.status, body, doing, resetHeader: response.headers.get("x-rate-limit-reset") }); + return body as T; + } + + /** The account a token belongs to. */ + async me(accessToken: string): Promise { + const { data } = await this.call<{ data?: XUser }>(accessToken, "/2/users/me", "lookup"); + if (!data?.id) throw new XApiError("other", "X did not say which account signed in: press Connect X again.", 200); + return { id: data.id, username: data.username, name: data.name }; + } + + /** A person's X user id, from their handle. */ + async userByHandle(accessToken: string, handle: string): Promise { + const body = await this.call<{ data?: XUser }>(accessToken, `/2/users/by/username/${encodeURIComponent(handle)}`, "lookup"); + if (!body.data?.id) throw new XApiError("notFound", `X has no account @${handle} (renamed, suspended or deleted): fix the handle in the list, or skip them.`, 404); + return { id: body.data.id, username: body.data.username, name: body.data.name }; + } + + /** Send one DM. X answers 201 with the message's event id. */ + async sendDm(accessToken: string, participantId: string, text: string): Promise<{ eventId: string; conversationId?: string }> { + const body = await this.call<{ data?: { dm_event_id?: string; dm_conversation_id?: string } }>(accessToken, `/2/dm_conversations/with/${encodeURIComponent(participantId)}/messages`, "send", { + method: "POST", + body: JSON.stringify({ text }), + }); + return { eventId: body.data?.dm_event_id ?? "", conversationId: body.data?.dm_conversation_id }; + } + + /** DM messages, newest first, up to `limit` (X keeps 30 days). With a participant, only that conversation. */ + async dmEvents(accessToken: string, options: { participantId?: string; limit?: number } = {}): Promise { + const path = options.participantId ? `/2/dm_conversations/with/${encodeURIComponent(options.participantId)}/dm_events` : "/2/dm_events"; + const limit = options.limit ?? 300; + const out: XDmEvent[] = []; + let next: string | undefined; + do { + const params = new URLSearchParams({ event_types: "MessageCreate", "dm_event.fields": "id,text,sender_id,created_at,dm_conversation_id", max_results: "100" }); + if (next) params.set("pagination_token", next); + const body = await this.call<{ data?: RawEvent[]; meta?: { next_token?: string } }>(accessToken, `${path}?${params}`, "read"); + for (const e of body.data ?? []) if (e.sender_id && e.created_at) out.push({ id: e.id, text: e.text ?? "", senderId: e.sender_id, createdAt: e.created_at, conversationId: e.dm_conversation_id }); + next = body.meta?.next_token; + } while (next && out.length < limit); + return out; + } +} + +/** One step of one sequence to one person, sent through X once. */ +export interface XSendKey { + account: string; + prospectKey: string; + sequence: string; + step: number; +} + +/** + * X connections, Connect X attempts, handle lookups and sends, in the app's + * database. Tokens are sealed with COLD_SECRET_KEY when it is set. Without + * it they are kept as they are, which is only allowed on your own computer + * (where .env.local holds your other keys the same way); a hosted server + * refuses. + */ +export class XStore { + constructor( + private readonly db: Db, + private readonly options: { key?: Buffer; requireKey: boolean; now: () => Date; random?: (bytes: number) => Uint8Array }, + ) {} + + private pack(account: string, tokens: XTokens): string { + const text = JSON.stringify(tokens); + if (this.options.key) return seal(this.options.key, `x:${account}`, text); + if (this.options.requireKey) throw new XApiError("access", "Keeping an X sign-in on this server needs COLD_SECRET_KEY in its environment, so it is stored encrypted. Add it, redeploy, and connect X again.", 409); + return `plain:${text}`; + } + + private unpack(account: string, stored: string): XTokens { + if (stored.startsWith("plain:")) return JSON.parse(stored.slice("plain:".length)) as XTokens; + if (!this.options.key) throw new XApiError("auth", "The X sign-in is stored encrypted and COLD_SECRET_KEY is not set: set it as it was, or connect X again.", 401); + return JSON.parse(unseal(this.options.key, `x:${account}`, stored)) as XTokens; + } + + /** Start Connect X for an account: a state and a code verifier, kept for ten minutes. */ + async begin(account: string, redirectUri: string): Promise<{ state: string; verifier: string }> { + const state = randomToken(this.options.random); + const verifier = randomToken(this.options.random); + const now = this.options.now(); + await this.db.batch([ + ["DELETE FROM x_oauth_states WHERE created_at < ?", new Date(now.getTime() - X_STATE_TTL_MS).toISOString()], + ["INSERT INTO x_oauth_states (state, account, verifier, redirect_uri, created_at) VALUES (?, ?, ?, ?, ?)", state, account, verifier, redirectUri, now.toISOString()], + ]); + return { state, verifier }; + } + + /** The Connect X attempt a state belongs to, once: reading it deletes it. Undefined when unknown or over ten minutes old. */ + async take(state: string): Promise<{ account: string; verifier: string; redirectUri: string } | undefined> { + const row = await this.db.get<{ account: string; verifier: string; redirect_uri: string; created_at: string }>("SELECT account, verifier, redirect_uri, created_at FROM x_oauth_states WHERE state = ?", state); + if (!row) return undefined; + const { changes } = await this.db.run("DELETE FROM x_oauth_states WHERE state = ?", state); + if (!changes || this.options.now().getTime() - Date.parse(row.created_at) > X_STATE_TTL_MS) return undefined; + return { account: row.account, verifier: row.verifier, redirectUri: row.redirect_uri }; + } + + async save(connection: XConnection): Promise { + await this.db.run( + "INSERT INTO x_connections (account, user_id, username, tokens, connected_at, updated_at) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (account) DO UPDATE SET user_id = excluded.user_id, username = excluded.username, tokens = excluded.tokens, connected_at = excluded.connected_at, updated_at = excluded.updated_at", + connection.account, + connection.userId, + connection.username, + this.pack(connection.account, connection.tokens), + connection.connectedAt, + this.options.now().toISOString(), + ); + } + + async saveTokens(account: string, tokens: XTokens): Promise { + await this.db.run("UPDATE x_connections SET tokens = ?, updated_at = ? WHERE account = ?", this.pack(account, tokens), this.options.now().toISOString(), account); + } + + async get(account: string): Promise { + const row = await this.db.get<{ account: string; user_id: string; username: string; tokens: string; connected_at: string }>("SELECT account, user_id, username, tokens, connected_at FROM x_connections WHERE account = ?", account); + if (!row) return undefined; + return { account: row.account, userId: row.user_id, username: row.username, tokens: this.unpack(row.account, row.tokens), connectedAt: row.connected_at }; + } + + /** Who each account is connected as, without the tokens. */ + async accounts(): Promise> { + return (await this.db.all<{ account: string; username: string; connected_at: string }>("SELECT account, username, connected_at FROM x_connections ORDER BY account")).map((r) => ({ account: r.account, username: r.username, connectedAt: r.connected_at })); + } + + async remove(account: string): Promise { + const connection = await this.get(account).catch(() => undefined); + await this.db.run("DELETE FROM x_connections WHERE account = ?", account); + return connection; + } + + async cachedUserId(handle: string): Promise { + return (await this.db.get<{ user_id: string }>("SELECT user_id FROM x_users WHERE handle = ?", handle.toLowerCase()))?.user_id; + } + + async rememberUser(handle: string, userId: string): Promise { + await this.db.run( + "INSERT INTO x_users (handle, user_id, updated_at) VALUES (?, ?, ?) ON CONFLICT (handle) DO UPDATE SET user_id = excluded.user_id, updated_at = excluded.updated_at", + handle.toLowerCase(), + userId, + this.options.now().toISOString(), + ); + } + + /** + * Claim one send so it goes out once, even after two clicks or on two + * servers. Undefined when it is claimed; otherwise what already holds it. + */ + async claim(key: XSendKey): Promise<{ status: string; eventId?: string; at: string } | undefined> { + const args = [key.account, key.prospectKey, key.sequence, key.step] as const; + const { changes } = await this.db.run( + "INSERT INTO x_sends (account, prospect_key, sequence, step, status, created_at) VALUES (?, ?, ?, ?, 'SENDING', ?) ON CONFLICT DO NOTHING", + ...args, + this.options.now().toISOString(), + ); + if (changes) return undefined; + const row = await this.db.get<{ status: string; event_id: string | null; created_at: string }>("SELECT status, event_id, created_at FROM x_sends WHERE account = ? AND prospect_key = ? AND sequence = ? AND step = ?", ...args); + return { status: row?.status ?? "SENDING", eventId: row?.event_id ?? undefined, at: row?.created_at ?? "" }; + } + + async markSent(key: XSendKey, eventId: string): Promise { + await this.db.run("UPDATE x_sends SET status = 'SENT', event_id = ? WHERE account = ? AND prospect_key = ? AND sequence = ? AND step = ?", eventId, key.account, key.prospectKey, key.sequence, key.step); + } + + /** X refused, so nothing went out: the step can be tried again. */ + async release(key: XSendKey): Promise { + await this.db.run("DELETE FROM x_sends WHERE account = ? AND prospect_key = ? AND sequence = ? AND step = ? AND status = 'SENDING'", key.account, key.prospectKey, key.sequence, key.step); + } +} + +/** A good access token for an account, renewed when it is about to run out (or when `force`). */ +export async function accessTokenFor(store: XStore, client: XClient, account: string, now: () => Date, force = false): Promise<{ token: string; connection: XConnection }> { + const connection = await store.get(account); + if (!connection) throw new XApiError("auth", `Account ${account} is not connected to X: connect it in Settings.`, 401); + const expiring = Date.parse(connection.tokens.expiresAt) - now().getTime() < X_REFRESH_EARLY_MS; + if (!force && !expiring) return { token: connection.tokens.accessToken, connection }; + if (!connection.tokens.refreshToken) throw new XApiError("auth", "The X sign-in has run out and cannot be renewed: connect X again in Settings.", 401); + const fresh = await client.refresh(connection.tokens.refreshToken); + const tokens = { ...fresh, refreshToken: fresh.refreshToken ?? connection.tokens.refreshToken }; + await store.saveTokens(account, tokens); + return { token: tokens.accessToken, connection: { ...connection, tokens } }; +} + +/** Run a call with the account's token. When X says the token is no good, renew it once and try again (a refused call sent nothing). */ +export async function withXToken(store: XStore, client: XClient, account: string, now: () => Date, call: (token: string, connection: XConnection) => Promise): Promise { + const first = await accessTokenFor(store, client, account, now); + try { + return await call(first.token, first.connection); + } catch (err) { + if (!(err instanceof XApiError) || err.kind !== "auth" || !first.connection.tokens.refreshToken) throw err; + const again = await accessTokenFor(store, client, account, now, true); + return call(again.token, again.connection); + } +} + +/** A person's X user id: remembered from before, or looked up once. */ +export async function xUserIdOf(store: XStore, client: XClient, token: string, handle: string): Promise { + const known = await store.cachedUserId(handle); + if (known) return known; + const user = await client.userByHandle(token, handle); + await store.rememberUser(handle, user.id); + return user.id; +} + +/** Start Connect X: the page on X to send the owner to. */ +export async function connectUrl(store: XStore, app: XApp, account: string, redirectUri: string): Promise { + const { state, verifier } = await store.begin(account, redirectUri); + return authorizeUrl({ clientId: app.clientId, redirectUri, state, challenge: await codeChallenge(verifier) }); +} + +/** Finish Connect X: trade the code for tokens and keep them with the account they belong to. */ +export async function finishConnect(store: XStore, client: XClient, input: { state: string; code: string }, now: () => Date): Promise { + const attempt = await store.take(input.state); + if (!attempt) throw new XApiError("auth", "That Connect X link is used or more than ten minutes old: press Connect X again.", 400); + const tokens = await client.exchangeCode(input.code, attempt.verifier, attempt.redirectUri); + const me = await client.me(tokens.accessToken); + const connection: XConnection = { account: attempt.account, userId: me.id, username: me.username, tokens, connectedAt: now().toISOString() }; + await store.save(connection); + return connection; +} + +/** The first answer on X from each person an account is waiting on. */ +export async function xReplies(input: { store: XStore; client: XClient; account: string; now: () => Date; awaiting: ReadonlyMap; logged: ReadonlySet; participantId?: string }): Promise { + return withXToken(input.store, input.client, input.account, input.now, async (token, connection) => { + const events = await input.client.dmEvents(token, { participantId: input.participantId, limit: input.participantId ? 100 : 300 }); + return repliesIn(events, { me: connection.userId, awaiting: input.awaiting, logged: input.logged }); + }); +} diff --git a/src/domains/xApi/types.ts b/src/domains/xApi/types.ts new file mode 100644 index 0000000..6015d45 --- /dev/null +++ b/src/domains/xApi/types.ts @@ -0,0 +1,83 @@ +/** + * X's official API (v2), the only way cold sends a DM: from the owner's own + * account, one approved message per click, through an X developer app the + * owner registers. Every endpoint cold calls is here, so a change on X's side + * is one edit. + */ +export const X_API = "https://api.x.com"; +export const X_AUTHORIZE_URL = "https://x.com/i/oauth2/authorize"; +export const X_TOKEN_URL = `${X_API}/2/oauth2/token`; +export const X_REVOKE_URL = `${X_API}/2/oauth2/revoke`; + +/** What cold asks X for: read and write DMs, read profiles, and stay connected (a refresh token). */ +export const X_SCOPES = ["dm.read", "dm.write", "tweet.read", "users.read", "offline.access"] as const; + +/** How long a Connect X link stays usable. */ +export const X_STATE_TTL_MS = 10 * 60_000; + +/** Refresh an access token this long before it runs out. */ +export const X_REFRESH_EARLY_MS = 60_000; + +/** The owner's X developer app (developer.x.com): OAuth 2.0, type "Web App" (confidential) or "Native App" (public). */ +export interface XApp { + clientId: string; + /** Only for a confidential client. */ + clientSecret?: string; +} + +export interface XTokens { + accessToken: string; + refreshToken?: string; + /** ISO time the access token stops working. */ + expiresAt: string; + scope: string; +} + +/** A cold.yaml account connected to an X account. */ +export interface XConnection { + account: string; + userId: string; + username: string; + tokens: XTokens; + connectedAt: string; +} + +export interface XUser { + id: string; + username: string; + name?: string; +} + +export interface XDmEvent { + id: string; + text: string; + senderId: string; + createdAt: string; + conversationId?: string; +} + +/** + * Why a call to X failed, in the terms cold acts on: + * auth the token is no good: refresh it, or reconnect X + * cannotDm X will not deliver a DM from you to this person + * rateLimited wait until retryAt + * notFound no such user + * access the developer app's plan or settings don't allow this + * other anything else; the message says what X said + */ +export type XFailureKind = "auth" | "cannotDm" | "rateLimited" | "notFound" | "access" | "other"; + +export class XApiError extends Error { + constructor( + readonly kind: XFailureKind, + message: string, + readonly status: number, + readonly retryAt?: Date, + ) { + super(message); + this.name = "XApiError"; + } +} + +/** fetch, or a fake one in tests. */ +export type XFetch = (input: string, init?: RequestInit) => Promise; diff --git a/src/domains/xApi/xApi.test.ts b/src/domains/xApi/xApi.test.ts new file mode 100644 index 0000000..049ca1a --- /dev/null +++ b/src/domains/xApi/xApi.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, test } from "bun:test"; +import { openDatabase } from "../db"; +import { accessTokenFor, connectUrl, finishConnect, withXToken, XApiError, XClient, XStore, xUserIdOf, type XFetch } from "."; + +const NOW = new Date("2026-11-02T15:00:00Z"); +const now = () => NOW; +const KEY = Buffer.alloc(32, 7); + +type Seen = { url: string; method: string; headers: Record; body: string }; + +/** A fake X: answers by a handler and keeps every request. */ +function fakeX(answer: (seen: Seen) => { status?: number; body?: unknown; headers?: Record }): { fetch: XFetch; seen: Seen[] } { + const seen: Seen[] = []; + const fetch: XFetch = async (url, init = {}) => { + const request: Seen = { url, method: init.method ?? "GET", headers: Object.fromEntries(new Headers(init.headers).entries()), body: String(init.body ?? "") }; + seen.push(request); + const { status = 200, body = {}, headers = {} } = answer(request); + return new Response(typeof body === "string" ? body : JSON.stringify(body), { status, headers }); + }; + return { fetch, seen }; +} + +const tokenAnswer = { body: { token_type: "bearer", access_token: "at1", refresh_token: "rt1", expires_in: 7200, scope: "dm.read dm.write tweet.read users.read offline.access" } }; + +async function store(options: { key?: Buffer; requireKey?: boolean } = { key: KEY }) { + return new XStore(await openDatabase(":memory:"), { key: options.key, requireKey: options.requireKey ?? false, now }); +} + +describe("the X client", () => { + test("the code is traded for tokens with the verifier, the app's Basic auth and the same redirect", async () => { + const x = fakeX(() => tokenAnswer); + const tokens = await new XClient({ clientId: "app1", clientSecret: "secret1" }, x.fetch, now).exchangeCode("code1", "verifier1", "http://127.0.0.1:4100/api/x/callback"); + expect(tokens).toEqual({ accessToken: "at1", refreshToken: "rt1", expiresAt: "2026-11-02T17:00:00.000Z", scope: "dm.read dm.write tweet.read users.read offline.access" }); + const [request] = x.seen; + expect(request!.url).toBe("https://api.x.com/2/oauth2/token"); + expect(request!.headers.authorization).toBe(`Basic ${Buffer.from("app1:secret1").toString("base64")}`); + expect(Object.fromEntries(new URLSearchParams(request!.body))).toEqual({ grant_type: "authorization_code", code: "code1", redirect_uri: "http://127.0.0.1:4100/api/x/callback", code_verifier: "verifier1", client_id: "app1" }); + }); + + test("a public client sends no secret", async () => { + const x = fakeX(() => tokenAnswer); + await new XClient({ clientId: "app1" }, x.fetch, now).refresh("rt0"); + expect(x.seen[0]!.headers.authorization).toBeUndefined(); + expect(Object.fromEntries(new URLSearchParams(x.seen[0]!.body))).toEqual({ grant_type: "refresh_token", refresh_token: "rt0", client_id: "app1" }); + }); + + test("a DM is one POST to the person's conversation, with the bearer token", async () => { + const x = fakeX(() => ({ status: 201, body: { data: { dm_conversation_id: "123-456", dm_event_id: "ev1" } } })); + const sent = await new XClient({ clientId: "app1" }, x.fetch, now).sendDm("at1", "456", "Hi Dana"); + expect(sent).toEqual({ eventId: "ev1", conversationId: "123-456" }); + expect(x.seen[0]).toMatchObject({ url: "https://api.x.com/2/dm_conversations/with/456/messages", method: "POST", body: JSON.stringify({ text: "Hi Dana" }) }); + expect(x.seen[0]!.headers.authorization).toBe("Bearer at1"); + expect(x.seen[0]!.headers["content-type"]).toBe("application/json"); + }); + + test("a closed inbox is a cannotDm error that says what to do", async () => { + const x = fakeX(() => ({ status: 403, body: { detail: "You are not permitted to send Direct Messages to this user.", status: 403 } })); + const err = await new XClient({ clientId: "app1" }, x.fetch, now).sendDm("at1", "456", "Hi").catch((e: unknown) => e); + expect(err).toBeInstanceOf(XApiError); + expect((err as XApiError).kind).toBe("cannotDm"); + expect((err as XApiError).message).toContain("Skip them, or reach them on another channel."); + }); + + test("a handle X doesn't know is notFound, even when X answers 200 with errors", async () => { + const x = fakeX(() => ({ body: { errors: [{ title: "Not Found Error", detail: "Could not find user with username: [gone]." }] } })); + const err = await new XClient({ clientId: "app1" }, x.fetch, now).userByHandle("at1", "gone").catch((e: unknown) => e); + expect((err as XApiError).kind).toBe("notFound"); + expect((err as XApiError).message).toContain("@gone"); + }); + + test("DM events follow X's pages and stop at the limit", async () => { + const page = (n: number, next?: string) => ({ body: { data: [{ id: `e${n}`, text: `t${n}`, sender_id: "u1", created_at: `2026-11-0${n}T00:00:00Z`, dm_conversation_id: "c" }], meta: next ? { next_token: next } : {} } }); + const x = fakeX((s) => (s.url.includes("pagination_token=p2") ? page(2, "p3") : s.url.includes("pagination_token=p3") ? page(3) : page(1, "p2"))); + const events = await new XClient({ clientId: "app1" }, x.fetch, now).dmEvents("at1"); + expect(events.map((e) => e.id)).toEqual(["e1", "e2", "e3"]); + expect(new URL(x.seen[0]!.url).searchParams.get("event_types")).toBe("MessageCreate"); + const one = await new XClient({ clientId: "app1" }, x.fetch, now).dmEvents("at1", { participantId: "456", limit: 1 }); + expect(one.map((e) => e.id)).toEqual(["e1"]); + expect(x.seen.at(-1)!.url).toStartWith("https://api.x.com/2/dm_conversations/with/456/dm_events?"); + }); +}); + +describe("the X store", () => { + const connection = { account: "personal", userId: "123", username: "matt", connectedAt: NOW.toISOString(), tokens: { accessToken: "at1", refreshToken: "rt1", expiresAt: "2026-11-02T17:00:00.000Z", scope: "dm.write" } }; + + test("tokens are sealed with COLD_SECRET_KEY and read back", async () => { + const db = await openDatabase(":memory:"); + const sealed = new XStore(db, { key: KEY, requireKey: true, now }); + await sealed.save(connection); + const raw = await db.get<{ tokens: string }>("SELECT tokens FROM x_connections"); + expect(raw!.tokens).not.toContain("at1"); + expect(raw!.tokens).toStartWith("v1:"); + expect(await sealed.get("personal")).toEqual(connection); + expect(await sealed.accounts()).toEqual([{ account: "personal", username: "matt", connectedAt: NOW.toISOString() }]); + }); + + test("without a key they are kept as they are on your own computer, and refused on a hosted server", async () => { + const local = await store({ requireKey: false }); + await local.save(connection); + expect((await local.get("personal"))?.tokens.accessToken).toBe("at1"); + const hosted = await store({ requireKey: true }); + const err = await hosted.save(connection).catch((e: unknown) => e); + expect((err as Error).message).toContain("COLD_SECRET_KEY"); + }); + + test("a Connect X attempt works once, and not after ten minutes", async () => { + const db = await openDatabase(":memory:"); + let at = NOW; + const s = new XStore(db, { key: KEY, requireKey: false, now: () => at }); + const { state, verifier } = await s.begin("personal", "http://127.0.0.1:4100/api/x/callback"); + expect(await s.take(state)).toEqual({ account: "personal", verifier, redirectUri: "http://127.0.0.1:4100/api/x/callback" }); + expect(await s.take(state)).toBeUndefined(); + const late = await s.begin("personal", "http://127.0.0.1:4100/api/x/callback"); + at = new Date(NOW.getTime() + 11 * 60_000); + expect(await s.take(late.state)).toBeUndefined(); + }); + + test("a step is claimed once: a second click finds it taken, a refusal frees it, a send keeps it", async () => { + const s = await store(); + const key = { account: "personal", prospectKey: "csv:p01", sequence: "dm/personal/x-opener", step: 1 }; + expect(await s.claim(key)).toBeUndefined(); + expect(await s.claim(key)).toMatchObject({ status: "SENDING" }); + await s.release(key); + expect(await s.claim(key)).toBeUndefined(); + await s.markSent(key, "ev1"); + await s.release(key); + expect(await s.claim(key)).toEqual({ status: "SENT", eventId: "ev1", at: NOW.toISOString() }); + }); + + test("handles are remembered without their case", async () => { + const s = await store(); + const x = fakeX(() => ({ body: { data: { id: "456", username: "ClearviewTint" } } })); + const client = new XClient({ clientId: "app1" }, x.fetch, now); + expect(await xUserIdOf(s, client, "at1", "ClearviewTint")).toBe("456"); + expect(await xUserIdOf(s, client, "at1", "clearviewtint")).toBe("456"); + expect(x.seen).toHaveLength(1); + }); +}); + +describe("staying signed in", () => { + test("a token about to run out is renewed, and X's new refresh token is kept", async () => { + const s = await store(); + await s.save({ account: "personal", userId: "123", username: "matt", connectedAt: NOW.toISOString(), tokens: { accessToken: "old", refreshToken: "rt-old", expiresAt: "2026-11-02T15:00:30.000Z", scope: "" } }); + const x = fakeX(() => ({ body: { access_token: "new", refresh_token: "rt-new", expires_in: 7200 } })); + const { token } = await accessTokenFor(s, new XClient({ clientId: "app1" }, x.fetch, now), "personal", now); + expect(token).toBe("new"); + expect((await s.get("personal"))?.tokens).toMatchObject({ accessToken: "new", refreshToken: "rt-new" }); + }); + + test("a call X refuses as unauthorized is tried once more with a renewed token", async () => { + const s = await store(); + await s.save({ account: "personal", userId: "123", username: "matt", connectedAt: NOW.toISOString(), tokens: { accessToken: "revoked", refreshToken: "rt1", expiresAt: "2026-11-02T17:00:00.000Z", scope: "" } }); + const x = fakeX(() => ({ body: { access_token: "renewed", expires_in: 7200 } })); + const client = new XClient({ clientId: "app1" }, x.fetch, now); + const used: string[] = []; + const result = await withXToken(s, client, "personal", now, async (token) => { + used.push(token); + if (token === "revoked") throw new XApiError("auth", "no", 401); + return "ok"; + }); + expect(result).toBe("ok"); + expect(used).toEqual(["revoked", "renewed"]); + expect((await s.get("personal"))?.tokens.refreshToken).toBe("rt1"); + }); + + test("an account that isn't connected says to connect it", async () => { + const err = await accessTokenFor(await store(), new XClient({ clientId: "app1" }), "company", now).catch((e: unknown) => e); + expect((err as Error).message).toBe("Account company is not connected to X: connect it in Settings."); + }); +}); + +describe("connecting X", () => { + test("the sign-in page carries a state, and the callback trades it once for a connection", async () => { + const s = await store(); + const x = fakeX((seen) => (seen.url.endsWith("/2/oauth2/token") ? tokenAnswer : { body: { data: { id: "123", username: "matt", name: "Matt" } } })); + const client = new XClient({ clientId: "app1", clientSecret: "secret1" }, x.fetch, now); + const url = new URL(await connectUrl(s, client.app, "personal", "http://127.0.0.1:4100/api/x/callback")); + const state = url.searchParams.get("state")!; + const connection = await finishConnect(s, client, { state, code: "code1" }, now); + expect(connection).toMatchObject({ account: "personal", userId: "123", username: "matt" }); + expect((await s.get("personal"))?.tokens.accessToken).toBe("at1"); + const again = await finishConnect(s, client, { state, code: "code1" }, now).catch((e: unknown) => e); + expect((again as Error).message).toContain("press Connect X again"); + }); +}); diff --git a/src/web/pages/campaign.tsx b/src/web/pages/campaign.tsx index c68bc38..8f8a53f 100644 --- a/src/web/pages/campaign.tsx +++ b/src/web/pages/campaign.tsx @@ -1,8 +1,8 @@ import { Ban, CalendarClock, Sparkles, CheckCircle2, ChevronDown, ChevronRight, Clock, Copy, Download, ExternalLink, Hourglass, Mail, MessageSquareReply, Pencil, RefreshCw, Send, SkipForward, Square, Trash2, UserX, XCircle } from "lucide-react"; import { Fragment, useCallback, useEffect, useRef, useState } from "react"; import type { AiDraftRecord } from "../../domains/ai/store"; -import type { Campaign, JobEvent, JobResponse, PlanResponse, PlanRow } from "../../domains/web/types"; -import { navigate } from "../app/router"; +import type { Campaign, JobEvent, JobResponse, PlanResponse, PlanRow, XSendResponse } from "../../domains/web/types"; +import { Link, navigate } from "../app/router"; import { useStatus } from "../app/status"; import { api, ApiError, useApi } from "../lib/api"; import { channelLabel, day, plural, sequenceName, sourceLabel, voiceLabel } from "../lib/format"; @@ -399,6 +399,23 @@ function DmQueue({ plan, campaign, drafts, onChange }: { plan: PlanResponse; cam const skip = useCallback(() => current && setSkipped((s) => new Set(s).add(current.key)), [current]); + // X DMs can go out from here, through X's API as the connected account: one click, one message. + const canSendOnX = plan.platform === "x" && !!plan.x?.ready; + const sendOnX = useCallback(async () => { + if (!current || busy || blocked || !canSendOnX) return; + setBusy("X"); + try { + const result = await api(`/api/campaigns/${campaign.id}/dm/x`, { method: "POST", body: { prospect: current.key } }); + setDone((s) => new Set(s).add(current.key)); + toast.success(result.message); + onChange(); + } catch (err) { + toast.error((err as Error).message); + } finally { + setBusy(null); + } + }, [current, busy, blocked, canSendOnX, campaign.id, onChange, toast]); + useEffect(() => { const onKey = (e: KeyboardEvent) => { if (e.metaKey || e.ctrlKey || e.altKey || (e.target as HTMLElement).closest("input, textarea, select")) return; @@ -406,13 +423,14 @@ function DmQueue({ plan, campaign, drafts, onChange }: { plan: PlanResponse; cam if (key === "c") copy(); else if (key === "p") open(); else if (key === "k") skip(); + else if (!blocked && key === "x") sendOnX(); else if (!blocked && key === "s") record("SENT"); else if (!blocked && key === "r") record("REPLIED"); else if (!blocked && key === "o") record("OPTED_OUT"); }; window.addEventListener("keydown", onKey); return () => window.removeEventListener("keydown", onKey); - }, [copy, open, skip, record, blocked]); + }, [copy, open, skip, record, sendOnX, blocked]); async function downloadChecklist() { const response = await fetch(`/api/campaigns/${campaign.id}/checklist`, { headers: { "x-cold": "1" } }); @@ -464,6 +482,11 @@ function DmQueue({ plan, campaign, drafts, onChange }: { plan: PlanResponse; cam d.prospectKey === current.key && d.step === (current.step ?? 1))} onChanged={onChange} />
+ {canSendOnX && ( + + )} @@ -471,6 +494,11 @@ function DmQueue({ plan, campaign, drafts, onChange }: { plan: PlanResponse; cam Open profile P
+ {plan.platform === "x" && !plan.x?.ready && plan.x?.why && ( +
+ {plan.x.why} Settings +
+ )} {blocked && Create cold.yaml in Settings to log DMs: right now they would be signed and logged as the example sender.}
+ Answers to DMs cold sent as {onX.map((a) => `@${a.username}`).join(", ")}: each one stops that sequence. +
+ )} + {xResult?.lines?.length ?
{xResult.lines.join("\n")}
: null} diff --git a/src/web/pages/settings.tsx b/src/web/pages/settings.tsx index 0886921..933bb56 100644 --- a/src/web/pages/settings.tsx +++ b/src/web/pages/settings.tsx @@ -1,6 +1,6 @@ -import { CheckCircle2, Database, FileText, KeyRound, LogOut, Mailbox, MonitorSmartphone, PlugZap, Save, ShieldCheck, Sparkles, UserRound, Wand2, XCircle } from "lucide-react"; +import { AtSign, CheckCircle2, Copy, Database, FileText, KeyRound, LogOut, Mailbox, MonitorSmartphone, PlugZap, Save, ShieldCheck, Sparkles, Unplug, UserRound, Wand2, XCircle } from "lucide-react"; import { useEffect, useState, type ReactNode } from "react"; -import type { ActionResponse, AiStatusResponse, SessionsResponse, SettingsResponse } from "../../domains/web/types"; +import type { ActionResponse, AiStatusResponse, SessionsResponse, SettingsResponse, XStatusResponse } from "../../domains/web/types"; import { usePwa, install } from "../app/pwa"; import { useSession } from "../app/session"; import { useStatus } from "../app/status"; @@ -132,12 +132,108 @@ export function SettingsPage() { + + {settings.loading && !settings.data ? : settings.error ? : settings.data && (status.reload(), settings.reload())} />} {settings.data && (status.reload(), settings.reload())} />} ); } +/** + * X DMs through X's API: the owner's own developer app, and each account + * connected to the X account it sends as. One DM goes out per click, from + * the campaign's queue; nothing is sent on its own. + */ +function XCard() { + const x = useApi("/api/x"); + const toast = useToast(); + const [busy, setBusy] = useState(null); + + async function connect(account: string) { + setBusy(account); + try { + const { url } = await api<{ url: string }>("/api/x/connect", { method: "POST", body: { account } }); + window.location.href = url; + } catch (err) { + toast.error((err as Error).message); + setBusy(null); + } + } + + async function disconnect(account: string) { + setBusy(account); + try { + toast.success((await api("/api/x/disconnect", { method: "POST", body: { account } })).message); + x.reload(); + } catch (err) { + toast.error((err as Error).message); + } finally { + setBusy(null); + } + } + + async function copyCallback() { + if (!x.data) return; + try { + await navigator.clipboard.writeText(x.data.callbackUrl); + toast.success("Copied the callback URL."); + } catch { + toast.error("The browser blocked the clipboard: select the URL and copy it."); + } + } + + const d = x.data; + return ( + + {x.loading && !d ? ( + + ) : x.error ? ( + + ) : ( + d && ( +
+

+ Send X DMs from a campaign's queue with one click each, from your own account, through X's API. You approve every one; nothing goes out on its own. LinkedIn, Instagram and Facebook stay copy and open: they have no way for an app to start a conversation. +

+ {d.configured ? "Your X developer app is set" : "Add X_CLIENT_ID (and X_CLIENT_SECRET) below, from your app at developer.x.com"} + {!d.available && This server has no database to keep an X sign-in.} +
+ Register this as the app's callback URL (User authentication settings, type Web App): +
+ {d.callbackUrl} + +
+
+ {d.accounts.map((a) => ( +
+ + {a.key} {a.voice} + + {a.username ? ( + + Sends as @{a.username} + + + ) : ( + + )} +
+ ))} +

Your app's X API plan has to include Direct Messages. cold asks X for: read and send DMs, read profiles, stay connected.

+
+ ) + )} +
+ ); +} + function Secrets({ settings, onSaved }: { settings: SettingsResponse; onSaved: () => void }) { const [values, setValues] = useState>({}); const [saving, setSaving] = useState(false); From 2be3a10b81927c4953d55e1120899fdf0f31d8ed Mon Sep 17 00:00:00 2001 From: operatoruplift Date: Sun, 1 Nov 2026 15:42:05 +0000 Subject: [PATCH 2/6] Leads from listening: OpenMagpie matches come in as prospects - POST /api/listen/magpie takes OpenMagpie's webhook body (instant or digest), proven by Authorization: Bearer , size and item limits checked. Each match is kept once by its key, however often OpenMagpie delivers it. - Each watch is a prospect source, listen: (and listen:all): the trigger line is the post itself, quoted short, with its link; a post on X carries its author's handle, so it can be answered with an X DM; Reddit usernames are never used as first names. - Save as a list (Settings, Listening) writes a watch's matches to one of your own lists, prospects-listen-.csv, to fill in and research; saving again keeps your edits and adds only new matches. - Routes other services call are one table (publicRoutes) the guard lets through by name; the prospect sources the app accepts are defined once; the CSV writer is shared. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuUQ7RbYzBy4coULGCu --- .env.example | 5 + CHANGELOG.md | 9 ++ README.md | 6 + src/domains/db/db.test.ts | 6 +- src/domains/db/index.ts | 12 ++ src/domains/listen/helpers/index.ts | 3 + src/domains/listen/helpers/listenCsv.ts | 27 +++++ src/domains/listen/helpers/listenProspect.ts | 61 ++++++++++ src/domains/listen/helpers/readMagpie.ts | 65 ++++++++++ src/domains/listen/index.ts | 54 +++++++++ src/domains/listen/listen.test.ts | 96 +++++++++++++++ src/domains/listen/types.ts | 49 ++++++++ src/domains/prospects/helpers/csvText.ts | 10 ++ src/domains/prospects/helpers/index.ts | 1 + src/domains/prospects/index.ts | 5 +- src/domains/prospects/types.ts | 2 +- src/domains/web/ai/index.ts | 11 +- src/domains/web/ai/shared.ts | 6 +- src/domains/web/api/index.ts | 8 +- src/domains/web/context/index.ts | 9 ++ src/domains/web/guard/index.ts | 12 +- src/domains/web/listen/index.ts | 82 +++++++++++++ src/domains/web/listen/listen.test.ts | 120 +++++++++++++++++++ src/domains/web/open/index.ts | 20 ++++ src/domains/web/server/handler.test.ts | 9 ++ src/domains/web/server/handler.ts | 14 ++- src/domains/web/server/index.ts | 8 +- src/domains/web/types.ts | 13 +- src/domains/web/vercel/index.ts | 8 +- src/web/pages/settings.tsx | 80 ++++++++++++- 30 files changed, 772 insertions(+), 39 deletions(-) create mode 100644 src/domains/listen/helpers/index.ts create mode 100644 src/domains/listen/helpers/listenCsv.ts create mode 100644 src/domains/listen/helpers/listenProspect.ts create mode 100644 src/domains/listen/helpers/readMagpie.ts create mode 100644 src/domains/listen/index.ts create mode 100644 src/domains/listen/listen.test.ts create mode 100644 src/domains/listen/types.ts create mode 100644 src/domains/prospects/helpers/csvText.ts create mode 100644 src/domains/web/listen/index.ts create mode 100644 src/domains/web/listen/listen.test.ts create mode 100644 src/domains/web/open/index.ts diff --git a/.env.example b/.env.example index 20be7a7..abf82d7 100644 --- a/.env.example +++ b/.env.example @@ -50,6 +50,11 @@ SUBQ_API_KEY= X_CLIENT_ID= X_CLIENT_SECRET= +# Listening (optional): OpenMagpie's watches deliver the posts they match to +# /api/listen/magpie (Settings, Listening shows the URL). Give the +# webhook action the header Authorization: Bearer . +COLD_LISTEN_SECRET= + # Hosting (docs/deploy.md). These belong in the server's real environment, # not in this file, since .env.local lives inside COLD_HOME. # COLD_PUBLIC_URL=https://cold.example.com diff --git a/CHANGELOG.md b/CHANGELOG.md index 04e63b1..1dcb5aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +- **Leads from listening.** OpenMagpie's webhook action delivers the posts + its watches match to `/api/listen/magpie` (with `Authorization: Bearer` + and `COLD_LISTEN_SECRET`). Each match is kept once, however often it is + delivered, and each watch is a prospect source, `listen:`: the + trigger line is the post itself, quoted, and a post on X carries its + author's handle for an X DM. **Save as a list** in Settings writes a + watch's matches to one of your own lists to fill in and research; saving + again only adds what is new. Prospect sources are defined once, so + research and campaigns accept the same ones. - **X DMs go out from COLD.** Add your own X developer app in Settings (`X_CLIENT_ID`, `X_CLIENT_SECRET`) and press **Connect X** for an account: a campaign's X queue then has **Send on X**, one approved DM per click, diff --git a/README.md b/README.md index eea23d1..206dbcc 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,12 @@ call scripts and the Do Not Contact list. web search, and every claim comes with the page it was read on. Drafts, call prep and reply answers wait for your approval; nothing goes out on its own. +- **Leads from listening.** Point a watch in + [OpenMagpie](https://github.com/obris-dev/openmagpie) (the self-hostable + listener behind ListeningKit) at COLD's webhook, and every post where + someone asks for what you sell comes in as a prospect whose trigger line is + the post itself, with its link. A post on X can be answered with an X DM; + save a watch as a list to fill in the rest (company, email) and research it. - **The messages are the product.** 45 messages across cold email, LinkedIn, X, Instagram and Facebook DMs, and call scripts, plus 11 objection answers. Every channel is covered twice, once as you and once as the company. Each message diff --git a/src/domains/db/db.test.ts b/src/domains/db/db.test.ts index e4a6e9b..732f94a 100644 --- a/src/domains/db/db.test.ts +++ b/src/domains/db/db.test.ts @@ -46,10 +46,10 @@ describe("the database", () => { test("a new database gets every table, and migrating again changes nothing", async () => { const db = await openDatabase(":memory:"); const tables = (await db.all<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")).map((t) => t.name); - expect(tables).toEqual(["ai_drafts", "ai_runs", "decisions", "files", "meta", "owner", "rate_limits", "score_job_items", "score_jobs", "secrets", "send_events", "send_jobs", "sessions", "style_assignments", "touches", "waitlist", "x_connections", "x_oauth_states", "x_sends", "x_users"]); - expect((await db.get<{ value: string }>("SELECT value FROM meta WHERE key = 'schema_version'"))?.value).toBe("6"); + expect(tables).toEqual(["ai_drafts", "ai_runs", "decisions", "files", "listen_items", "meta", "owner", "rate_limits", "score_job_items", "score_jobs", "secrets", "send_events", "send_jobs", "sessions", "style_assignments", "touches", "waitlist", "x_connections", "x_oauth_states", "x_sends", "x_users"]); + expect((await db.get<{ value: string }>("SELECT value FROM meta WHERE key = 'schema_version'"))?.value).toBe("7"); await migrate(db); - expect((await db.get<{ value: string }>("SELECT value FROM meta WHERE key = 'schema_version'"))?.value).toBe("6"); + expect((await db.get<{ value: string }>("SELECT value FROM meta WHERE key = 'schema_version'"))?.value).toBe("7"); }); test("a file from before the version moved into meta keeps its data and gains the new tables", async () => { diff --git a/src/domains/db/index.ts b/src/domains/db/index.ts index 1b4a521..52220bb 100644 --- a/src/domains/db/index.ts +++ b/src/domains/db/index.ts @@ -278,6 +278,18 @@ const MIGRATIONS: string[] = [ created_at TEXT NOT NULL, PRIMARY KEY (account, prospect_key, sequence, step) );`, + // Listening: the matches OpenMagpie's webhook delivers, each key once. + `CREATE TABLE listen_items ( + key TEXT PRIMARY KEY, + watch_id TEXT NOT NULL, + watch_name TEXT NOT NULL, + source_kind TEXT NOT NULL, + occurred_at TEXT NOT NULL, + received_at TEXT NOT NULL, + item TEXT NOT NULL + ); + CREATE INDEX listen_items_watch ON listen_items (watch_id, occurred_at); + CREATE INDEX listen_items_newest ON listen_items (occurred_at);`, ]; const statements = (script: string): Statement[] => diff --git a/src/domains/listen/helpers/index.ts b/src/domains/listen/helpers/index.ts new file mode 100644 index 0000000..77d7a3f --- /dev/null +++ b/src/domains/listen/helpers/index.ts @@ -0,0 +1,3 @@ +export * from "./listenCsv"; +export * from "./listenProspect"; +export * from "./readMagpie"; diff --git a/src/domains/listen/helpers/listenCsv.ts b/src/domains/listen/helpers/listenCsv.ts new file mode 100644 index 0000000..a94243a --- /dev/null +++ b/src/domains/listen/helpers/listenCsv.ts @@ -0,0 +1,27 @@ +import { csvText, parseCsv } from "../../prospects"; +import type { ListenItem } from "../types"; +import { listenProspect } from "./listenProspect"; + +/** A saved list's columns: what you fill in (company, email) first, then what the match gave. */ +export const LISTEN_COLUMNS = ["id", "first_name", "full_name", "company", "email", "website", "x_handle", "trigger", "post_url", "post_title", "post_text", "posted_at", "listen_source", "listen_watch"]; + +/** + * Matches as a prospect list you can edit and research like any of your own. + * Saving again keeps every row and column already there (your edits, your own + * columns) and adds only the matches it doesn't have, by id (OpenMagpie's + * key). Pure. + */ +export function listenCsv(items: readonly ListenItem[], existing?: string): { text: string; added: number; rows: number } { + const [head = [], ...rest] = existing ? parseCsv(existing) : []; + const headers = [...head, ...LISTEN_COLUMNS.filter((c) => !head.includes(c))]; + const rows = rest.map((r) => Object.fromEntries(head.map((h, i) => [h, r[i]])) as Record); + const have = new Set(rows.map((r) => r.id).filter(Boolean)); + let added = 0; + for (const item of items) { + if (have.has(item.key)) continue; + have.add(item.key); + rows.push({ id: item.key, ...listenProspect(item).variables }); + added++; + } + return { text: csvText(headers, rows), added, rows: rows.length }; +} diff --git a/src/domains/listen/helpers/listenProspect.ts b/src/domains/listen/helpers/listenProspect.ts new file mode 100644 index 0000000..125a25d --- /dev/null +++ b/src/domains/listen/helpers/listenProspect.ts @@ -0,0 +1,61 @@ +import { finishProspect, type Prospect } from "../../prospects"; +import type { ListenItem } from "../types"; + +/** A post's words cut at a word, for quoting in one line. Pure. */ +export function quoteOf(words: string, max = 90): string { + const flat = words.replace(/\s+/g, " ").trim().replace(/^["“]+|["”]+$/g, ""); + if (flat.length <= max) return flat; + const cut = flat.slice(0, max); + const space = cut.lastIndexOf(" "); + return `${(space > max / 2 ? cut.slice(0, space) : cut).replace(/[\s,;:.!?-]+$/, "")}…`; +} + +const isReddit = (item: ListenItem) => !!item.subreddit || /reddit/i.test(item.sourceKind); +const isX = (item: ListenItem) => !!item.handle || /twitter|^x_/i.test(item.sourceKind); + +/** + * The {{trigger}} line for a match: why you're writing, in one true sentence + * built from the post itself. Pure. + */ +export function triggerOf(item: ListenItem): string { + const what = quoteOf(item.title || item.content); + if (/github/i.test(item.sourceKind)) return `Came across ${item.title || "your project"} on GitHub.`; + if (!what) return `Saw your post${item.sourceLabel ? ` in ${item.sourceLabel}` : ""}.`; + if (isReddit(item)) return `Saw your post in r/${item.subreddit ?? item.sourceLabel.replace(/^r\//, "")}: "${what}".`; + if (isX(item)) return `Saw your post on X: "${what}".`; + if (/hacker ?news|hackernews|^hn/i.test(item.sourceKind)) return `Saw your post on Hacker News: "${what}".`; + if (/rss/i.test(item.sourceKind)) return `Read your post "${what}".`; + return `Saw your post: "${what}".`; +} + +/** Who wrote it, for lists: "u/name on r/sub", "@handle on X", "name (source)". Pure. */ +export function listenLabel(item: ListenItem): string { + if (isX(item) && item.handle) return `${item.author && item.author !== item.handle ? `${item.author} ` : ""}@${item.handle} on X`; + if (isReddit(item)) return `${item.author ? `u/${item.author}` : "Someone"} on r/${item.subreddit ?? item.sourceLabel.replace(/^r\//, "")}`; + return `${item.author ?? "Someone"}${item.sourceLabel ? ` (${item.sourceLabel})` : ""}`; +} + +/** + * A match as a prospect: the post is the trigger line, the author's X handle + * (when it came from X) makes them reachable by an X DM, and the post's link + * and words are variables for research and drafts. Reddit usernames are not + * names, so they are never used as first_name. Pure. + */ +export function listenProspect(item: ListenItem): Prospect { + const variables: Record = Object.fromEntries( + Object.entries({ + full_name: isX(item) && item.author && item.author !== item.handle ? item.author : undefined, + x_handle: item.handle, + website: item.website, + trigger: triggerOf(item), + post_url: item.url, + post_title: item.title, + post_text: quoteOf(item.content, 400), + posted_at: item.occurredAt, + listen_source: item.sourceLabel, + listen_watch: item.watchName, + }).filter((e): e is [string, string] => !!e[1]), + ); + const prospect = finishProspect({ source: "listen", variables, doNotContact: false, links: {}, sourceId: item.key }); + return { ...prospect, label: listenLabel(item) }; +} diff --git a/src/domains/listen/helpers/readMagpie.ts b/src/domains/listen/helpers/readMagpie.ts new file mode 100644 index 0000000..592a987 --- /dev/null +++ b/src/domains/listen/helpers/readMagpie.ts @@ -0,0 +1,65 @@ +import { MAX_LISTEN_ITEMS, type ListenItem, type MagpiePayload } from "../types"; + +type Rec = Record; +const text = (v: unknown, max: number) => (typeof v === "string" ? v.replace(/\u0000/g, "").trim().slice(0, max) : ""); +const isRec = (v: unknown): v is Rec => !!v && typeof v === "object" && !Array.isArray(v); + +/** A site's origin, for an author's own site. Undefined for the big platforms, which are no one's own site. */ +function siteOf(url: string): string | undefined { + try { + const u = new URL(url); + if (!/^https?:$/.test(u.protocol) || /(^|\.)(reddit|x|twitter|github|ycombinator|youtube|medium|substack)\.com$/i.test(u.hostname)) return undefined; + return u.origin; + } catch { + return undefined; + } +} + +/** Where the post is: Reddit's permalink (made absolute), else the item's own URL. */ +function postUrl(item: Rec): string { + const permalink = text(item.permalink, 600); + if (permalink) return permalink.startsWith("/") ? `https://www.reddit.com${permalink}` : permalink; + return text(item.url, 600) || text(item.external_url, 600); +} + +/** One OpenMagpie item as a match. Pure. */ +function toItem(entry: MagpiePayload["items"][number], watch: MagpiePayload["watch"], receivedAt: string): ListenItem | undefined { + const key = text(entry.key, 300); + if (!key) return undefined; + const item = isRec(entry.item) ? entry.item : {}; + const sourceKind = text(entry.source?.kind, 64) || text(item.source, 64) || "unknown"; + const url = postUrl(item); + const handle = /twitter|^x_/i.test(sourceKind) ? text(item.handle, 30).replace(/^@/, "") || undefined : undefined; + const homepage = text(item.homepage, 300); + const occurred = text(item.occurred_at, 40); + return { + key, + watchId: text(watch.id, 64), + watchName: text(watch.name, 200) || "Listening", + sourceKind, + sourceLabel: text(entry.source?.label, 200), + title: text(item.title, 300), + content: text(item.content, 2000), + url, + author: text(item.author, 120) || text(item.owner, 120) || undefined, + handle, + subreddit: text(item.subreddit, 80).replace(/^r\//i, "") || undefined, + website: (homepage && siteOf(homepage)) || (/rss/i.test(sourceKind) ? siteOf(url) : undefined), + occurredAt: occurred && !Number.isNaN(Date.parse(occurred)) ? new Date(occurred).toISOString() : undefined, + receivedAt, + }; +} + +/** + * OpenMagpie's webhook body, read and checked: the matches in it, or what is + * wrong with it. Unknown fields are ignored; a body with too many items is + * refused rather than cut. Pure. + */ +export function readMagpie(body: unknown, now: Date): { items: ListenItem[]; error?: string } { + if (!isRec(body) || !isRec(body.watch) || !Array.isArray(body.items)) return { items: [], error: "Not an OpenMagpie webhook body: it needs watch and items." }; + if (body.items.length > MAX_LISTEN_ITEMS) return { items: [], error: `Too many items in one delivery (${body.items.length}, at most ${MAX_LISTEN_ITEMS}): use a shorter digest interval.` }; + const watch = body.watch as MagpiePayload["watch"]; + const receivedAt = now.toISOString(); + const items = (body.items as MagpiePayload["items"]).flatMap((entry) => (isRec(entry) ? toItem(entry, watch, receivedAt) ?? [] : [])); + return { items }; +} diff --git a/src/domains/listen/index.ts b/src/domains/listen/index.ts new file mode 100644 index 0000000..4b1f843 --- /dev/null +++ b/src/domains/listen/index.ts @@ -0,0 +1,54 @@ +import type { Db } from "../db"; +import type { ListenItem } from "./types"; + +export * from "./types"; +export * from "./helpers"; + +/** A watch that has sent matches: which, how many, the latest. */ +export interface ListenWatch { + id: string; + name: string; + count: number; + latest: string; +} + +/** The matches OpenMagpie sent, in the app's database. Each key is kept once. */ +export class ListenStore { + constructor(private readonly db: Db) {} + + /** Keep new matches; one already kept (a redelivery) is left as it is. */ + async receive(items: ListenItem[]): Promise<{ added: number }> { + let added = 0; + for (const item of items) { + const { changes } = await this.db.run( + "INSERT INTO listen_items (key, watch_id, watch_name, source_kind, occurred_at, received_at, item) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (key) DO NOTHING", + item.key, + item.watchId, + item.watchName, + item.sourceKind, + item.occurredAt ?? item.receivedAt, + item.receivedAt, + JSON.stringify(item), + ); + added += changes; + } + return { added }; + } + + /** Matches, newest post first: every watch's, or one's. */ + async items(options: { watch?: string; limit?: number } = {}): Promise { + const limit = Math.max(1, Math.min(options.limit ?? 500, 5000)); + const rows = options.watch + ? await this.db.all<{ item: string }>("SELECT item FROM listen_items WHERE watch_id = ? ORDER BY occurred_at DESC LIMIT ?", options.watch, limit) + : await this.db.all<{ item: string }>("SELECT item FROM listen_items ORDER BY occurred_at DESC LIMIT ?", limit); + return rows.map((r) => JSON.parse(r.item) as ListenItem); + } + + async watches(): Promise { + return ( + await this.db.all<{ id: string; name: string; count: number; latest: string }>( + "SELECT watch_id AS id, MAX(watch_name) AS name, COUNT(*) AS count, MAX(received_at) AS latest FROM listen_items GROUP BY watch_id ORDER BY latest DESC", + ) + ).map((w) => ({ id: w.id, name: w.name, count: Number(w.count), latest: w.latest })); + } +} diff --git a/src/domains/listen/listen.test.ts b/src/domains/listen/listen.test.ts new file mode 100644 index 0000000..ac73475 --- /dev/null +++ b/src/domains/listen/listen.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, test } from "bun:test"; +import { openDatabase } from "../db"; +import { ListenStore, listenProspect, quoteOf, readMagpie, triggerOf, type ListenItem } from "."; + +const NOW = new Date("2026-11-02T15:00:00Z"); + +/** A digest as OpenMagpie's webhook action sends it: one Reddit post, one X post, one GitHub repo, one blog post. */ +const digest = { + watch: { id: "01JWATCH", name: "Needs booking software" }, + action_id: "01JACTION", + delivery: "digest", + window: { since: "2026-11-02T14:00:00Z", until: "2026-11-02T15:00:00Z" }, + items: [ + { + key: "reddit_subreddit:t3_abc", + source: { label: "r/smallbusiness", kind: "reddit_subreddit" }, + item: { external_id: "t3_abc", occurred_at: "2026-11-02T14:10:00Z", title: "Any booking tool for a two-person detailing shop? We keep missing calls", content: "We tried three apps.", url: "https://example.com/linked", permalink: "/r/smallbusiness/comments/abc/any_booking_tool/", author: "shinyhands", subreddit: "smallbusiness" }, + }, + { + key: "twitter_search:1850", + source: { label: "X search: booking app", kind: "twitter_search" }, + item: { external_id: "1850", occurred_at: "2026-11-02T14:20:00Z", title: "", content: "Is there a booking page that turns Google reviews into appointments? Asking for my tint shop", url: "https://x.com/clearviewtint/status/1850", author: "Dana Reyes", handle: "@ClearviewTint" }, + }, + { + key: "github_search:42", + source: { label: "GitHub: booking", kind: "github_search" }, + item: { external_id: "42", occurred_at: "2026-11-02T13:00:00Z", title: "acme/bookit", content: "A tiny booking widget", url: "https://github.com/acme/bookit", owner: "acme", homepage: "https://bookit.acme.example/docs" }, + }, + { + key: "rss:entry-9", + source: { label: "Detailing Weekly", kind: "rss" }, + item: { external_id: "entry-9", occurred_at: "2026-11-01T09:00:00Z", title: "Why we stopped taking bookings by phone", content: "", url: "https://detailingweekly.example/posts/phones", author: "Priya" }, + }, + ], +}; + +describe("OpenMagpie's webhook body", () => { + test("each connector's item becomes a match with its link, author and handle", () => { + const { items, error } = readMagpie(digest, NOW); + expect(error).toBeUndefined(); + expect(items.map((i) => [i.key, i.url, i.author, i.handle, i.website])).toEqual([ + ["reddit_subreddit:t3_abc", "https://www.reddit.com/r/smallbusiness/comments/abc/any_booking_tool/", "shinyhands", undefined, undefined], + ["twitter_search:1850", "https://x.com/clearviewtint/status/1850", "Dana Reyes", "ClearviewTint", undefined], + ["github_search:42", "https://github.com/acme/bookit", "acme", undefined, "https://bookit.acme.example"], + ["rss:entry-9", "https://detailingweekly.example/posts/phones", "Priya", undefined, "https://detailingweekly.example"], + ]); + expect(items[0]).toMatchObject({ watchId: "01JWATCH", watchName: "Needs booking software", subreddit: "smallbusiness", occurredAt: "2026-11-02T14:10:00.000Z", receivedAt: NOW.toISOString() }); + }); + + test("a body that isn't one, or a digest too big to keep, is refused in words", () => { + expect(readMagpie({ hello: 1 }, NOW).error).toContain("Not an OpenMagpie webhook body"); + expect(readMagpie({ watch: { id: "w", name: "n" }, items: Array.from({ length: 501 }, (_, i) => ({ key: `k${i}` })) }, NOW).error).toContain("at most 500"); + expect(readMagpie({ watch: { id: "w", name: "n" }, items: [{ key: "" }, "nonsense", { key: "ok" }] }, NOW).items.map((i) => i.key)).toEqual(["ok"]); + }); +}); + +describe("a match as a prospect", () => { + const [reddit, tweet, repo, post] = readMagpie(digest, NOW).items as [ListenItem, ListenItem, ListenItem, ListenItem]; + + test("the trigger line is the post itself, quoted short", () => { + expect(triggerOf(reddit)).toBe('Saw your post in r/smallbusiness: "Any booking tool for a two-person detailing shop? We keep missing calls".'); + expect(triggerOf(tweet)).toBe('Saw your post on X: "Is there a booking page that turns Google reviews into appointments? Asking for my tint…".'); + expect(triggerOf(repo)).toBe("Came across acme/bookit on GitHub."); + expect(triggerOf(post)).toBe('Read your post "Why we stopped taking bookings by phone".'); + expect(quoteOf("one two three four five six seven", 20)).toBe("one two three four…"); + }); + + test("an X post is reachable by X DM and keyed by the handle, so opt-outs elsewhere apply", () => { + const p = listenProspect(tweet); + expect(p.key).toBe("x:clearviewtint"); + expect(p.identities).toEqual(["x:clearviewtint", "listen:twitter_search:1850"]); + expect(p.xHandle).toBe("clearviewtint"); + expect(p.label).toBe("Dana Reyes @ClearviewTint on X"); + expect(p.variables).toMatchObject({ first_name: "Dana", post_url: "https://x.com/clearviewtint/status/1850", listen_watch: "Needs booking software" }); + }); + + test("a Reddit username is never a first name", () => { + const p = listenProspect(reddit); + expect(p.key).toBe("listen:reddit_subreddit:t3_abc"); + expect(p.label).toBe("u/shinyhands on r/smallbusiness"); + expect(p.variables.first_name).toBeUndefined(); + expect(p.variables.trigger).toStartWith("Saw your post in r/smallbusiness"); + }); +}); + +describe("the matches kept", () => { + test("a redelivery is kept once, and lists come newest post first, per watch", async () => { + const store = new ListenStore(await openDatabase(":memory:")); + const { items } = readMagpie(digest, NOW); + expect(await store.receive(items)).toEqual({ added: 4 }); + expect(await store.receive(items)).toEqual({ added: 0 }); + expect((await store.items()).map((i) => i.key)).toEqual(["twitter_search:1850", "reddit_subreddit:t3_abc", "github_search:42", "rss:entry-9"]); + expect((await store.items({ watch: "nope" })).length).toBe(0); + expect(await store.watches()).toEqual([{ id: "01JWATCH", name: "Needs booking software", count: 4, latest: NOW.toISOString() }]); + }); +}); diff --git a/src/domains/listen/types.ts b/src/domains/listen/types.ts new file mode 100644 index 0000000..392c89f --- /dev/null +++ b/src/domains/listen/types.ts @@ -0,0 +1,49 @@ +/** + * Listening: posts where someone asks for what you sell, found by OpenMagpie + * (the self-hostable social listener behind ListeningKit) and delivered to + * cold by its webhook action. Each one becomes a prospect whose trigger line + * is the post itself. + */ + +/** + * OpenMagpie's webhook body (openmagpie_schema/watch_actions/webhook.py, + * WebhookPayload): one item on instant delivery, several on a digest. Each + * item's body is the connector's payload: title, content, url and the + * connector's own fields (Reddit's author, permalink and subreddit, X's + * author and handle, RSS's author and feed_url, GitHub's owner and homepage). + */ +export interface MagpiePayload { + watch: { id: string; name: string }; + action_id?: string; + delivery?: string; + items: Array<{ key: string; source?: { label?: string; kind?: string }; item?: Record }>; +} + +/** One match, as cold keeps it. */ +export interface ListenItem { + /** OpenMagpie's stable key, source:external_id. Delivery is at least once, so this dedupes. */ + key: string; + watchId: string; + watchName: string; + /** The connector: reddit_subreddit, twitter_search, hackernews, github_search, rss... */ + sourceKind: string; + /** "r/smallbusiness", a feed's name... */ + sourceLabel: string; + title: string; + content: string; + /** Where the post is. */ + url: string; + author?: string; + /** Their X handle, when the post came from X. */ + handle?: string; + subreddit?: string; + /** A site the author runs: a GitHub project's homepage, the blog an RSS post is on. */ + website?: string; + occurredAt?: string; + receivedAt: string; +} + +export const LISTEN_PREFIX = "listen:"; +/** One delivery's limits, so a runaway digest can't fill the database. */ +export const MAX_LISTEN_ITEMS = 500; +export const MAX_LISTEN_BYTES = 2_000_000; diff --git a/src/domains/prospects/helpers/csvText.ts b/src/domains/prospects/helpers/csvText.ts new file mode 100644 index 0000000..1440c52 --- /dev/null +++ b/src/domains/prospects/helpers/csvText.ts @@ -0,0 +1,10 @@ +/** One CSV cell: quoted when it must be, and never read as a formula by a spreadsheet. Pure. */ +export function csvCell(value: string | undefined): string { + let s = value ?? ""; + if (/^[=+\-@\t\r]/.test(s)) s = `'${s}`; + return /[",\n\r]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; +} + +/** Rows as CSV text with these headers, in this order. Pure. */ +export const csvText = (headers: string[], rows: Array>): string => + [headers.map(csvCell).join(","), ...rows.map((r) => headers.map((h) => csvCell(r[h])).join(","))].join("\n") + "\n"; diff --git a/src/domains/prospects/helpers/index.ts b/src/domains/prospects/helpers/index.ts index 9d4d16a..81e4f1c 100644 --- a/src/domains/prospects/helpers/index.ts +++ b/src/domains/prospects/helpers/index.ts @@ -1,3 +1,4 @@ +export * from "./csvText"; export * from "./normalize"; export * from "./parseCsv"; export * from "./recordVariables"; diff --git a/src/domains/prospects/index.ts b/src/domains/prospects/index.ts index 41ac1df..d695601 100644 --- a/src/domains/prospects/index.ts +++ b/src/domains/prospects/index.ts @@ -7,7 +7,10 @@ import type { Prospect } from "./types"; export * from "./types"; export * from "./helpers"; -export const SOURCES_HELP = "csv:, twenty:people, twenty:agencyProspects or twenty:agencyPeople"; +export const SOURCES_HELP = "csv:, twenty:people, twenty:agencyProspects, twenty:agencyPeople or twenty:agencyLeads"; + +/** Every prospect source the app accepts: a CSV, a Twenty object, or a listening watch (listen: or listen:all). */ +export const PROSPECT_SOURCE = /^(csv:.+\.csv|twenty:(people|agencyProspects|agencyPeople|agencyLeads)|listen:[\w-]+)$/; export interface LoadOptions { /** A Twenty REST filter, e.g. coldCallStatus[eq]:NEW */ diff --git a/src/domains/prospects/types.ts b/src/domains/prospects/types.ts index a5935e2..03fb047 100644 --- a/src/domains/prospects/types.ts +++ b/src/domains/prospects/types.ts @@ -1,5 +1,5 @@ /** Where prospects come from. */ -export type ProspectSourceKind = "csv" | "people" | "agencyProspects" | "agencyPeople"; +export type ProspectSourceKind = "csv" | "people" | "agencyProspects" | "agencyPeople" | "agencyLeads" | "listen"; /** Twenty record ids a touch is linked to, so it shows on the right timelines. */ export interface ProspectLinks { diff --git a/src/domains/web/ai/index.ts b/src/domains/web/ai/index.ts index 50734ed..a467824 100644 --- a/src/domains/web/ai/index.ts +++ b/src/domains/web/ai/index.ts @@ -25,7 +25,7 @@ import { prepareDmRun } from "../../dm"; import { prepareEmailRun } from "../../email"; import { findSequence } from "../../messages"; import { buildVariables, renderForProspect } from "../../outreach"; -import { parseCsv, type Prospect } from "../../prospects"; +import { csvText, parseCsv, type Prospect } from "../../prospects"; import { twentyClient } from "../../touches"; import { HttpError, json, readBody, type Route } from "../api"; import { OWN_LIST, type WebContext } from "../context"; @@ -34,15 +34,6 @@ import { offerOf, prospectOf, senderOf, SOURCE, startRun } from "./shared"; export { offerOf, prospectOf, senderOf, SOURCE, startRun } from "./shared"; -/** A CSV cell: quoted when it needs to be, and never read as a formula. */ -function cell(value: string | undefined): string { - let s = value ?? ""; - if (/^[=+\-@\t\r]/.test(s)) s = `'${s}`; - return /[",\n\r]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; -} -const csvText = (headers: string[], rows: Array>) => - [headers.map(cell).join(","), ...rows.map((r) => headers.map((h) => cell(r[h])).join(","))].join("\n") + "\n"; - /** Only http(s) links from research reach the page. */ const httpOnly = (urls: string[] | undefined) => (urls ?? []).filter((u) => /^https?:\/\//i.test(u)); diff --git a/src/domains/web/ai/shared.ts b/src/domains/web/ai/shared.ts index 704ce51..2389e79 100644 --- a/src/domains/web/ai/shared.ts +++ b/src/domains/web/ai/shared.ts @@ -1,6 +1,6 @@ import { explainError, type AiOffer, type AiProspect, type AiProvider, type AiResult, type AiRunKind, type AiSender, type AiStore } from "../../ai"; import type { Account, Offer } from "../../config"; -import type { Prospect } from "../../prospects"; +import { PROSPECT_SOURCE, type Prospect } from "../../prospects"; import { keepAlive } from "../background"; export const offerOf = (offer?: Offer): AiOffer => ({ @@ -18,8 +18,8 @@ export const senderOf = (account: Account): AiSender & { phone?: string } => ({ }); export const prospectOf = (p: Prospect): AiProspect => ({ name: p.label, ...p.variables }); -/** The prospect sources the API accepts, as in the main API. */ -export const SOURCE = /^(csv:.+\.csv|twenty:(people|agencyProspects|agencyPeople))$/; +/** The prospect sources the API accepts, the same as the main API's. */ +export const SOURCE = PROSPECT_SOURCE; /** One AI run, ready to go: what it is about, whose API it uses, and the work. */ export interface PreparedRun { diff --git a/src/domains/web/api/index.ts b/src/domains/web/api/index.ts index 27508a1..85b4e9b 100644 --- a/src/domains/web/api/index.ts +++ b/src/domains/web/api/index.ts @@ -9,7 +9,7 @@ import { dmChecklist, prepareDmRun, recordDmOutcome, type DmOutcome } from "../. import { describeWindow, prepareEmailRun, smtpTransport, syncReplies, testImap, toSendPlan, type EmailOverride } from "../../email"; import { findMessage, findSequence, lintMessage, lintSequences, toMessage, toSequences, type Library, type Message } from "../../messages"; import { buildVariables, renderForProspect } from "../../outreach"; -import { findProspect, parseCsv, type Prospect } from "../../prospects"; +import { findProspect, parseCsv, PROSPECT_SOURCE, type Prospect } from "../../prospects"; import { renderTemplate, VARIABLES } from "../../render"; import { summarize } from "../../stats"; import { TOUCH_KINDS, twentyClient, type Touch } from "../../touches"; @@ -48,7 +48,7 @@ export class HttpError extends Error { export const json = (data: unknown, status = 200) => Response.json(data, { status, headers: { "cache-control": "no-store" } }); const MESSAGE_ID = /^(email|dm)\/(personal|company)\/[a-z0-9]+(?:-[a-z0-9]+)*\/\d+-[a-z0-9]+(?:-[a-z0-9]+)*$|^call\/(personal|company)\/[a-z0-9]+(?:-[a-z0-9]+)*$/; -const SOURCE = /^(csv:.+\.csv|twenty:(people|agencyProspects|agencyPeople))$/; +const SOURCE = PROSPECT_SOURCE; const ANSI = /\x1b\[[0-9;]*m/g; export async function readBody(request: Request): Promise { @@ -113,6 +113,7 @@ function envFields(config: ColdConfig | undefined): SettingsResponse["env"] { fields.set("REACHER_SECRET", "Optional: Reacher's header secret"); fields.set("X_CLIENT_ID", "Optional: your X developer app's OAuth 2.0 Client ID, so X DMs go out from cold through X's API (developer.x.com). Then Connect X below"); fields.set("X_CLIENT_SECRET", "Optional: that app's OAuth 2.0 Client Secret (a Web App; leave it empty for a Native App)"); + fields.set("COLD_LISTEN_SECRET", "Optional: a long random string. OpenMagpie's webhook action sends it as Authorization: Bearer , so its matches come in as prospects"); return [...fields.entries()].map(([name, hint]) => ({ name, set: !!process.env[name]?.trim(), hint })); } @@ -401,6 +402,9 @@ export function createApi(ctx: WebContext, extraRoutes: Route[] = []): (request: sources.push({ id: "twenty:agencyProspects", label: "Twenty: dialer prospects (businesses)", kind: "twenty" }); sources.push({ id: "twenty:agencyPeople", label: "Twenty: people at dialer prospects", kind: "twenty" }); } + const watches = (await ctx.listening?.watches()) ?? []; + if (watches.length > 1) sources.push({ id: "listen:all", label: `Listening: every watch (${watches.reduce((n, w) => n + w.count, 0)} matches)`, kind: "listen" }); + for (const w of watches) sources.push({ id: `listen:${w.id}`, label: `Listening: ${w.name} (${w.count} ${w.count === 1 ? "match" : "matches"})`, kind: "listen" }); return json(sources); }, ], diff --git a/src/domains/web/context/index.ts b/src/domains/web/context/index.ts index 16a63d9..c8d901d 100644 --- a/src/domains/web/context/index.ts +++ b/src/domains/web/context/index.ts @@ -10,6 +10,7 @@ import { loadProspects, prospectsFromCsv, type Prospect } from "../../prospects" import { DbFiles, DiskFiles, EnvFileSecrets, secretKey, type Files, type Secrets } from "../../storage"; import { DbStore, filesJournal, openStore, twentyClient, TwentyStore, type TouchStore } from "../../touches"; import { xAppFromEnv, XClient, XStore, type XFetch } from "../../xApi"; +import { LISTEN_PREFIX, ListenStore, listenProspect } from "../../listen"; import { CampaignStore } from "../campaigns"; import { processScoreItem } from "../decide/engine"; import { ScoreJobs } from "../decide/jobs"; @@ -79,6 +80,8 @@ export class WebContext { readonly scoring?: ScoreJobs; /** X connections, sign-ins and sends through X's API; only with the database. */ readonly x?: XStore; + /** Listening matches OpenMagpie delivered; only with the database. */ + readonly listening?: ListenStore; private readonly prospectCache = new Map(); private storeCache?: { at: number; key: string; store: Promise }; private kept: { config?: Promise; library?: ReturnType } = {}; @@ -104,6 +107,7 @@ export class WebContext { random: options.random, }); this.x = new XStore(options.db, { key: usableSecretKey(), requireKey: this.storage === "database", now: () => this.now() }); + this.listening = new ListenStore(options.db); } } @@ -274,6 +278,11 @@ export class WebContext { if (source.startsWith("csv:")) { if (where) throw new Error("A Twenty filter only applies to a Twenty source."); prospects = prospectsFromCsv((await this.csv(source)).text); + } else if (source.startsWith(LISTEN_PREFIX)) { + if (where) throw new Error("A Twenty filter only applies to a Twenty source."); + if (!this.listening) throw new Error("Listening needs the app's database."); + const watch = source.slice(LISTEN_PREFIX.length); + prospects = (await this.listening.items({ watch: watch === "all" ? undefined : watch, limit: 2000 })).map(listenProspect); } else { prospects = await loadProspects(source, { where: where || undefined, client: twentyClient() }); } diff --git a/src/domains/web/guard/index.ts b/src/domains/web/guard/index.ts index 4622c86..5759180 100644 --- a/src/domains/web/guard/index.ts +++ b/src/domains/web/guard/index.ts @@ -15,18 +15,20 @@ export interface Access { * - An Origin, when the browser sends one, must be this server's. * * /api/health is open to anyone, for a host's uptime checks; /api/cron/ - * carries its own secret instead (Vercel's cron calls it). /api/x/callback is - * where X sends the owner back after Connect X: a plain page load, so no - * header, answered on this server's own hosts only and trusted only through - * its one-time state. + * carries its own secret instead (Vercel's cron calls it). The routes in + * OPEN_ROUTES are called by other services, not the app, so they carry no + * header: they are answered on this server's own hosts only, and each checks + * its own proof (Connect X's one-time state, the listening webhook's secret). */ +export const OPEN_ROUTES: ReadonlySet = new Set(["GET /api/x/callback", "POST /api/listen/magpie"]); + export function refuseRequest(request: Request, access: Access): string | undefined { const { pathname } = new URL(request.url); if (pathname === "/api/health" || pathname.startsWith("/api/cron/")) return undefined; const host = request.headers.get("host") ?? ""; if (!access.hosts.includes(host)) return `cold does not answer on ${host || "this address"}`; if (!pathname.startsWith("/api/")) return undefined; - if (pathname === "/api/x/callback" && request.method === "GET") return undefined; + if (OPEN_ROUTES.has(`${request.method} ${pathname}`)) return undefined; if (request.headers.get("x-cold") !== "1") return "missing the x-cold header"; const origin = request.headers.get("origin"); if (origin && !access.origins.includes(origin)) return "cross-origin requests are not allowed"; diff --git a/src/domains/web/listen/index.ts b/src/domains/web/listen/index.ts new file mode 100644 index 0000000..9b73cbb --- /dev/null +++ b/src/domains/web/listen/index.ts @@ -0,0 +1,82 @@ +import { createHash, timingSafeEqual } from "node:crypto"; +import { listenCsv, MAX_LISTEN_BYTES, readMagpie } from "../../listen"; +import { HttpError, json, readBody, type Route } from "../api"; +import type { WebContext } from "../context"; +import type { ListenStatusResponse } from "../types"; + +export const LISTEN_WEBHOOK_PATH = "/api/listen/magpie"; + +/** Two secrets compared in constant time, whatever their lengths. */ +function sameSecret(given: string | null, expected: string): boolean { + const a = createHash("sha256").update(given ?? "").digest(); + const b = createHash("sha256").update(expected).digest(); + return timingSafeEqual(a, b); +} + +/** + * OpenMagpie's webhook action delivers matches here (POST, instant or + * digest). It proves itself with `Authorization: Bearer `, + * set in the action's headers. Delivery is at least once: a key already kept + * is counted as received, not added again. + */ +export function listenWebhook(ctx: WebContext): (request: Request) => Promise { + return async (request) => { + const secret = process.env.COLD_LISTEN_SECRET?.trim(); + if (!secret) return json({ error: "Listening is not set up here: set COLD_LISTEN_SECRET in cold's Settings, and the same value in OpenMagpie's webhook headers." }, 503); + if (!sameSecret(request.headers.get("authorization"), `Bearer ${secret}`)) return json({ error: "Not for you: the Authorization header must be Bearer and COLD_LISTEN_SECRET." }, 401); + if (!ctx.listening) return json({ error: "This server has no database to keep matches." }, 503); + if (Number(request.headers.get("content-length") ?? 0) > MAX_LISTEN_BYTES) return json({ error: "That delivery is too large: use a shorter digest interval." }, 413); + const text = await request.text(); + if (text.length > MAX_LISTEN_BYTES) return json({ error: "That delivery is too large: use a shorter digest interval." }, 413); + let body: unknown; + try { + body = JSON.parse(text); + } catch { + return json({ error: "The body must be JSON." }, 400); + } + const { items, error } = readMagpie(body, ctx.now()); + if (error) return json({ error }, 400); + const { added } = await ctx.listening.receive(items); + if (added) ctx.invalidate(); + return json({ ok: true, received: items.length, added }); + }; +} + +/** Settings, Listening: where OpenMagpie should deliver, and what has come in. Behind the login. */ +export function listenRoutes(ctx: WebContext): Route[] { + return [ + [ + "GET", + /^\/api\/listen$/, + async (request) => { + const response: ListenStatusResponse = { + configured: !!process.env.COLD_LISTEN_SECRET?.trim(), + available: !!ctx.listening, + webhookUrl: new URL(LISTEN_WEBHOOK_PATH, new URL(request.url).origin).href, + watches: (await ctx.listening?.watches()) ?? [], + }; + return json(response); + }, + ], + [ + "POST", + /^\/api\/listen\/export$/, + async (request) => { + // A watch's matches as one of your own lists, to fill in and research; saving again only adds new matches. + const { watch } = await readBody<{ watch: string }>(request); + if (!ctx.listening) throw new HttpError(409, "Listening needs the app's database."); + if (!/^[\w-]+$/.test(watch ?? "")) throw new HttpError(400, "Pick a watch, or all."); + const name = watch === "all" ? "all" : (await ctx.listening.watches()).find((w) => w.id === watch)?.name; + if (!name) throw new HttpError(404, "No matches have come in from that watch yet."); + const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60) || "matches"; + const file = `prospects-listen-${slug}.csv`; + const existing = await ctx.files.read(file).catch(() => undefined); + const { text, added, rows } = listenCsv(await ctx.listening.items({ watch: watch === "all" ? undefined : watch, limit: 5000 }), existing); + await ctx.files.write(file, text); + ctx.invalidate(); + const message = `${added ? `Added ${added} new ${added === 1 ? "match" : "matches"} to` : "Nothing new for"} ${file} (${rows} in all). Fill in company where it's empty, or research them, then run a campaign on it.`; + return json({ ok: true, source: `csv:${file}`, added, rows, message }); + }, + ], + ]; +} diff --git a/src/domains/web/listen/listen.test.ts b/src/domains/web/listen/listen.test.ts new file mode 100644 index 0000000..b057c71 --- /dev/null +++ b/src/domains/web/listen/listen.test.ts @@ -0,0 +1,120 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { openDatabase } from "../../db"; +import { createApi } from "../api"; +import { WebContext } from "../context"; +import type { Campaign, ListenStatusResponse, PlanResponse, ProspectSummary, SourceOption } from "../types"; +import { listenRoutes, listenWebhook } from "."; + +const REPO = path.resolve(import.meta.dir, "../../../.."); +const ORIGIN = "http://127.0.0.1:4100"; +let root: string; +let ctx: WebContext; +let api: (request: Request) => Promise; +let webhook: (request: Request) => Promise; +const before = process.env.COLD_LISTEN_SECRET; + +const delivery = { + watch: { id: "01JWATCH", name: "Needs booking software" }, + action_id: "01JACTION", + delivery: "digest", + items: [ + { key: "reddit_subreddit:t3_abc", source: { label: "r/smallbusiness", kind: "reddit_subreddit" }, item: { occurred_at: "2026-11-02T14:10:00Z", title: "Any booking tool for a two-person detailing shop?", permalink: "/r/smallbusiness/comments/abc/x/", author: "shinyhands", subreddit: "smallbusiness" } }, + { key: "twitter_search:1850", source: { label: "X search", kind: "twitter_search" }, item: { occurred_at: "2026-11-02T14:20:00Z", content: "Is there a booking page that turns Google reviews into appointments?", url: "https://x.com/clearviewtint/status/1850", author: "Dana Reyes", handle: "clearviewtint" } }, + ], +}; + +const post = (body: unknown, headers: Record = {}) => + webhook(new Request(`${ORIGIN}/api/listen/magpie`, { method: "POST", headers: { "content-type": "application/json", ...headers }, body: typeof body === "string" ? body : JSON.stringify(body) })); + +async function call(method: string, url: string, body?: unknown): Promise<{ status: number; data: T }> { + const response = await api(new Request(`${ORIGIN}${url}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) })); + return { status: response.status, data: (await response.json()) as T }; +} + +beforeAll(async () => { + delete process.env.COLD_CONFIG; + root = mkdtempSync(path.join(tmpdir(), "cold-listen-")); + cpSync(path.join(REPO, "messages"), path.join(root, "messages"), { recursive: true }); + writeFileSync(path.join(root, "cold.yaml"), readFileSync(path.join(REPO, "cold.example.yaml"), "utf8").replace("store: auto", "store: file")); + ctx = new WebContext({ root, db: await openDatabase(":memory:"), now: () => new Date("2026-11-02T15:00:00Z") }); + api = createApi(ctx, listenRoutes(ctx)); + webhook = listenWebhook(ctx); +}); + +afterAll(() => { + rmSync(root, { recursive: true, force: true }); + if (before === undefined) delete process.env.COLD_LISTEN_SECRET; + else process.env.COLD_LISTEN_SECRET = before; +}); + +describe("OpenMagpie's webhook", () => { + test("until COLD_LISTEN_SECRET is set, it says how to set it up", async () => { + delete process.env.COLD_LISTEN_SECRET; + const response = await post(delivery, { authorization: "Bearer anything" }); + expect(response.status).toBe(503); + expect(((await response.json()) as { error: string }).error).toContain("set COLD_LISTEN_SECRET"); + }); + + test("a missing or wrong secret is refused and nothing is kept", async () => { + process.env.COLD_LISTEN_SECRET = "a-long-listening-secret"; + expect((await post(delivery)).status).toBe(401); + expect((await post(delivery, { authorization: "Bearer wrong" })).status).toBe(401); + expect(await ctx.listening!.items()).toEqual([]); + }); + + test("a delivery is kept, and the same delivery again adds nothing", async () => { + const auth = { authorization: "Bearer a-long-listening-secret" }; + const first = await post(delivery, auth); + expect(first.status).toBe(200); + expect(await first.json()).toEqual({ ok: true, received: 2, added: 2 }); + expect(await (await post(delivery, auth)).json()).toEqual({ ok: true, received: 2, added: 0 }); + expect((await post("not json", auth)).status).toBe(400); + expect((await post({ hello: "world" }, auth)).status).toBe(400); + }); +}); + +describe("matches as prospects", () => { + test("Settings shows where to point OpenMagpie, and the watches that delivered", async () => { + const { data } = await call("GET", "/api/listen"); + expect(data).toMatchObject({ configured: true, available: true, webhookUrl: `${ORIGIN}/api/listen/magpie` }); + expect(data.watches).toEqual([{ id: "01JWATCH", name: "Needs booking software", count: 2, latest: "2026-11-02T15:00:00.000Z" }]); + }); + + test("each watch is a prospect list, with the post as the trigger line", async () => { + const { data: sources } = await call("GET", "/api/sources"); + expect(sources.filter((s) => s.kind === "listen")).toEqual([{ id: "listen:01JWATCH", label: "Listening: Needs booking software (2 matches)", kind: "listen" }]); + const { data } = await call<{ prospects: ProspectSummary[] }>("GET", "/api/prospects?source=listen:01JWATCH"); + expect(data.prospects.map((p) => [p.label, p.trigger])).toEqual([ + ["Dana Reyes @clearviewtint on X", 'Saw your post on X: "Is there a booking page that turns Google reviews into appointments?".'], + ["u/shinyhands on r/smallbusiness", 'Saw your post in r/smallbusiness: "Any booking tool for a two-person detailing shop?".'], + ]); + }); + + test("a campaign runs on it like any list: the X poster waits for their company, the Reddit one has no channel", async () => { + const { status, data: campaign } = await call("POST", "/api/campaigns", { name: "Booking askers", sequence: "dm/personal/x", account: "personal", source: "listen:01JWATCH" }); + expect(status).toBe(201); + const { data: plan } = await call("GET", `/api/campaigns/${campaign.id}/plan`); + expect(plan.due).toEqual([]); + expect(plan.held.map((r) => [r.label, r.reason])).toEqual([["Dana Reyes @clearviewtint on X", "missing {{company}}"]]); + expect(plan.skipped.map((r) => [r.label, r.reason])).toEqual([["u/shinyhands on r/smallbusiness", "no x profile"]]); + }); + + test("saved as a list, a match can be filled in, and saving again only adds what is new", async () => { + const { data } = await call<{ source: string; added: number; rows: number; message: string }>("POST", "/api/listen/export", { watch: "01JWATCH" }); + expect(data).toMatchObject({ source: "csv:prospects-listen-needs-booking-software.csv", added: 2, rows: 2 }); + const file = path.join(root, "prospects-listen-needs-booking-software.csv"); + const text = readFileSync(file, "utf8"); + expect(text.split("\n")[0]).toBe("id,first_name,full_name,company,email,website,x_handle,trigger,post_url,post_title,post_text,posted_at,listen_source,listen_watch"); + writeFileSync(file, text.replace("twitter_search:1850,Dana,Dana Reyes,,", "twitter_search:1850,Dana,Dana Reyes,Clearview Tint,")); + expect((await call<{ added: number; rows: number }>("POST", "/api/listen/export", { watch: "01JWATCH" })).data).toMatchObject({ added: 0, rows: 2 }); + expect(readFileSync(file, "utf8")).toContain("Clearview Tint"); + const { data: campaign } = await call("POST", "/api/campaigns", { name: "Booking askers, filled in", sequence: "dm/personal/x", account: "personal", source: data.source }); + const { data: plan } = await call("GET", `/api/campaigns/${campaign.id}/plan?refresh=1`); + expect(plan.due.map((r) => r.key)).toEqual(["x:clearviewtint"]); + expect(plan.due[0]!.text).toContain('Saw your post on X: "Is there a booking page that turns Google reviews into appointments?".'); + expect(plan.due[0]!.text).toContain("for Clearview Tint?"); + }); +}); diff --git a/src/domains/web/open/index.ts b/src/domains/web/open/index.ts new file mode 100644 index 0000000..0851cb4 --- /dev/null +++ b/src/domains/web/open/index.ts @@ -0,0 +1,20 @@ +import type { WebContext } from "../context"; +import { OPEN_ROUTES } from "../guard"; +import { listenWebhook } from "../listen"; +import { xCallback } from "../x"; + +/** + * The routes other services call, outside the login, each with its own proof: + * X's Connect X callback (a one-time state) and OpenMagpie's webhook (a shared + * secret). The guard lets exactly these through without the app's header. + */ +export function publicRoutes(ctx: WebContext): Record Promise> { + const callback = xCallback(ctx); + const webhook = listenWebhook(ctx); + const routes: Record Promise> = { + "GET /api/x/callback": (_request, url) => callback(url), + "POST /api/listen/magpie": (request) => webhook(request), + }; + for (const route of Object.keys(routes)) if (!OPEN_ROUTES.has(route)) throw new Error(`${route} is public but the guard doesn't let it through`); + return routes; +} diff --git a/src/domains/web/server/handler.test.ts b/src/domains/web/server/handler.test.ts index 27242cd..d2eb283 100644 --- a/src/domains/web/server/handler.test.ts +++ b/src/domains/web/server/handler.test.ts @@ -36,6 +36,7 @@ beforeEach(async () => { privateCalls.push(new URL(request.url).pathname); return Response.json({ private: true }); }, + publicRoutes: { "POST /api/listen/magpie": async (request) => Response.json({ open: true, auth: request.headers.get("authorization") }) }, clientIp: () => "127.0.0.1", }); }); @@ -78,6 +79,14 @@ describe("the public side", () => { expect(privateCalls).toEqual([]); expect((await call("GET", "/api/waitlist")).status).toBe(401); }); + + test("a route another service calls answers without a session or the app's header, and checks its own proof", async () => { + const response = await call("POST", "/api/listen/magpie", { items: [] }, { "x-cold": "", authorization: "Bearer s3cret" }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ open: true, auth: "Bearer s3cret" }); + expect(privateCalls).toEqual([]); + expect((await call("GET", "/api/listen/magpie", undefined, { "x-cold": "" })).status).toBe(403); + }); }); describe("signing in", () => { diff --git a/src/domains/web/server/handler.ts b/src/domains/web/server/handler.ts index e5c1a91..aebdaa2 100644 --- a/src/domains/web/server/handler.ts +++ b/src/domains/web/server/handler.ts @@ -161,8 +161,13 @@ export interface HandlerDeps { limiter?: (limit: number, windowMs: number) => Limiter; /** What Vercel's cron job runs every minute (the next emails of a send), behind CRON_SECRET. */ cron?: () => Promise; - /** Where X sends the owner back after Connect X. Trusted through its one-time state, not the session cookie (a strict cookie doesn't come back from x.com). */ - xCallback?: (url: URL) => Promise; + /** + * Routes outside the login, by "METHOD /path", each trusted through its own + * check instead of the session cookie: X's Connect X callback (its one-time + * state; a strict cookie doesn't come back from x.com) and the listening + * webhook (its shared secret). + */ + publicRoutes?: Record Promise>; now?: () => Date; } @@ -237,13 +242,12 @@ export function createHandler(deps: HandlerDeps): (request: Request) => Promise< const route = `${request.method} ${url.pathname}`; const ip = deps.clientIp(request); const token = parseCookies(request.headers.get("cookie"))[SESSION_COOKIE]; + const open = deps.publicRoutes?.[route]; + if (open) return open(request, url); switch (route) { case "GET /api/health": return json({ ok: true, version }); - case "GET /api/x/callback": - if (!deps.xCallback) throw new HttpError(404, "X is not set up on this server."); - return deps.xCallback(url); case "GET /api/cron/tick": { const secret = process.env.CRON_SECRET; if (!deps.cron || !secret || request.headers.get("authorization") !== `Bearer ${secret}`) throw new HttpError(401, "Not for you."); diff --git a/src/domains/web/server/index.ts b/src/domains/web/server/index.ts index 54b05e1..d82efbd 100644 --- a/src/domains/web/server/index.ts +++ b/src/domains/web/server/index.ts @@ -22,7 +22,9 @@ import { aiRoutes } from "../ai"; import { decideRoutes } from "../decide"; import { createApi } from "../api"; import { WebContext } from "../context"; -import { xCallback, xRoutes } from "../x"; +import { listenRoutes } from "../listen"; +import { publicRoutes } from "../open"; +import { xRoutes } from "../x"; import { buildAssets, bundle, DIST, readBundle } from "./assets"; import { clientAddress, createHandler, SetupCode } from "./handler"; import { serverSettings } from "./settings"; @@ -61,10 +63,10 @@ const handler = createHandler({ version: assets.version, auth, waitlist, - privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx), ...xRoutes(ctx)]), + privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx), ...xRoutes(ctx), ...listenRoutes(ctx)]), setup, clientIp: clientAddress(settings.proxyHops, (request) => server.requestIP(request)?.address), - xCallback: xCallback(ctx), + publicRoutes: publicRoutes(ctx), }); try { diff --git a/src/domains/web/types.ts b/src/domains/web/types.ts index 7b61107..b8834e0 100644 --- a/src/domains/web/types.ts +++ b/src/domains/web/types.ts @@ -97,7 +97,18 @@ export interface PreviewResponse { export interface SourceOption { id: string; label: string; - kind: "csv" | "twenty"; + kind: "csv" | "twenty" | "listen"; +} + +/** Settings, Listening: OpenMagpie's webhook target and the watches that have delivered. */ +export interface ListenStatusResponse { + /** COLD_LISTEN_SECRET is set. */ + configured: boolean; + /** The app's database is there to keep matches. */ + available: boolean; + /** Point OpenMagpie's webhook action here, with Authorization: Bearer . */ + webhookUrl: string; + watches: Array<{ id: string; name: string; count: number; latest: string }>; } export interface ProspectSummary { diff --git a/src/domains/web/vercel/index.ts b/src/domains/web/vercel/index.ts index 236eb02..f376a6a 100644 --- a/src/domains/web/vercel/index.ts +++ b/src/domains/web/vercel/index.ts @@ -11,7 +11,9 @@ import { invocationDeadline } from "../background"; import { WebContext } from "../context"; import { clientAddress, createHandler, DbSetupCode } from "../server/handler"; import { serverSettings } from "../server/settings"; -import { xCallback, xRoutes } from "../x"; +import { listenRoutes } from "../listen"; +import { publicRoutes } from "../open"; +import { xRoutes } from "../x"; /** Things that only need doing now and then: expired rate-limit windows and sessions. */ async function tidy(db: Db): Promise { @@ -44,11 +46,11 @@ export async function createVercelApp(db: Db, options: { now?: () => Date } = {} version: (process.env.VERCEL_GIT_COMMIT_SHA ?? process.env.VERCEL_DEPLOYMENT_ID ?? "dev").slice(0, 12), auth, waitlist: new Waitlist(db), - privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx), ...xRoutes(ctx)]), + privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx), ...xRoutes(ctx), ...listenRoutes(ctx)]), setup, // Vercel sets X-Forwarded-For to the visitor's address itself. clientIp: clientAddress(settings.proxyHops, () => undefined), - xCallback: xCallback(ctx), + publicRoutes: publicRoutes(ctx), limiter: (limit, windowMs) => new DbRateLimiter(db, limit, windowMs), cron: async () => { await ctx.freshen(); diff --git a/src/web/pages/settings.tsx b/src/web/pages/settings.tsx index 933bb56..ed209ac 100644 --- a/src/web/pages/settings.tsx +++ b/src/web/pages/settings.tsx @@ -1,6 +1,6 @@ -import { AtSign, CheckCircle2, Copy, Database, FileText, KeyRound, LogOut, Mailbox, MonitorSmartphone, PlugZap, Save, ShieldCheck, Sparkles, Unplug, UserRound, Wand2, XCircle } from "lucide-react"; +import { AtSign, CheckCircle2, Copy, Database, Ear, FileText, KeyRound, ListPlus, LogOut, Mailbox, MonitorSmartphone, PlugZap, Save, ShieldCheck, Sparkles, Unplug, UserRound, Wand2, XCircle } from "lucide-react"; import { useEffect, useState, type ReactNode } from "react"; -import type { ActionResponse, AiStatusResponse, SessionsResponse, SettingsResponse, XStatusResponse } from "../../domains/web/types"; +import type { ActionResponse, AiStatusResponse, ListenStatusResponse, SessionsResponse, SettingsResponse, XStatusResponse } from "../../domains/web/types"; import { usePwa, install } from "../app/pwa"; import { useSession } from "../app/session"; import { useStatus } from "../app/status"; @@ -133,6 +133,7 @@ export function SettingsPage() { + {settings.loading && !settings.data ? : settings.error ? : settings.data && (status.reload(), settings.reload())} />} {settings.data && (status.reload(), settings.reload())} />} @@ -140,6 +141,81 @@ export function SettingsPage() { ); } +/** + * Listening: OpenMagpie (ListeningKit's self-hostable listener) delivers the + * posts its watches match to cold's webhook, and each watch becomes a + * prospect list whose trigger line is the post. + */ +function ListenCard() { + const listen = useApi("/api/listen"); + const toast = useToast(); + const [busy, setBusy] = useState(null); + + async function save(watch: string) { + setBusy(watch); + try { + toast.success((await api("/api/listen/export", { method: "POST", body: { watch } })).message); + } catch (err) { + toast.error((err as Error).message); + } finally { + setBusy(null); + } + } + + async function copy(text: string, what: string) { + try { + await navigator.clipboard.writeText(text); + toast.success(`Copied the ${what}.`); + } catch { + toast.error("The browser blocked the clipboard: select it and copy it."); + } + } + + const d = listen.data; + return ( + + {listen.loading && !d ? ( + + ) : listen.error ? ( + + ) : ( + d && ( +
+

+ Posts where someone asks for what you sell, found by OpenMagpie's watches, come in here as prospects. Each watch is a prospect list whose trigger line is the post itself. Save a watch as a list to fill in what's missing (company, email) and research it. +

+ {d.configured ? "COLD_LISTEN_SECRET is set" : "Set COLD_LISTEN_SECRET below: any long random string"} + {!d.available && This server has no database to keep matches.} +
+ In OpenMagpie, give the watch a webhook action that POSTs here, with the header Authorization: Bearer and your COLD_LISTEN_SECRET: +
+ {d.webhookUrl} + +
+
+ {d.watches.length === 0 ? ( + No matches have come in yet. + ) : ( + d.watches.map((w) => ( +
+ + {w.name} {w.count} {w.count === 1 ? "match" : "matches"}, latest {ago(w.latest)} + + +
+ )) + )} +
+ ) + )} +
+ ); +} + /** * X DMs through X's API: the owner's own developer app, and each account * connected to the X account it sends as. One DM goes out per click, from From 556c83b2f466ea8a99131a8029b8c57dca7b318d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 16:00:38 +0000 Subject: [PATCH 3/6] Quiz leads get called first: offer's leads as a source, a push, a list on Calls - twenty:agencyLeads is a prospect source: the person who took offer's quiz (read from the note offer writes: contact lines, quiz answers, offer and UTM tags), the business's details when they came through its own funnel link, never its decision maker's name, and "Saw you took our quiz on Nov 2." as the trigger when the business has none. - Calls lists the last two weeks' qualified leads nobody has called yet at the top, newest first. Picking one (or opening its push) fills the script in for them, with a Call button and their answers. - With COLD_NTFY_URL (ntfy), a push goes out within a minute of each new qualified lead; more than five at once come as one. The look is claimed in the database, so it runs once a minute across every server, and the newest lead seen is kept there: a lead is pushed once, a failed push is retried, leads from before alerts were set up are never pushed. The local server's timer and the Vercel cron both run it. Settings, Lead alerts sends a test. - Fix: findProspect lowercased the query but compared it with keys such as agencyProspect: exactly, so a dialer prospect picked in a call sheet or a research run was never found and the script kept its braces. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuUQ7RbYzBy4coULGCu --- .env.example | 7 + CHANGELOG.md | 17 ++ README.md | 6 + src/domains/leads/helpers/index.ts | 2 + src/domains/leads/helpers/ntfy.ts | 40 +++ src/domains/leads/helpers/quizLead.ts | 75 ++++++ src/domains/leads/index.ts | 124 +++++++++ src/domains/leads/leads.test.ts | 252 ++++++++++++++++++ src/domains/leads/types.ts | 66 +++++ .../prospects/helpers/agencyLead.test.ts | 149 +++++++++++ src/domains/prospects/helpers/agencyLead.ts | 97 +++++++ src/domains/prospects/helpers/index.ts | 1 + src/domains/prospects/index.ts | 25 +- src/domains/prospects/types.ts | 2 + src/domains/web/api/index.ts | 3 + src/domains/web/context/index.ts | 18 +- src/domains/web/leads/index.ts | 66 +++++ src/domains/web/leads/leads.test.ts | 116 ++++++++ src/domains/web/server/index.ts | 7 +- src/domains/web/types.ts | 30 +++ src/domains/web/vercel/index.ts | 9 +- src/web/pages/calls.tsx | 81 +++++- src/web/pages/settings.tsx | 52 +++- 23 files changed, 1228 insertions(+), 17 deletions(-) create mode 100644 src/domains/leads/helpers/index.ts create mode 100644 src/domains/leads/helpers/ntfy.ts create mode 100644 src/domains/leads/helpers/quizLead.ts create mode 100644 src/domains/leads/index.ts create mode 100644 src/domains/leads/leads.test.ts create mode 100644 src/domains/leads/types.ts create mode 100644 src/domains/prospects/helpers/agencyLead.test.ts create mode 100644 src/domains/prospects/helpers/agencyLead.ts create mode 100644 src/domains/web/leads/index.ts create mode 100644 src/domains/web/leads/leads.test.ts diff --git a/.env.example b/.env.example index abf82d7..facc6e9 100644 --- a/.env.example +++ b/.env.example @@ -55,6 +55,13 @@ X_CLIENT_SECRET= # webhook action the header Authorization: Bearer . COLD_LISTEN_SECRET= +# A push when one of offer's quiz leads qualifies (Settings, Lead alerts): +# the address of an ntfy topic, on ntfy.sh or your own ntfy server. The push +# carries the lead's name and number, so use a long random topic name, or a +# topic that needs COLD_NTFY_TOKEN (an ntfy access token, tk_...). +COLD_NTFY_URL= +COLD_NTFY_TOKEN= + # Hosting (docs/deploy.md). These belong in the server's real environment, # not in this file, since .env.local lives inside COLD_HOME. # COLD_PUBLIC_URL=https://cold.example.com diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dcb5aa..59ea8ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ ## Unreleased +- **Quiz leads get called first.** offer's quiz leads (`agencyLeads` in + Twenty) are a prospect source, `twenty:agencyLeads`: the person who took + the quiz (never the business's decision maker, when they came through a + business's own link), their answers and UTM tags as variables, and "Saw + you took our quiz on Nov 2." as the trigger when the business has none. + Calls lists the last two weeks' qualified leads nobody has called yet at + the top, newest first; picking one fills the script in for them, with a + Call button and their answers. With `COLD_NTFY_URL`, a push reaches your + phone through ntfy within a minute of each qualified lead (several at once + come as one), tapping it opens the lead; it looks once a minute across + every server on the database, never pushes a lead twice, retries one that + failed, and never pushes leads from before it was set up. Settings, Lead + alerts sends a test. +- **Picking a dialer prospect fills the script in again.** Prospects from + `twenty:agencyProspects` and `twenty:agencyPeople` were never found by + their key in a call sheet or a research run, so the script kept its + braces. - **Leads from listening.** OpenMagpie's webhook action delivers the posts its watches match to `/api/listen/magpie` (with `Authorization: Bearer` and `COLD_LISTEN_SECRET`). Each match is kept once, however often it is diff --git a/README.md b/README.md index 206dbcc..4d0f925 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,12 @@ call scripts and the Do Not Contact list. someone asks for what you sell comes in as a prospect whose trigger line is the post itself, with its link. A post on X can be answered with an X DM; save a watch as a list to fill in the rest (company, email) and research it. +- **Quiz leads get called first.** Leads from your + [offer](https://github.com/matthewdonsemail-lab/offer) quiz funnel are a + prospect source (`twenty:agencyLeads`) with their answers as variables, + listed at the top of Calls, newest first, until someone calls them. Set + `COLD_NTFY_URL` and a push reaches your phone ([ntfy](https://ntfy.sh)) a + minute after each qualified lead comes in, with their number. - **The messages are the product.** 45 messages across cold email, LinkedIn, X, Instagram and Facebook DMs, and call scripts, plus 11 objection answers. Every channel is covered twice, once as you and once as the company. Each message diff --git a/src/domains/leads/helpers/index.ts b/src/domains/leads/helpers/index.ts new file mode 100644 index 0000000..4e9ce15 --- /dev/null +++ b/src/domains/leads/helpers/index.ts @@ -0,0 +1,2 @@ +export * from "./ntfy"; +export * from "./quizLead"; diff --git a/src/domains/leads/helpers/ntfy.ts b/src/domains/leads/helpers/ntfy.ts new file mode 100644 index 0000000..ed99861 --- /dev/null +++ b/src/domains/leads/helpers/ntfy.ts @@ -0,0 +1,40 @@ +import type { NtfyMessage, NtfyTarget } from "../types"; + +/** + * The alert target from the environment: COLD_NTFY_URL is the topic's + * address (https://ntfy.sh/, or the topic on your own server) and + * COLD_NTFY_TOKEN an access token for a topic that needs one. Unset means no + * alerts; set but unusable comes back as a problem, in words. Pure. + */ +export function readNtfy(env: Record = process.env): { target?: NtfyTarget; problem?: string } { + const raw = env.COLD_NTFY_URL?.trim(); + if (!raw) return {}; + let url: URL; + try { + url = new URL(raw); + } catch { + return { problem: "COLD_NTFY_URL is not a URL: use the topic's address, e.g. https://ntfy.sh/cold-leads-7f3k2q." }; + } + if (url.protocol !== "https:" && url.protocol !== "http:") return { problem: "COLD_NTFY_URL must start with https://." }; + if (url.username || url.password) return { problem: "Put an access token in COLD_NTFY_TOKEN instead of a login in COLD_NTFY_URL." }; + const parts = url.pathname.split("/").filter(Boolean); + const topic = parts.pop(); + if (!topic || !/^[\w-]{1,64}$/.test(topic)) return { problem: "COLD_NTFY_URL must end in the topic's name (letters, digits, - and _), e.g. https://ntfy.sh/cold-leads-7f3k2q." }; + const server = `${url.origin}/${parts.map((p) => `${p}/`).join("")}`; + return { target: { server, topic, token: env.COLD_NTFY_TOKEN?.trim() || undefined } }; +} + +/** Publish one push, as JSON to the server's root. Throws with ntfy's answer when it refuses. */ +export async function sendNtfy(target: NtfyTarget, message: Omit, fetchImpl: typeof fetch = fetch): Promise { + const body: NtfyMessage = { ...message, topic: target.topic }; + const response = await fetchImpl(target.server, { + method: "POST", + headers: { "Content-Type": "application/json", ...(target.token ? { Authorization: `Bearer ${target.token}` } : {}) }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(10_000), + }); + if (response.ok) return; + const detail = (await response.text().catch(() => "")).trim().slice(0, 200); + const hint = response.status === 401 || response.status === 403 ? " (check COLD_NTFY_TOKEN)" : ""; + throw new Error(`ntfy answered ${response.status}${detail ? `: ${detail}` : ""}${hint}`); +} diff --git a/src/domains/leads/helpers/quizLead.ts b/src/domains/leads/helpers/quizLead.ts new file mode 100644 index 0000000..3924e76 --- /dev/null +++ b/src/domains/leads/helpers/quizLead.ts @@ -0,0 +1,75 @@ +import { leadNote, type Prospect } from "../../prospects"; +import type { Touch } from "../../touches"; +import type { NtfyMessage, QuizLead } from "../types"; + +const text = (v: unknown) => (typeof v === "string" && v.trim() ? v.trim() : undefined); + +/** One quiz lead for the Calls page: its record in Twenty, as a prospect, and when it was last called. Pure. */ +export function quizLead(record: Record, prospect: Prospect, calledAt?: string): QuizLead { + const v = prospect.variables; + const qualification = (text(record.qualificationStatus) ?? "").toUpperCase(); + const status = (text(record.status) ?? "").toUpperCase(); + return { + id: text(record.id) ?? prospect.key, + prospectKey: prospect.key, + label: prospect.label, + name: v.full_name, + company: v.company, + phone: prospect.phone, + email: prospect.email, + createdAt: text(record.createdAt), + qualified: qualification !== "DISQUALIFIED" && status !== "LOST", + answers: Object.entries(leadNote(text(record.note)).answers).map(([q, a]) => `${q}: ${a}`), + from: v.utm_source ?? v.lead_source, + calledAt, + doNotContact: prospect.doNotContact, + }; +} + +/** The latest call to this prospect in the touch log, by any of their identities. Pure. */ +export function lastCall(prospect: Prospect, calls: Touch[]): string | undefined { + const ids = new Set(prospect.identities); + let latest: string | undefined; + for (const t of calls) if (t.channel === "CALL" && ids.has(t.prospectKey) && (!latest || t.occurredAt > latest)) latest = t.occurredAt; + return latest; +} + +/** + * Who to call first: qualified leads nobody has called yet, newest first; + * then the ones already called; then the disqualified; Do Not Contact last. + * Pure. + */ +export function rankLeads(leads: QuizLead[]): QuizLead[] { + const group = (l: QuizLead) => (l.doNotContact ? 3 : !l.qualified ? 2 : l.calledAt ? 1 : 0); + return [...leads].sort((a, b) => group(a) - group(b) || (b.createdAt ?? "").localeCompare(a.createdAt ?? "")); +} + +/** The push for one new lead: who, their company, the number to call, the first answers. Tapping it opens the lead on the Calls page. Pure. */ +export function leadPush(lead: QuizLead, appUrl?: string): Omit { + const who = lead.name ?? lead.label; + const lines = [ + lead.company && lead.company !== who ? lead.company : undefined, + lead.phone ? `Call ${lead.phone}` : lead.email ? `No phone given: ${lead.email}` : undefined, + ...lead.answers.slice(0, 4), + ].filter((l): l is string => !!l); + return { + title: `New quiz lead: ${who}`.slice(0, 120), + message: (lines.join("\n") || "They took your quiz.").slice(0, 1000), + tags: ["telephone_receiver"], + priority: 4, + click: appUrl ? `${appUrl}calls?lead=${encodeURIComponent(lead.id)}` : undefined, + }; +} + +/** One push for many new leads at once: how many, the first names. Pure. */ +export function countPush(leads: QuizLead[], appUrl?: string, atLeast = false): Omit { + const names = leads.slice(0, 5).map((l) => l.name ?? l.label); + const more = leads.length - names.length; + return { + title: `${leads.length}${atLeast ? "+" : ""} new quiz leads`, + message: `${names.join(", ")}${more > 0 ? ` and ${more} more` : ""}. Call them first: they're at the top of Calls.`.slice(0, 1000), + tags: ["telephone_receiver"], + priority: 4, + click: appUrl ? `${appUrl}calls` : undefined, + }; +} diff --git a/src/domains/leads/index.ts b/src/domains/leads/index.ts new file mode 100644 index 0000000..a3cd814 --- /dev/null +++ b/src/domains/leads/index.ts @@ -0,0 +1,124 @@ +import type { Db } from "../db"; +import { loadAgencyLeads } from "../prospects"; +import type { Touch } from "../touches"; +import { filterValue, type TwentyClient } from "../twenty"; +import { countPush, lastCall, leadPush, quizLead, rankLeads, readNtfy, sendNtfy } from "./helpers"; +import { LEAD_CHECK_MS, LEAD_WINDOW_DAYS, MAX_LEAD_PUSHES, MAX_LEADS_PER_CHECK, type LeadCheck, type NtfyTarget, type QuizLead } from "./types"; + +export * from "./types"; +export * from "./helpers"; + +/** When the last look ran, and the newest lead it has seen (meta keys). */ +const LOOKED_KEY = "leads_looked_at"; +const SEEN_KEY = "leads_seen_until"; +/** A look counts as recent for a little less than a minute, so a cron that fires a moment early still looks. */ +const CLAIM_MS = LEAD_CHECK_MS - 5000; + +export interface LeadWatchOptions { + db: Db; + /** Twenty, read on each look so keys saved in Settings apply. */ + client: () => TwentyClient | undefined; + /** Where pushes go; COLD_NTFY_URL by default. */ + target?: () => NtfyTarget | undefined; + /** The app's address, for the link a push opens: "https://cold.example.com/app/". */ + appUrl?: string; + fetch?: typeof fetch; + now?: () => Date; +} + +/** + * Watches offer's quiz leads and pushes one alert per new qualified lead, so + * they get a call while the quiz is fresh. It looks at most once a minute + * across every server on the database (the look is claimed in it), and only + * at leads newer than the last one it saw. The first look only marks the + * time: leads from before alerts were set up are never pushed. + */ +export class LeadWatch { + private nextAt = 0; + + constructor(private readonly options: LeadWatchOptions) {} + + private now(): Date { + return this.options.now?.() ?? new Date(); + } + + async tick(): Promise { + const skipped: LeadCheck = { looked: false, alerted: 0, found: 0 }; + const target = (this.options.target ?? (() => readNtfy().target))(); + const client = target ? this.options.client() : undefined; + if (!target || !client) return skipped; + const now = this.now(); + if (now.getTime() < this.nextAt) return skipped; + this.nextAt = now.getTime() + CLAIM_MS; + if (!(await this.claim(now))) return skipped; + + const seen = await this.options.db.get<{ value: string }>("SELECT value FROM meta WHERE key = ?", SEEN_KEY); + if (!seen) { + await this.setSeen(now.toISOString()); + return { looked: true, alerted: 0, found: 0 }; + } + let alerted = 0; + let found = 0; + try { + const rows = await loadAgencyLeads(client, { where: `createdAt[gt]:${filterValue(seen.value)}`, limit: MAX_LEADS_PER_CHECK }); + const leads = rows + .map(({ record, prospect }) => quizLead(record, prospect)) + .filter((l) => l.createdAt && l.createdAt > seen.value) + .sort((a, b) => a.createdAt!.localeCompare(b.createdAt!)); + found = leads.length; + const due = leads.filter((l) => l.qualified && !l.doNotContact); + const newest = leads.at(-1)?.createdAt; + const lots = rows.length >= MAX_LEADS_PER_CHECK; + if (due.length > MAX_LEAD_PUSHES || (lots && due.length)) { + await sendNtfy(target, countPush(due, this.options.appUrl, lots), this.options.fetch); + alerted = 1; + if (newest) await this.setSeen(newest); + return { looked: true, alerted, found }; + } + for (const [i, lead] of leads.entries()) { + if (lead.qualified && !lead.doNotContact) { + await sendNtfy(target, leadPush(lead, this.options.appUrl), this.options.fetch); + alerted++; + } + // Leads that came in the same millisecond are marked seen together, so a failed push is retried, never skipped. + if (leads[i + 1]?.createdAt !== lead.createdAt) await this.setSeen(lead.createdAt!); + } + return { looked: true, alerted, found }; + } catch (err) { + return { looked: true, alerted, found, error: (err as Error).message }; + } + } + + /** Claim this minute's look in the database; false when another server (or this one) looked under a minute ago. */ + private async claim(now: Date): Promise { + const { changes } = await this.options.db.run( + "INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value WHERE meta.value <= ?", + LOOKED_KEY, + now.toISOString(), + new Date(now.getTime() - CLAIM_MS).toISOString(), + ); + return changes > 0; + } + + private async setSeen(at: string): Promise { + await this.options.db.run("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value", SEEN_KEY, at); + } + + /** When the watch last looked, and the newest lead it has seen. */ + async state(): Promise<{ lookedAt?: string; seenUntil?: string }> { + const rows = await this.options.db.all<{ key: string; value: string }>("SELECT key, value FROM meta WHERE key IN (?, ?)", LOOKED_KEY, SEEN_KEY); + const value = (key: string) => rows.find((r) => r.key === key)?.value; + return { lookedAt: value(LOOKED_KEY), seenUntil: value(SEEN_KEY) }; + } +} + +/** + * The quiz leads of the last days, best to call first, each with when it was + * last called. calls gives the touch log's calls since a time. + */ +export async function recentLeads(client: TwentyClient, options: { now: Date; days?: number; calls?: (since: string) => Promise }): Promise { + const since = new Date(options.now.getTime() - (options.days ?? LEAD_WINDOW_DAYS) * 86_400_000).toISOString(); + const rows = await loadAgencyLeads(client, { where: `createdAt[gte]:${filterValue(since)}`, limit: 200 }); + const calls = (await options.calls?.(since)) ?? []; + return rankLeads(rows.map(({ record, prospect }) => quizLead(record, prospect, lastCall(prospect, calls)))); +} diff --git a/src/domains/leads/leads.test.ts b/src/domains/leads/leads.test.ts new file mode 100644 index 0000000..06fe230 --- /dev/null +++ b/src/domains/leads/leads.test.ts @@ -0,0 +1,252 @@ +import { beforeEach, describe, expect, test } from "bun:test"; +import { openDatabase, type Db } from "../db"; +import { fromAgencyLead } from "../prospects"; +import type { Touch } from "../touches"; +import { TwentyClient } from "../twenty"; +import { countPush, lastCall, LeadWatch, leadPush, quizLead, rankLeads, readNtfy, recentLeads, sendNtfy, type NtfyMessage, type QuizLead } from "."; + +const note = (name: string, phone = "512 555 0147") => `Name: ${name}\nEmail: —\nPhone: ${phone}\n\nQuiz answers: ${JSON.stringify({ "Cars a week": "10-20" }, null, 2)}\n\nUTM source: facebook`; + +/** A fake Twenty holding agencyLeads and agencyProspects, answering the filters cold sends. */ +function fakeTwenty() { + const state = { leads: [] as Array>, prospects: [] as Array>, down: false }; + const fetchImpl = (async (input: string | URL | Request) => { + if (state.down) return new Response(JSON.stringify({ messages: ["Service unavailable"] }), { status: 503 }); + const url = new URL(String(input)); + const filter = url.searchParams.get("filter") ?? ""; + const after = /createdAt\[(gt|gte)\]:"([^"]+)"/.exec(filter); + const ids = /id\[in\]:\[([^\]]*)\]/.exec(filter)?.[1]?.split(",").map((s) => s.replace(/"/g, "")); + const rows = (url.pathname === "/rest/agencyLeads" ? state.leads : state.prospects).filter((r) => { + if (ids && !ids.includes(r.id as string)) return false; + if (after) return after[1] === "gt" ? (r.createdAt as string) > after[2]! : (r.createdAt as string) >= after[2]!; + return true; + }); + return Response.json({ data: { [url.pathname.slice(6)]: rows } }); + }) as typeof fetch; + return { state, client: new TwentyClient({ baseUrl: "https://crm.example", apiKey: "k" }, fetchImpl) }; +} + +/** A fake ntfy server that keeps what it was sent, and can refuse. */ +function fakeNtfy() { + const state = { sent: [] as Array<{ url: string; auth: string | null; body: NtfyMessage }>, status: 200 }; + const fetchImpl = (async (input: string | URL | Request, init?: RequestInit) => { + if (state.status !== 200) return new Response('{"code":50001,"http":500,"error":"internal"}', { status: state.status }); + state.sent.push({ url: String(input), auth: new Headers(init?.headers).get("authorization"), body: JSON.parse(String(init?.body)) }); + return Response.json({ id: "m1" }); + }) as typeof fetch; + return { state, fetchImpl }; +} + +const TARGET = { server: "https://ntfy.sh/", topic: "cold-leads-7f3k2q" }; + +describe("readNtfy", () => { + test("the topic's address, on ntfy.sh or a server under a path, with its token", () => { + expect(readNtfy({})).toEqual({}); + expect(readNtfy({ COLD_NTFY_URL: " https://ntfy.sh/cold-leads-7f3k2q " })).toEqual({ target: { server: "https://ntfy.sh/", topic: "cold-leads-7f3k2q", token: undefined } }); + expect(readNtfy({ COLD_NTFY_URL: "https://push.example.com/ntfy/cold_leads/", COLD_NTFY_TOKEN: "tk_abc" })).toEqual({ target: { server: "https://push.example.com/ntfy/", topic: "cold_leads", token: "tk_abc" } }); + }); + + test("an unusable address says what is wrong", () => { + expect(readNtfy({ COLD_NTFY_URL: "ntfy.sh/topic" }).problem).toMatch(/not a URL/); + expect(readNtfy({ COLD_NTFY_URL: "ftp://ntfy.sh/topic" }).problem).toMatch(/https/); + expect(readNtfy({ COLD_NTFY_URL: "https://me:pw@ntfy.sh/topic" }).problem).toMatch(/COLD_NTFY_TOKEN/); + expect(readNtfy({ COLD_NTFY_URL: "https://ntfy.sh/" }).problem).toMatch(/topic's name/); + expect(readNtfy({ COLD_NTFY_URL: "https://ntfy.sh/a%20b" }).problem).toMatch(/topic's name/); + expect(readNtfy({ COLD_NTFY_URL: "https://ntfy.sh/a%20b" }).target).toBeUndefined(); + }); +}); + +describe("sendNtfy", () => { + test("publishes JSON to the server's root, with the topic and the token", async () => { + const ntfy = fakeNtfy(); + await sendNtfy({ ...TARGET, token: "tk_abc" }, { title: "t", message: "m", tags: [], priority: 4 }, ntfy.fetchImpl); + expect(ntfy.state.sent).toEqual([{ url: "https://ntfy.sh/", auth: "Bearer tk_abc", body: { title: "t", message: "m", tags: [], priority: 4, topic: "cold-leads-7f3k2q" } }]); + }); + + test("a refusal throws with ntfy's answer, and a token hint for 401 and 403", async () => { + const ntfy = fakeNtfy(); + ntfy.state.status = 403; + await expect(sendNtfy(TARGET, { title: "t", message: "m", tags: [], priority: 4 }, ntfy.fetchImpl)).rejects.toThrow(/ntfy answered 403.*check COLD_NTFY_TOKEN/); + }); +}); + +function lead(over: Partial = {}): QuizLead { + return { id: "lead-1", prospectKey: "agencyLeads:lead-1", label: "Sam Lee", name: "Sam Lee", phone: "512 555 0147", createdAt: "2026-11-01T16:20:00.000Z", qualified: true, answers: ["Cars a week: 10-20"], doNotContact: false, ...over }; +} + +describe("quiz leads", () => { + test("a lead from its record: qualified unless offer said otherwise, answers as lines", () => { + const record = { id: "lead-1", contactName: "Sam Lee", note: note("Sam Lee"), qualificationStatus: "QUALIFIED", createdAt: "2026-11-01T16:20:00.000Z" }; + const q = quizLead(record, fromAgencyLead(record)); + expect(q).toMatchObject({ id: "lead-1", label: "Sam Lee", phone: "512 555 0147", qualified: true, answers: ["Cars a week: 10-20"], from: "facebook" }); + expect(quizLead({ ...record, qualificationStatus: "DISQUALIFIED" }, fromAgencyLead(record)).qualified).toBe(false); + expect(quizLead({ ...record, qualificationStatus: undefined, status: "LOST" }, fromAgencyLead(record)).qualified).toBe(false); + expect(quizLead({ ...record, qualificationStatus: undefined }, fromAgencyLead(record)).qualified).toBe(true); + }); + + test("called, by any of the prospect's identities, the latest call", () => { + const record = { id: "lead-1", contactName: "Sam Lee", note: note("Sam Lee"), createdAt: "2026-11-01T16:20:00.000Z" }; + const prospect = fromAgencyLead(record); + const call = (prospectKey: string, occurredAt: string): Touch => ({ kind: "CALLED", channel: "CALL", prospectKey, occurredAt }); + expect(lastCall(prospect, [call("agencyLeads:lead-1", "2026-11-01T17:00:00Z"), call("agencyLeads:lead-1", "2026-11-01T18:00:00Z"), call("email:other@x.com", "2026-11-01T19:00:00Z")])).toBe("2026-11-01T18:00:00Z"); + expect(lastCall(prospect, [{ ...call("agencyLeads:lead-1", "2026-11-01T17:00:00Z"), channel: "EMAIL", kind: "SENT" }])).toBeUndefined(); + }); + + test("who to call first: uncalled qualified leads, newest first, then called, disqualified, Do Not Contact", () => { + const order = rankLeads([ + lead({ id: "dnc", doNotContact: true, createdAt: "2026-11-01T20:00:00Z" }), + lead({ id: "dq", qualified: false, createdAt: "2026-11-01T19:00:00Z" }), + lead({ id: "called", calledAt: "2026-11-01T18:30:00Z", createdAt: "2026-11-01T18:00:00Z" }), + lead({ id: "older", createdAt: "2026-11-01T10:00:00Z" }), + lead({ id: "newer", createdAt: "2026-11-01T12:00:00Z" }), + ]).map((l) => l.id); + expect(order).toEqual(["newer", "older", "called", "dq", "dnc"]); + }); + + test("a push names the lead, the company, the number, their first answers, and opens the lead", () => { + const push = leadPush(lead({ company: "Clearview Tint", label: "Sam Lee (Clearview Tint)" }), "https://cold.example.com/app/"); + expect(push.title).toBe("New quiz lead: Sam Lee"); + expect(push.message).toBe("Clearview Tint\nCall 512 555 0147\nCars a week: 10-20"); + expect(push.click).toBe("https://cold.example.com/app/calls?lead=lead-1"); + expect(push.priority).toBe(4); + expect(leadPush(lead({ phone: undefined, email: "sam@example.com", answers: [] })).message).toBe("No phone given: sam@example.com"); + expect(leadPush(lead({ phone: undefined, answers: [] })).message).toBe("They took your quiz."); + }); + + test("many leads at once come as one push", () => { + const leads = Array.from({ length: 7 }, (_, i) => lead({ id: `l${i}`, name: `Lead ${i}` })); + const push = countPush(leads, "https://cold.example.com/app/"); + expect(push.title).toBe("7 new quiz leads"); + expect(push.message).toBe("Lead 0, Lead 1, Lead 2, Lead 3, Lead 4 and 2 more. Call them first: they're at the top of Calls."); + expect(push.click).toBe("https://cold.example.com/app/calls"); + expect(countPush(leads, undefined, true).title).toBe("7+ new quiz leads"); + }); +}); + +describe("LeadWatch", () => { + let db: Db; + let clock: Date; + let twenty: ReturnType; + let ntfy: ReturnType; + const watch = (target: typeof TARGET | null = TARGET) => new LeadWatch({ db, client: () => twenty.client, target: () => target ?? undefined, appUrl: "https://cold.example.com/app/", fetch: ntfy.fetchImpl, now: () => clock }); + const later = (seconds: number) => (clock = new Date(clock.getTime() + seconds * 1000)); + const add = (id: string, over: Record = {}) => + twenty.state.leads.push({ id, contactName: `Lead ${id}`, note: note(`Lead ${id}`), qualificationStatus: "QUALIFIED", createdAt: new Date(clock.getTime() - 1000).toISOString(), ...over }); + + beforeEach(async () => { + db = await openDatabase(":memory:"); + clock = new Date("2026-11-01T16:00:00.000Z"); + twenty = fakeTwenty(); + ntfy = fakeNtfy(); + }); + + test("does nothing until alerts are set up", async () => { + add("old"); + expect(await watch(null).tick()).toEqual({ looked: false, alerted: 0, found: 0 }); + expect(await watch(null).state()).toEqual({ lookedAt: undefined, seenUntil: undefined }); + }); + + test("the first look only marks the time: leads from before are never pushed", async () => { + add("old"); + const w = watch(); + expect(await w.tick()).toEqual({ looked: true, alerted: 0, found: 0 }); + expect(ntfy.state.sent).toHaveLength(0); + expect((await w.state()).seenUntil).toBe("2026-11-01T16:00:00.000Z"); + }); + + test("pushes each new qualified lead once, never a disqualified or Do Not Contact one", async () => { + const w = watch(); + await w.tick(); + later(90); + add("sam"); + add("dq", { qualificationStatus: "DISQUALIFIED" }); + twenty.state.prospects.push({ id: "ap-9", name: "Shine Detail", coldCallStatus: "DO_NOT_CONTACT" }); + add("dnc", { agencyProspectId: "ap-9" }); + expect(await w.tick()).toEqual({ looked: true, alerted: 1, found: 3 }); + expect(ntfy.state.sent.map((s) => s.body.title)).toEqual(["New quiz lead: Lead sam"]); + expect(ntfy.state.sent[0]!.body.topic).toBe("cold-leads-7f3k2q"); + later(90); + expect(await w.tick()).toEqual({ looked: true, alerted: 0, found: 0 }); + expect(ntfy.state.sent).toHaveLength(1); + }); + + test("looks at most once a minute, across every server on the database", async () => { + const one = watch(); + const two = watch(); + await one.tick(); + later(90); + add("sam"); + later(1); + expect((await one.tick()).alerted).toBe(1); + expect(await two.tick()).toEqual({ looked: false, alerted: 0, found: 0 }); + later(30); + expect((await one.tick()).looked).toBe(false); + expect((await two.tick()).looked).toBe(false); + later(31); + expect((await two.tick()).looked).toBe(true); + expect(ntfy.state.sent).toHaveLength(1); + }); + + test("a push that fails is tried again on the next look, never skipped", async () => { + const w = watch(); + await w.tick(); + later(90); + add("sam"); + ntfy.state.status = 500; + const failed = await w.tick(); + expect(failed.alerted).toBe(0); + expect(failed.error).toMatch(/ntfy answered 500/); + ntfy.state.status = 200; + later(60); + expect((await w.tick()).alerted).toBe(1); + expect(ntfy.state.sent.map((s) => s.body.title)).toEqual(["New quiz lead: Lead sam"]); + }); + + test("Twenty being down is said, and the leads are pushed once it is back", async () => { + const w = watch(); + await w.tick(); + later(90); + add("sam"); + twenty.state.down = true; + expect((await w.tick()).error).toMatch(/503/); + twenty.state.down = false; + later(60); + expect((await w.tick()).alerted).toBe(1); + }); + + test("more than five new leads at once come as one push", async () => { + const w = watch(); + await w.tick(); + later(90); + for (const id of ["a", "b", "c", "d", "e", "f"]) add(id); + expect(await w.tick()).toEqual({ looked: true, alerted: 1, found: 6 }); + expect(ntfy.state.sent[0]!.body.title).toBe("6 new quiz leads"); + later(60); + expect((await w.tick()).alerted).toBe(0); + }); +}); + +describe("recentLeads", () => { + test("the last days' leads, best to call first, with when each was called", async () => { + const twenty = fakeTwenty(); + const now = new Date("2026-11-01T18:00:00.000Z"); + twenty.state.leads.push( + { id: "old", contactName: "Old Lead", note: note("Old Lead"), createdAt: "2026-10-01T10:00:00.000Z" }, + { id: "sam", contactName: "Sam Lee", note: note("Sam Lee"), createdAt: "2026-11-01T16:00:00.000Z" }, + { id: "ana", contactName: "Ana Ruiz", note: note("Ana Ruiz"), createdAt: "2026-11-01T17:00:00.000Z" }, + ); + let since = ""; + const leads = await recentLeads(twenty.client, { + now, + calls: async (from) => { + since = from; + return [{ kind: "CALLED", channel: "CALL", prospectKey: "agencyLeads:ana", occurredAt: "2026-11-01T17:30:00Z" }]; + }, + }); + expect(since).toBe("2026-10-18T18:00:00.000Z"); + expect(leads.map((l) => [l.id, l.calledAt])).toEqual([ + ["sam", undefined], + ["ana", "2026-11-01T17:30:00Z"], + ]); + }); +}); diff --git a/src/domains/leads/types.ts b/src/domains/leads/types.ts new file mode 100644 index 0000000..998b73e --- /dev/null +++ b/src/domains/leads/types.ts @@ -0,0 +1,66 @@ +/** Where new-lead alerts go: a topic on ntfy.sh or on your own ntfy server. */ +export interface NtfyTarget { + /** The server's root, where messages are published as JSON: "https://ntfy.sh/". */ + server: string; + topic: string; + /** An access token, for a topic that needs one. */ + token?: string; +} + +/** One push, in ntfy's JSON publish shape. */ +export interface NtfyMessage { + topic: string; + title: string; + message: string; + tags: string[]; + /** 1 (min) to 5 (max); 3 is ntfy's default. */ + priority: number; + /** Opened when the notification is tapped. */ + click?: string; +} + +/** One of offer's quiz leads, as the Calls page lists it. */ +export interface QuizLead { + /** The agencyLead's id in Twenty. */ + id: string; + /** The prospect key, to pick them in a call script (source twenty:agencyLeads). */ + prospectKey: string; + label: string; + name?: string; + company?: string; + phone?: string; + email?: string; + /** When they took the quiz. */ + createdAt?: string; + /** offer marked them qualified (or didn't say otherwise). */ + qualified: boolean; + /** "question: answer", one each. */ + answers: string[]; + /** Where they came from: offer's source, or a UTM source. */ + from?: string; + /** When they were last called, from the touch log. */ + calledAt?: string; + doNotContact: boolean; +} + +/** What one look for new leads did. */ +export interface LeadCheck { + /** False when it was skipped: alerts aren't set up, or another look ran within the minute. */ + looked: boolean; + /** Pushes sent. */ + alerted: number; + /** Leads that came in, qualified or not. */ + found: number; + error?: string; +} + +/** The source the quiz leads load from. */ +export const LEADS_SOURCE = "twenty:agencyLeads"; +/** A look for new leads at most this often, across every server on the database. */ +export const LEAD_CHECK_MS = 60_000; +/** The Calls page lists the leads of this many days. */ +export const LEAD_WINDOW_DAYS = 14; +/** More new leads than this in one look come as one push that counts them. */ +export const MAX_LEAD_PUSHES = 5; +/** A look reads at most this many new leads. */ +export const MAX_LEADS_PER_CHECK = 500; diff --git a/src/domains/prospects/helpers/agencyLead.test.ts b/src/domains/prospects/helpers/agencyLead.test.ts new file mode 100644 index 0000000..10c161b --- /dev/null +++ b/src/domains/prospects/helpers/agencyLead.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, test } from "bun:test"; +import { TwentyClient } from "../../twenty"; +import { findProspect, loadProspects } from ".."; +import { fromAgencyLead, leadNote, leadTrigger } from "./agencyLead"; + +/** A note exactly as offer writes it (backend/src/routes/leads.ts). */ +function offerNote(contact: { name?: string; email?: string; phone?: string }, answers: Record | string = {}): string { + const contactNote = [`Name: ${contact.name || "—"}`, `Email: ${contact.email || "—"}`, `Phone: ${contact.phone || "—"}`].join("\n"); + const parts = [ + typeof answers === "string" ? answers : `Quiz answers: ${JSON.stringify(answers, null, 2)}`, + `Quiz detail: ${JSON.stringify({ score: 7, steps: [{ id: "q1", dq: false }] }, null, 2)}`, + "OfferId: offer-9", + "Source URL: https://offer.example/q/clearview?utm_source=facebook", + "visitor_id: v-1", + "UTM source: facebook", + "UTM medium: paid", + "UTM campaign: fall-tint", + ]; + return [contactNote, parts.join("\n\n")].join("\n\n"); +} + +const ANSWERS = { "Cars a week": "10-20", "Biggest problem": "No-shows", Services: ["Tint", "PPF"] }; + +describe("leadNote", () => { + test("reads offer's contact lines, the quiz answers and the tagged lines, not the quiz detail", () => { + const note = leadNote(offerNote({ name: "Sam Lee", email: "Sam@Example.com", phone: "(512) 555-0147" }, ANSWERS)); + expect(note.name).toBe("Sam Lee"); + expect(note.email).toBe("sam@example.com"); + expect(note.phone).toBe("(512) 555-0147"); + expect(note.answers).toEqual({ "Cars a week": "10-20", "Biggest problem": "No-shows", Services: "Tint, PPF" }); + expect(note.fields).toMatchObject({ offerid: "offer-9", source_url: "https://offer.example/q/clearview?utm_source=facebook", utm_source: "facebook", utm_medium: "paid", utm_campaign: "fall-tint", visitor_id: "v-1" }); + expect(Object.keys(note.fields)).not.toContain("score"); + }); + + test("an empty contact line is offer's dash, read as nothing", () => { + const note = leadNote(offerNote({}, { q: "a" })); + expect(note.name).toBeUndefined(); + expect(note.email).toBeUndefined(); + expect(note.phone).toBeUndefined(); + }); + + test("answers cut off by offer's 9000-character limit are dropped, and the other lines still read", () => { + const note = leadNote(offerNote({ name: "Sam Lee" }, 'Quiz answers: {\n "Cars a week": "10-2')); + expect(note.answers).toEqual({}); + expect(note.name).toBe("Sam Lee"); + expect(note.fields.utm_source).toBe("facebook"); + }); + + test("no note at all", () => { + expect(leadNote(undefined)).toEqual({ name: undefined, email: undefined, phone: undefined, answers: {}, fields: {} }); + }); +}); + +describe("fromAgencyLead", () => { + const lead = { + id: "lead-1", + name: "Sam Lee", + contactName: "Sam Lee", + note: offerNote({ name: "Sam Lee", email: "sam@example.com", phone: "(512) 555-0147" }, ANSWERS), + source: "quiz", + qualificationStatus: "QUALIFIED", + status: "QUALIFIED", + createdAt: "2026-11-01T16:20:00.000Z", + }; + + test("a lead on its own: who took the quiz, their answers, where they came from", () => { + const p = fromAgencyLead(lead); + expect(p.key).toBe("email:sam@example.com"); + expect(p.identities).toContain("agencyLeads:lead-1"); + expect(p.label).toBe("Sam Lee"); + expect(p.phone).toBe("(512) 555-0147"); + expect(p.variables).toMatchObject({ + first_name: "Sam", + full_name: "Sam Lee", + lead_id: "lead-1", + qualification: "QUALIFIED", + utm_source: "facebook", + offer_id: "offer-9", + quiz_answers: "Cars a week: 10-20; Biggest problem: No-shows; Services: Tint, PPF", + trigger: "Saw you took our quiz on Nov 1.", + }); + expect(p.links).toEqual({ agencyLeadId: "lead-1" }); + expect(p.source).toBe("agencyLeads"); + }); + + test("through a business's own funnel link: the business's details, the quiz taker's name, one history with the business", () => { + const business = { id: "ap-1", name: "Clearview Tint", decisionMaker: "Dana Reyes", decisionMakerTitle: "Owner", city: "Austin", coldTrigger: "Saw your 212 reviews.", coldCallStatus: "DO_NOT_CONTACT" }; + const p = fromAgencyLead({ ...lead, agencyProspectId: "ap-1" }, business); + expect(p.key).toBe("agencyProspect:ap-1"); + expect(p.label).toBe("Sam Lee (Clearview Tint)"); + expect(p.variables).toMatchObject({ company: "Clearview Tint", city: "Austin", first_name: "Sam", trigger: "Saw your 212 reviews." }); + expect(p.variables.title).toBeUndefined(); + expect(p.links).toEqual({ agencyProspectId: "ap-1", agencyLeadId: "lead-1" }); + expect(p.doNotContact).toBe(true); + }); + + test("a lead who gave no name is never called by the business's decision maker's", () => { + const p = fromAgencyLead({ ...lead, contactName: "", note: offerNote({ phone: "512 555 0100" }) }, { id: "ap-1", name: "Clearview Tint", decisionMaker: "Dana Reyes" }); + expect(p.variables.first_name).toBeUndefined(); + expect(p.label).toBe("Clearview Tint"); + }); + + test("the trigger is the day they took the quiz, and nothing without a date", () => { + expect(leadTrigger("2026-11-02T23:30:00Z")).toBe("Saw you took our quiz on Nov 2."); + expect(leadTrigger("not a date")).toBeUndefined(); + expect(leadTrigger(undefined)).toBeUndefined(); + }); +}); + +describe("loadProspects twenty:agencyLeads", () => { + test("reads the leads and only the businesses they link to", async () => { + const asked: string[] = []; + const fetchImpl = (async (input: string | URL | Request) => { + const url = new URL(String(input)); + asked.push(`${url.pathname} ${url.searchParams.get("filter") ?? ""}`); + if (url.pathname === "/rest/agencyLeads") + return Response.json({ + data: { + agencyLeads: [ + { id: "lead-1", contactName: "Sam Lee", note: offerNote({ name: "Sam Lee", phone: "512 555 0147" }), agencyProspectId: "ap-1", createdAt: "2026-11-01T16:20:00.000Z" }, + { id: "lead-2", contactName: "Ana Ruiz", note: offerNote({ name: "Ana Ruiz", email: "ana@example.com" }), agencyProspectId: null, createdAt: "2026-11-01T17:00:00.000Z" }, + ], + }, + }); + if (url.pathname === "/rest/agencyProspects") return Response.json({ data: { agencyProspects: [{ id: "ap-1", name: "Clearview Tint" }] } }); + return Response.json({ data: {} }); + }) as typeof fetch; + const client = new TwentyClient({ baseUrl: "https://crm.example", apiKey: "k" }, fetchImpl); + const prospects = await loadProspects("twenty:agencyLeads", { client, where: 'createdAt[gte]:"2026-11-01T00:00:00.000Z"' }); + expect(prospects.map((p) => p.label)).toEqual(["Sam Lee (Clearview Tint)", "Ana Ruiz"]); + expect(asked[0]).toBe('/rest/agencyLeads createdAt[gte]:"2026-11-01T00:00:00.000Z"'); + expect(asked[1]).toBe('/rest/agencyProspects id[in]:["ap-1"]'); + expect(asked).toHaveLength(2); + }); +}); + +describe("findProspect", () => { + test("finds a Twenty prospect by its key as the pickers send it, whatever the case", () => { + const sam = fromAgencyLead({ id: "lead-1", contactName: "Sam Lee", agencyProspectId: "ap-1" }, { id: "ap-1", name: "Clearview Tint" }); + const ana = fromAgencyLead({ id: "lead-2", contactName: "Ana Ruiz", note: "Email: Ana@Example.com" }); + const all = [ana, sam]; + expect(findProspect(all, "agencyProspect:ap-1")).toBe(sam); + expect(findProspect(all, "agencyprospect:AP-1")).toBe(sam); + expect(findProspect(all, "agencyLeads:lead-2")).toBe(ana); + expect(findProspect(all, "ana@example.com")).toBe(ana); + expect(findProspect(all, "clearview")).toBe(sam); + expect(findProspect(all, "nobody")).toBeUndefined(); + }); +}); diff --git a/src/domains/prospects/helpers/agencyLead.ts b/src/domains/prospects/helpers/agencyLead.ts new file mode 100644 index 0000000..a30971f --- /dev/null +++ b/src/domains/prospects/helpers/agencyLead.ts @@ -0,0 +1,97 @@ +import type { Prospect } from "../types"; +import { normalizeEmail } from "./normalize"; +import { finishProspect, fromAgencyProspect } from "./toProspect"; + +type Rec = Record; +const str = (v: unknown) => (typeof v === "string" && v.trim() && v.trim() !== "—" ? v.trim() : undefined); + +/** One quiz answer as words: a pick, several picks ("a, b"), or a number. */ +function answerText(value: unknown): string | undefined { + if (Array.isArray(value)) return str(value.map((v) => answerText(v)).filter(Boolean).join(", ")); + if (typeof value === "number" || typeof value === "boolean") return String(value); + if (value && typeof value === "object") return str(JSON.stringify(value)); + return str(value); +} + +/** What offer wrote into a lead's note: the contact lines, the quiz answers and the other tagged lines (UTM, source URL, offer). */ +export interface LeadNote { + name?: string; + email?: string; + phone?: string; + answers: Record; + fields: Record; +} + +/** + * offer's agencyLead note, read back (backend/src/routes/leads.ts writes it): + * "Name: …", "Email: …", "Phone: …" ("—" when empty), "Quiz answers: {json}", + * then one "Label: value" line each for the offer, source URL and UTM tags. + * Pure. + */ +export function leadNote(note: string | undefined): LeadNote { + const text = note ?? ""; + const answers: Record = {}; + const json = /Quiz answers: (\{[\s\S]*?\n\})/.exec(text)?.[1]; + if (json) { + try { + for (const [k, v] of Object.entries(JSON.parse(json) as Rec)) { + const answer = answerText(v); + if (answer) answers[k] = answer; + } + } catch { + // A note edited by hand: keep the other lines. + } + } + const fields: Record = {}; + for (const line of text.split("\n")) { + const m = /^([A-Za-z_ ]{2,30}):\s*(.+)$/.exec(line.trim()); + if (m && !m[2]!.startsWith("{") && str(m[2])) fields[m[1]!.trim().toLowerCase().replace(/\s+/g, "_")] = m[2]!.trim(); + } + return { name: fields.name, email: normalizeEmail(fields.email), phone: fields.phone, answers, fields }; +} + +/** "Saw you took our quiz on Nov 2." from when the lead came in. Pure. */ +export function leadTrigger(createdAt: string | undefined): string | undefined { + const at = createdAt ? new Date(createdAt) : undefined; + if (!at || Number.isNaN(at.getTime())) return undefined; + return `Saw you took our quiz on ${at.toLocaleDateString("en-US", { month: "short", day: "numeric", timeZone: "UTC" })}.`; +} + +/** + * One of offer's quiz leads (an agencyLead in Twenty) as a prospect: who they + * are from the note, their quiz answers and UTM tags as variables, and the + * business's details when the lead came through a prospect's own funnel link + * (its agencyProspect). The person is whoever took the quiz: the business's + * decision maker is never assumed. Keyed to that prospect when there is one, + * so its calls and emails share one history. + */ +export function fromAgencyLead(lead: Rec, prospect?: Rec): Prospect { + const note = leadNote(str(lead.note)); + const business = prospect ? fromAgencyProspect(prospect) : undefined; + const { first_name: _first, last_name: _last, full_name: _full, title: _title, ...businessVariables } = business?.variables ?? {}; + const answers = Object.entries(note.answers); + const variables: Record = { + ...businessVariables, + ...Object.fromEntries( + Object.entries({ + full_name: str(lead.contactName) ?? note.name, + email: note.email ?? business?.variables.email, + phone: note.phone ?? business?.variables.phone, + lead_id: str(lead.id), + lead_source: str(lead.source), + qualification: str(lead.qualificationStatus), + lead_status: str(lead.status), + lead_created_at: str(lead.createdAt), + offer_id: note.fields.offerid, + source_url: note.fields.source_url, + utm_source: note.fields.utm_source, + utm_medium: note.fields.utm_medium, + utm_campaign: note.fields.utm_campaign, + quiz_answers: answers.length ? answers.map(([q, a]) => `${q}: ${a}`).join("; ") : undefined, + trigger: business?.variables.trigger ?? leadTrigger(str(lead.createdAt)), + }).filter((e): e is [string, string] => !!e[1]), + ), + }; + const links = { ...(business?.links ?? {}), agencyLeadId: str(lead.id) }; + return finishProspect({ source: "agencyLeads", variables, doNotContact: business?.doNotContact ?? false, links, sourceId: str(lead.id) }); +} diff --git a/src/domains/prospects/helpers/index.ts b/src/domains/prospects/helpers/index.ts index 81e4f1c..26a8799 100644 --- a/src/domains/prospects/helpers/index.ts +++ b/src/domains/prospects/helpers/index.ts @@ -1,3 +1,4 @@ +export * from "./agencyLead"; export * from "./csvText"; export * from "./normalize"; export * from "./parseCsv"; diff --git a/src/domains/prospects/index.ts b/src/domains/prospects/index.ts index d695601..95b7f13 100644 --- a/src/domains/prospects/index.ts +++ b/src/domains/prospects/index.ts @@ -1,7 +1,7 @@ import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; import type { TwentyClient, TwentyRecord } from "../twenty"; -import { fromAgencyPerson, fromAgencyProspect, fromCsvRow, fromPerson, parseCsv } from "./helpers"; +import { fromAgencyLead, fromAgencyPerson, fromAgencyProspect, fromCsvRow, fromPerson, parseCsv } from "./helpers"; import type { Prospect } from "./types"; export * from "./types"; @@ -45,6 +45,8 @@ export async function loadProspects(source: string, options: LoadOptions = {}): } case "agencyProspects": return (await list("agencyProspects")).map(fromAgencyProspect); + case "agencyLeads": + return (await loadAgencyLeads(client, { where: options.where, limit: options.limit })).map((l) => l.prospect); case "agencyPeople": { const people = await list("agencyPeople"); const prospects = byId(await client.getMany("agencyProspects", people.map((p) => p.prospectId as string))); @@ -55,6 +57,14 @@ export async function loadProspects(source: string, options: LoadOptions = {}): } } +/** offer's quiz leads (agencyLeads), each with its record and as a prospect, the business it came through filled in. */ +export async function loadAgencyLeads(client: TwentyClient, options: { where?: string; limit?: number } = {}): Promise> { + const leads = await client.list("agencyLeads", { filter: options.where, limit: options.limit }); + const linked = leads.map((l) => l.agencyProspectId as string).filter(Boolean); + const businesses = new Map((linked.length ? await client.getMany("agencyProspects", linked) : []).map((r) => [r.id, r])); + return leads.map((record) => ({ record, prospect: fromAgencyLead(record, businesses.get(record.agencyProspectId as string)) })); +} + /** The prospects in CSV text: a header row, then one prospect per row. */ export function prospectsFromCsv(text: string, limit?: number): Prospect[] { const [headers, ...rows] = parseCsv(text); @@ -62,11 +72,18 @@ export function prospectsFromCsv(text: string, limit?: number): Prospect[] { return rows.slice(0, limit ?? rows.length).map((row, i) => fromCsvRow(headers, row, i)); } -/** Pick one prospect by key, any identity, email, or a case-insensitive name/company match. */ +/** + * Pick one prospect: by its exact key, then by any identity, email or id in + * any case ("agencyProspect:" and "agencyprospect:" alike), then by a + * case-insensitive name/company match. + */ export function findProspect(prospects: Prospect[], query: string): Prospect | undefined { - const q = query.trim().toLowerCase(); + const raw = query.trim(); + const q = raw.toLowerCase(); + const same = (id: string | undefined) => !!id && id.toLowerCase() === q; return ( - prospects.find((p) => p.key === q || p.identities.includes(q) || p.email === q || p.variables.id?.toLowerCase() === q) ?? + prospects.find((p) => p.key === raw) ?? + prospects.find((p) => same(p.key) || p.identities.some(same) || p.email === q || same(p.variables.id)) ?? prospects.find((p) => p.label.toLowerCase().includes(q)) ); } diff --git a/src/domains/prospects/types.ts b/src/domains/prospects/types.ts index 03fb047..9395184 100644 --- a/src/domains/prospects/types.ts +++ b/src/domains/prospects/types.ts @@ -7,6 +7,8 @@ export interface ProspectLinks { companyId?: string; agencyProspectId?: string; agencyPersonId?: string; + /** offer's quiz lead, when the prospect is one. */ + agencyLeadId?: string; } export interface Prospect { diff --git a/src/domains/web/api/index.ts b/src/domains/web/api/index.ts index 85b4e9b..497b7ce 100644 --- a/src/domains/web/api/index.ts +++ b/src/domains/web/api/index.ts @@ -114,6 +114,8 @@ function envFields(config: ColdConfig | undefined): SettingsResponse["env"] { fields.set("X_CLIENT_ID", "Optional: your X developer app's OAuth 2.0 Client ID, so X DMs go out from cold through X's API (developer.x.com). Then Connect X below"); fields.set("X_CLIENT_SECRET", "Optional: that app's OAuth 2.0 Client Secret (a Web App; leave it empty for a Native App)"); fields.set("COLD_LISTEN_SECRET", "Optional: a long random string. OpenMagpie's webhook action sends it as Authorization: Bearer , so its matches come in as prospects"); + fields.set("COLD_NTFY_URL", "Optional: an ntfy topic's address, e.g. https://ntfy.sh/cold-leads-7f3k2q, for a push to your phone when a quiz lead comes in. The push carries the lead's name and number: pick a long random topic, or your own ntfy server with a token"); + fields.set("COLD_NTFY_TOKEN", "Optional: an ntfy access token (tk_...), for a topic that needs one"); return [...fields.entries()].map(([name, hint]) => ({ name, set: !!process.env[name]?.trim(), hint })); } @@ -401,6 +403,7 @@ export function createApi(ctx: WebContext, extraRoutes: Route[] = []): (request: sources.push({ id: "twenty:people", label: "Twenty: People", kind: "twenty" }); sources.push({ id: "twenty:agencyProspects", label: "Twenty: dialer prospects (businesses)", kind: "twenty" }); sources.push({ id: "twenty:agencyPeople", label: "Twenty: people at dialer prospects", kind: "twenty" }); + sources.push({ id: "twenty:agencyLeads", label: "Twenty: quiz leads from offer", kind: "twenty" }); } const watches = (await ctx.listening?.watches()) ?? []; if (watches.length > 1) sources.push({ id: "listen:all", label: `Listening: every watch (${watches.reduce((n, w) => n + w.count, 0)} matches)`, kind: "listen" }); diff --git a/src/domains/web/context/index.ts b/src/domains/web/context/index.ts index c8d901d..2a5c044 100644 --- a/src/domains/web/context/index.ts +++ b/src/domains/web/context/index.ts @@ -11,6 +11,8 @@ import { DbFiles, DiskFiles, EnvFileSecrets, secretKey, type Files, type Secrets import { DbStore, filesJournal, openStore, twentyClient, TwentyStore, type TouchStore } from "../../touches"; import { xAppFromEnv, XClient, XStore, type XFetch } from "../../xApi"; import { LISTEN_PREFIX, ListenStore, listenProspect } from "../../listen"; +import { LeadWatch } from "../../leads"; +import type { TwentyClient } from "../../twenty"; import { CampaignStore } from "../campaigns"; import { processScoreItem } from "../decide/engine"; import { ScoreJobs } from "../decide/jobs"; @@ -34,6 +36,12 @@ export interface WebContextOptions { transport?: (account: Account) => MailTransport; /** How cold reaches X's API (tests hand it a fake X). */ xFetch?: XFetch; + /** Twenty, when not from TWENTY_BASE_URL and TWENTY_API_KEY (tests hand it a fake). */ + twenty?: () => TwentyClient | undefined; + /** How lead alerts reach ntfy (tests hand it a fake). */ + ntfyFetch?: typeof fetch; + /** The app's address, for links in lead alerts: "https://cold.example.com/app/". */ + appUrl?: string; now?: () => Date; random?: () => number; } @@ -82,6 +90,8 @@ export class WebContext { readonly x?: XStore; /** Listening matches OpenMagpie delivered; only with the database. */ readonly listening?: ListenStore; + /** Pushes an alert when one of offer's quiz leads comes in; only with the database. */ + readonly leads?: LeadWatch; private readonly prospectCache = new Map(); private storeCache?: { at: number; key: string; store: Promise }; private kept: { config?: Promise; library?: ReturnType } = {}; @@ -108,9 +118,15 @@ export class WebContext { }); this.x = new XStore(options.db, { key: usableSecretKey(), requireKey: this.storage === "database", now: () => this.now() }); this.listening = new ListenStore(options.db); + this.leads = new LeadWatch({ db: options.db, client: () => this.twenty(), appUrl: options.appUrl, fetch: options.ntfyFetch, now: () => this.now() }); } } + /** Twenty, once TWENTY_BASE_URL and TWENTY_API_KEY are set. */ + twenty(): TwentyClient | undefined { + return this.options.twenty ? this.options.twenty() : twentyClient(); + } + /** X's API through the owner's developer app, once X_CLIENT_ID is set. */ xClient(): XClient | undefined { const app = xAppFromEnv(); @@ -284,7 +300,7 @@ export class WebContext { const watch = source.slice(LISTEN_PREFIX.length); prospects = (await this.listening.items({ watch: watch === "all" ? undefined : watch, limit: 2000 })).map(listenProspect); } else { - prospects = await loadProspects(source, { where: where || undefined, client: twentyClient() }); + prospects = await loadProspects(source, { where: where || undefined, client: this.twenty() }); } this.prospectCache.set(key, { at: Date.now(), prospects }); return prospects; diff --git a/src/domains/web/leads/index.ts b/src/domains/web/leads/index.ts new file mode 100644 index 0000000..20aa4dc --- /dev/null +++ b/src/domains/web/leads/index.ts @@ -0,0 +1,66 @@ +import { readNtfy, recentLeads, sendNtfy, type LeadCheck } from "../../leads"; +import { HttpError, json, type Route } from "../api"; +import type { WebContext } from "../context"; +import type { LeadAlertsResponse, LeadsResponse } from "../types"; + +/** One line in the server's log when a look for new leads pushed something or failed. */ +export function logLeadCheck(check: LeadCheck | undefined): void { + if (!check?.looked) return; + if (check.error) console.error(`leads: ${check.error}${check.alerted ? ` (after ${check.alerted} alert${check.alerted === 1 ? "" : "s"})` : ""}`); + else if (check.alerted) console.log(`leads: ${check.found} new, ${check.alerted} alert${check.alerted === 1 ? "" : "s"} pushed`); +} + +/** Whether lead alerts are set up, without reading any leads. */ +async function alertState(ctx: WebContext): Promise { + const ntfy = readNtfy(); + const state = (await ctx.leads?.state()) ?? {}; + return { twenty: !!ctx.twenty(), alerts: { configured: !!ntfy.target, problem: ntfy.problem, available: !!ctx.leads, ...state } }; +} + +/** The Calls page's quiz leads, and Settings' lead alerts. Behind the login. */ +export function leadRoutes(ctx: WebContext): Route[] { + return [ + ["GET", /^\/api\/leads\/alerts$/, async () => json(await alertState(ctx))], + [ + "GET", + /^\/api\/leads$/, + async (_request, _match, url) => { + const response: LeadsResponse = { ...(await alertState(ctx)), leads: [] }; + const client = ctx.twenty(); + if (!client) return json(response); + const days = Math.max(1, Math.min(Number(url.searchParams.get("days")) || 14, 90)); + try { + response.leads = await recentLeads(client, { + now: ctx.now(), + days, + // Called or not, from the touch log; a log that can't be opened only leaves that out. + calls: async (since) => { + try { + return await (await ctx.store(await ctx.requireConfig())).since(since, { channel: "CALL" }); + } catch { + return []; + } + }, + }); + } catch (err) { + response.error = (err as Error).message; + } + return json(response); + }, + ], + [ + "POST", + /^\/api\/leads\/test$/, + async () => { + const { target, problem } = readNtfy(); + if (!target) throw new HttpError(409, problem ?? "Set COLD_NTFY_URL in Settings first: your ntfy topic's address, e.g. https://ntfy.sh/cold-leads-7f3k2q."); + try { + await sendNtfy(target, { title: "COLD lead alerts work", message: "A quiz lead will look like this: who, their company, the number to call and their first answers.", tags: ["white_check_mark"], priority: 3 }, ctx.options.ntfyFetch); + } catch (err) { + throw new HttpError(502, `The test alert didn't go out: ${(err as Error).message}`); + } + return json({ ok: true, message: `Sent to ${target.topic}. If it didn't arrive, check that your phone is subscribed to that topic.` }); + }, + ], + ]; +} diff --git a/src/domains/web/leads/leads.test.ts b/src/domains/web/leads/leads.test.ts new file mode 100644 index 0000000..533a75b --- /dev/null +++ b/src/domains/web/leads/leads.test.ts @@ -0,0 +1,116 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { openDatabase } from "../../db"; +import { TwentyClient } from "../../twenty"; +import { createApi } from "../api"; +import { WebContext } from "../context"; +import type { ActionResponse, LeadAlertsResponse, LeadsResponse, SourceOption } from "../types"; +import { leadRoutes } from "."; + +const REPO = path.resolve(import.meta.dir, "../../../.."); +const ORIGIN = "http://127.0.0.1:4100"; +const saved = { url: process.env.COLD_NTFY_URL, token: process.env.COLD_NTFY_TOKEN, twentyUrl: process.env.TWENTY_BASE_URL, twentyKey: process.env.TWENTY_API_KEY }; +let root: string; +let ctx: WebContext; +let api: (request: Request) => Promise; +const pushed: Array> = []; + +const note = (name: string) => `Name: ${name}\nEmail: —\nPhone: 512 555 0147\n\nQuiz answers: ${JSON.stringify({ "Cars a week": "10-20" }, null, 2)}`; +const leads = [ + { id: "sam", contactName: "Sam Lee", note: note("Sam Lee"), qualificationStatus: "QUALIFIED", createdAt: "2026-11-01T16:00:00.000Z" }, + { id: "ana", contactName: "Ana Ruiz", note: note("Ana Ruiz"), qualificationStatus: "QUALIFIED", createdAt: "2026-11-01T17:00:00.000Z" }, + { id: "dq", contactName: "Bo Diaz", note: note("Bo Diaz"), qualificationStatus: "DISQUALIFIED", createdAt: "2026-11-01T17:30:00.000Z" }, +]; +const twentyFetch = (async (input: string | URL | Request) => { + const url = new URL(String(input)); + return Response.json({ data: { [url.pathname.slice(6)]: url.pathname === "/rest/agencyLeads" ? leads : [] } }); +}) as typeof fetch; +const ntfyFetch = (async (_input: string | URL | Request, init?: RequestInit) => { + pushed.push(JSON.parse(String(init?.body))); + return Response.json({ id: "m1" }); +}) as typeof fetch; + +async function call(method: string, url: string, body?: unknown): Promise<{ status: number; data: T }> { + const response = await api(new Request(`${ORIGIN}${url}`, { method, headers: { "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) })); + return { status: response.status, data: (await response.json()) as T }; +} + +beforeAll(async () => { + delete process.env.COLD_CONFIG; + delete process.env.COLD_NTFY_URL; + delete process.env.COLD_NTFY_TOKEN; + root = mkdtempSync(path.join(tmpdir(), "cold-leads-")); + cpSync(path.join(REPO, "messages"), path.join(root, "messages"), { recursive: true }); + writeFileSync(path.join(root, "cold.yaml"), readFileSync(path.join(REPO, "cold.example.yaml"), "utf8").replace("store: auto", "store: file")); + ctx = new WebContext({ + root, + db: await openDatabase(":memory:"), + now: () => new Date("2026-11-01T18:00:00Z"), + twenty: () => new TwentyClient({ baseUrl: "https://crm.example", apiKey: "k" }, twentyFetch), + ntfyFetch, + }); + api = createApi(ctx, leadRoutes(ctx)); +}); + +afterAll(() => { + rmSync(root, { recursive: true, force: true }); + for (const [name, value] of [["COLD_NTFY_URL", saved.url], ["COLD_NTFY_TOKEN", saved.token]] as const) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +}); + +describe("lead routes", () => { + test("alerts say what is missing, without reading any leads", async () => { + const off = await call("GET", "/api/leads/alerts"); + expect(off.data).toEqual({ twenty: true, alerts: { configured: false, problem: undefined, available: true, lookedAt: undefined, seenUntil: undefined } }); + process.env.COLD_NTFY_URL = "https://ntfy.sh/"; + expect((await call("GET", "/api/leads/alerts")).data.alerts.problem).toMatch(/topic's name/); + process.env.COLD_NTFY_URL = "https://ntfy.sh/cold-leads-7f3k2q"; + expect((await call("GET", "/api/leads/alerts")).data.alerts.configured).toBe(true); + }); + + test("a test alert goes to the topic, and says so", async () => { + delete process.env.COLD_NTFY_URL; + expect((await call<{ error: string }>("POST", "/api/leads/test")).status).toBe(409); + process.env.COLD_NTFY_URL = "https://ntfy.sh/cold-leads-7f3k2q"; + const sent = await call("POST", "/api/leads/test"); + expect(sent.status).toBe(200); + expect(sent.data.message).toMatch(/Sent to cold-leads-7f3k2q/); + expect(pushed.at(-1)).toMatchObject({ topic: "cold-leads-7f3k2q", title: "COLD lead alerts work" }); + }); + + test("the Calls page's leads: the ones to call first on top, called ones after, with when", async () => { + const config = await ctx.requireConfig(); + await (await ctx.store(config)).record({ kind: "CALLED", channel: "CALL", prospectKey: "agencyLeads:ana", occurredAt: "2026-11-01T17:45:00.000Z" }); + const { status, data } = await call("GET", "/api/leads"); + expect(status).toBe(200); + expect(data.twenty).toBe(true); + expect(data.leads.map((l) => [l.id, l.qualified, l.calledAt])).toEqual([ + ["sam", true, undefined], + ["ana", true, "2026-11-01T17:45:00.000Z"], + ["dq", false, undefined], + ]); + expect(data.leads[0]).toMatchObject({ prospectKey: "agencyLeads:sam", phone: "512 555 0147", answers: ["Cars a week: 10-20"] }); + }); + + test("the leads are a prospect source, so a call script fills in for one", async () => { + process.env.TWENTY_BASE_URL = "https://crm.example"; + process.env.TWENTY_API_KEY = "k"; + try { + const all = createApi(ctx); + const response = await all(new Request(`${ORIGIN}/api/sources`, { headers: { "content-type": "application/json" } })); + const sources = (await response.json()) as SourceOption[]; + expect(sources.find((s) => s.id === "twenty:agencyLeads")?.label).toBe("Twenty: quiz leads from offer"); + const prospects = await ctx.prospects("twenty:agencyLeads"); + expect(prospects.map((p) => p.label)).toEqual(["Sam Lee", "Ana Ruiz", "Bo Diaz"]); + } finally { + if (saved.twentyUrl === undefined) delete process.env.TWENTY_BASE_URL; + else process.env.TWENTY_BASE_URL = saved.twentyUrl; + if (saved.twentyKey === undefined) delete process.env.TWENTY_API_KEY; + else process.env.TWENTY_API_KEY = saved.twentyKey; + } + }); +}); diff --git a/src/domains/web/server/index.ts b/src/domains/web/server/index.ts index d82efbd..34bdd0d 100644 --- a/src/domains/web/server/index.ts +++ b/src/domains/web/server/index.ts @@ -22,6 +22,7 @@ import { aiRoutes } from "../ai"; import { decideRoutes } from "../decide"; import { createApi } from "../api"; import { WebContext } from "../context"; +import { leadRoutes, logLeadCheck } from "../leads"; import { listenRoutes } from "../listen"; import { publicRoutes } from "../open"; import { xRoutes } from "../x"; @@ -42,10 +43,12 @@ try { const db = await openDatabase(path.join(ROOT, ".cold", "cold.db")); const auth = new OwnerAuth(db, { envOwner: OwnerAuth.envOwner() }); const waitlist = new Waitlist(db); -const ctx = new WebContext({ root: ROOT, db }); +const ctx = new WebContext({ root: ROOT, db, appUrl: new URL("/app/", settings.publicUrl).href }); // Sends and scoring jobs are kept in the database: carry on with any that were under way when the server last stopped. setInterval(() => void ctx.sends?.tick().catch((err) => console.error("send:", err)), 5000).unref(); setInterval(() => void ctx.scoring?.tick().catch((err) => console.error("scoring:", err)), 5000).unref(); +// New quiz leads: a look a minute, once alerts are set up (COLD_NTFY_URL). +setInterval(() => void ctx.leads?.tick().then(logLeadCheck, (err) => console.error("leads:", err)), 5000).unref(); // In production the site and the app come ready from `bun run build` (the Dockerfile runs it), so the // server starts faster and smaller. Otherwise they're bundled now, so `bun run ui` shows your latest changes. const bundled = (process.env.NODE_ENV === "production" && readBundle(path.join(APP_ROOT, DIST))) || (await bundle(APP_ROOT)); @@ -63,7 +66,7 @@ const handler = createHandler({ version: assets.version, auth, waitlist, - privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx), ...xRoutes(ctx), ...listenRoutes(ctx)]), + privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx), ...xRoutes(ctx), ...listenRoutes(ctx), ...leadRoutes(ctx)]), setup, clientIp: clientAddress(settings.proxyHops, (request) => server.requestIP(request)?.address), publicRoutes: publicRoutes(ctx), diff --git a/src/domains/web/types.ts b/src/domains/web/types.ts index b8834e0..38377c0 100644 --- a/src/domains/web/types.ts +++ b/src/domains/web/types.ts @@ -8,6 +8,7 @@ import type { CallSheet } from "../call"; import type { DecideSettings } from "../config"; import type { Action, Decision, Dossier, PickedBy, StyleInsights, Tier } from "../decide"; import type { EmailEvent, EmailRunResult } from "../email"; +import type { QuizLead } from "../leads"; import type { LintIssue, Platform, Voice } from "../messages"; import type { ApplyReport } from "../twenty"; import type { Touch } from "../touches"; @@ -111,6 +112,35 @@ export interface ListenStatusResponse { watches: Array<{ id: string; name: string; count: number; latest: string }>; } +/** Lead alerts: whether they are set up, and when they last looked. */ +export interface LeadAlertState { + /** COLD_NTFY_URL is set and usable. */ + configured: boolean; + /** What is wrong with COLD_NTFY_URL, in words. */ + problem?: string; + /** The app's database is there to remember which leads were pushed. */ + available: boolean; + /** When the last look for new leads ran. */ + lookedAt?: string; + /** The newest lead it has seen. */ + seenUntil?: string; +} + +/** Settings, Lead alerts. */ +export interface LeadAlertsResponse { + /** Twenty is connected, so offer's agencyLeads can be read. */ + twenty: boolean; + alerts: LeadAlertState; +} + +/** The Calls page's quiz leads from offer. */ +export interface LeadsResponse extends LeadAlertsResponse { + /** The last two weeks' leads, best to call first. */ + leads: QuizLead[]; + /** Why the leads couldn't be read. */ + error?: string; +} + export interface ProspectSummary { key: string; label: string; diff --git a/src/domains/web/vercel/index.ts b/src/domains/web/vercel/index.ts index f376a6a..7815c93 100644 --- a/src/domains/web/vercel/index.ts +++ b/src/domains/web/vercel/index.ts @@ -11,6 +11,7 @@ import { invocationDeadline } from "../background"; import { WebContext } from "../context"; import { clientAddress, createHandler, DbSetupCode } from "../server/handler"; import { serverSettings } from "../server/settings"; +import { leadRoutes, logLeadCheck } from "../leads"; import { listenRoutes } from "../listen"; import { publicRoutes } from "../open"; import { xRoutes } from "../x"; @@ -35,8 +36,8 @@ async function tidy(db: Db): Promise { export async function createVercelApp(db: Db, options: { now?: () => Date } = {}): Promise<(request: Request) => Promise> { const files = new DbFiles(db); await seedMessagesInto(files); - const ctx = new WebContext({ files, secrets: new DbSecrets(db, secretKey()), db, now: options.now }); const settings = serverSettings(); + const ctx = new WebContext({ files, secrets: new DbSecrets(db, secretKey()), db, now: options.now, appUrl: new URL("/app/", settings.publicUrl).href }); const auth = new OwnerAuth(db, { envOwner: OwnerAuth.envOwner() }); const appUrl = new URL("/app/", settings.publicUrl).href; const setup = new DbSetupCode(db, (code) => console.log(`First run: create your login at ${appUrl}?setup=${code} (setup code ${code}; it only works until the login exists)`)); @@ -46,7 +47,7 @@ export async function createVercelApp(db: Db, options: { now?: () => Date } = {} version: (process.env.VERCEL_GIT_COMMIT_SHA ?? process.env.VERCEL_DEPLOYMENT_ID ?? "dev").slice(0, 12), auth, waitlist: new Waitlist(db), - privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx), ...xRoutes(ctx), ...listenRoutes(ctx)]), + privateApi: createApi(ctx, [...aiRoutes(ctx), ...decideRoutes(ctx), ...xRoutes(ctx), ...listenRoutes(ctx), ...leadRoutes(ctx)]), setup, // Vercel sets X-Forwarded-For to the visitor's address itself. clientIp: clientAddress(settings.proxyHops, () => undefined), @@ -54,9 +55,9 @@ export async function createVercelApp(db: Db, options: { now?: () => Date } = {} limiter: (limit, windowMs) => new DbRateLimiter(db, limit, windowMs), cron: async () => { await ctx.freshen(); - // Sends and scoring side by side: a send waiting out its pause doesn't hold up a list being scored. + // Sends, scoring and the look for new quiz leads side by side: a send waiting out its pause holds up neither. const deadline = invocationDeadline(); - await Promise.all([ctx.sends?.tick({ deadline, lookaheadMs: 60_000 }), ctx.scoring?.tick({ deadline })]); + await Promise.all([ctx.sends?.tick({ deadline, lookaheadMs: 60_000 }), ctx.scoring?.tick({ deadline }), ctx.leads?.tick().then(logLeadCheck, (err) => console.error("leads:", err))]); await tidy(db); }, }); diff --git a/src/web/pages/calls.tsx b/src/web/pages/calls.tsx index dfc9e92..22b0ec6 100644 --- a/src/web/pages/calls.tsx +++ b/src/web/pages/calls.tsx @@ -1,13 +1,14 @@ -import { ListChecks, Phone, RefreshCw, Search, Sparkles } from "lucide-react"; -import { useMemo, useState } from "react"; +import { ListChecks, Phone, PhoneCall, RefreshCw, Search, Sparkles, X } from "lucide-react"; +import { useEffect, useMemo, useState } from "react"; import type { CallPrep } from "../../domains/ai/tasks"; import type { CallSheet } from "../../domains/call"; -import type { ActionResponse, LibraryResponse } from "../../domains/web/types"; +import type { QuizLead } from "../../domains/leads"; +import type { ActionResponse, LeadsResponse, LibraryResponse } from "../../domains/web/types"; import { setQuery, useLocation } from "../app/router"; import { useStatus } from "../app/status"; import { runAndWait } from "../lib/ai"; import { api, ApiError, useApi } from "../lib/api"; -import { voiceLabel } from "../lib/format"; +import { ago, voiceLabel } from "../lib/format"; import { Button, Empty, ErrorNotice, Field, Loading, Modal, Notice, PageHeader, Pill, Tabs } from "../ui/kit"; import { ProspectPicker } from "../ui/prospectPicker"; @@ -19,6 +20,16 @@ export function CallsPage() { const [source, setSource] = useState(""); const [prospect, setProspect] = useState(""); const [syncing, setSyncing] = useState(false); + const leads = useApi("/api/leads"); + const leadId = query.get("lead") ?? ""; + const lead = leads.data?.leads.find((l) => l.id === leadId); + const due = (leads.data?.leads ?? []).filter((l) => l.qualified && !l.calledAt && !l.doNotContact); + // A lead opened from its alert, or picked below, fills the script in for them. + useEffect(() => { + if (!lead) return; + setSource(LEADS_SOURCE); + setProspect(lead.prospectKey); + }, [lead?.id]); const scripts = library.data?.callScripts ?? []; const shown = [...scripts.filter((s) => s.voice === "personal"), ...scripts.filter((s) => s.voice === "company")]; const selected = query.get("id") ?? shown[0]?.id ?? ""; @@ -36,6 +47,21 @@ export function CallsPage() { setSearch(e.target.value)} /> + {due.length > 0 && ( +
+
Quiz leads: call these first
+ {due.slice(0, 8).map((l) => ( + + ))} + {due.length > 8 &&
{due.length - 8} more waiting
} +
+ )} {(["personal", "company"] as const).map((voice) => { const list = filtered.filter((s) => s.voice === voice); if (!list.length) return null; @@ -75,6 +101,7 @@ export function CallsPage() { } /> + {lead && setQuery({ lead: undefined })} />}
{ setSource(s); setProspect(p); + if (lead && (s !== LEADS_SOURCE || p !== lead.prospectKey)) setQuery({ lead: undefined }); }} />
@@ -128,6 +156,51 @@ export function CallsPage() { ); } +const LEADS_SOURCE = "twenty:agencyLeads"; + +/** The quiz lead being called: how to reach them and what they answered. */ +function LeadCard({ lead, onClose }: { lead: QuizLead; onClose: () => void }) { + return ( +
+
+
+ + {lead.label} + {lead.qualified ? "Qualified" : "Disqualified"} + {lead.doNotContact && Do Not Contact} + {lead.calledAt && Called {ago(lead.calledAt)}} + + +
+ + {lead.createdAt ? `Took your quiz ${ago(lead.createdAt)}` : "Took your quiz"} + {lead.from ? `, from ${lead.from}` : ""} + {lead.email ? `. ${lead.email}` : ""} + + {lead.phone && !lead.doNotContact && ( + + )} + {lead.answers.length > 0 && ( +
+
Their answers
+
    + {lead.answers.map((a) => ( +
  • {a}
  • + ))} +
+
+ )} +
+
+ ); +} + /** Claude's notes for this call: an opener, questions, objection answers and a voicemail, from what's known about the prospect. */ function CallPrepPanel({ script, source, prospect }: { script: string; source: string; prospect: string }) { const [prep, setPrep] = useState(); diff --git a/src/web/pages/settings.tsx b/src/web/pages/settings.tsx index ed209ac..46139b3 100644 --- a/src/web/pages/settings.tsx +++ b/src/web/pages/settings.tsx @@ -1,6 +1,6 @@ -import { AtSign, CheckCircle2, Copy, Database, Ear, FileText, KeyRound, ListPlus, LogOut, Mailbox, MonitorSmartphone, PlugZap, Save, ShieldCheck, Sparkles, Unplug, UserRound, Wand2, XCircle } from "lucide-react"; +import { AtSign, BellRing, CheckCircle2, Copy, Database, Ear, FileText, KeyRound, ListPlus, LogOut, Mailbox, MonitorSmartphone, PlugZap, Save, Send, ShieldCheck, Sparkles, Unplug, UserRound, Wand2, XCircle } from "lucide-react"; import { useEffect, useState, type ReactNode } from "react"; -import type { ActionResponse, AiStatusResponse, ListenStatusResponse, SessionsResponse, SettingsResponse, XStatusResponse } from "../../domains/web/types"; +import type { ActionResponse, AiStatusResponse, LeadAlertsResponse, ListenStatusResponse, SessionsResponse, SettingsResponse, XStatusResponse } from "../../domains/web/types"; import { usePwa, install } from "../app/pwa"; import { useSession } from "../app/session"; import { useStatus } from "../app/status"; @@ -134,6 +134,7 @@ export function SettingsPage() { + {settings.loading && !settings.data ? : settings.error ? : settings.data && (status.reload(), settings.reload())} />} {settings.data && (status.reload(), settings.reload())} />} @@ -216,6 +217,53 @@ function ListenCard() { ); } +/** A push to your phone when one of offer's quiz leads comes in, through ntfy. */ +function LeadAlertsCard() { + const leads = useApi("/api/leads/alerts"); + const toast = useToast(); + const [busy, setBusy] = useState(false); + + async function test() { + setBusy(true); + try { + toast.success((await api("/api/leads/test", { method: "POST" })).message); + } catch (err) { + toast.error((err as Error).message); + } finally { + setBusy(false); + } + } + + const d = leads.data; + return ( + + {leads.loading && !d ? ( + + ) : leads.error ? ( + + ) : ( + d && ( +
+

+ When someone takes your quiz in offer and qualifies, COLD pushes their name, company and number to your phone, so you call while it's fresh. It uses ntfy: install the ntfy app, subscribe to a topic with a long random name, and put its address in COLD_NTFY_URL below. The push carries the lead's name and number, so keep the topic to yourself, or use your own ntfy server with a token. +

+ {d.twenty ? "Twenty is connected, where offer saves its leads" : "Connect Twenty: offer saves its leads there"} + {d.alerts.configured ? "COLD_NTFY_URL is set" : "Set COLD_NTFY_URL below"} + {d.alerts.problem && {d.alerts.problem}} + {!d.alerts.available && This server has no database to remember which leads were pushed.} + {d.alerts.lookedAt && Last looked for new leads {ago(d.alerts.lookedAt)}. Leads from before alerts were set up are never pushed.} +
+ +
+
+ ) + )} +
+ ); +} + /** * X DMs through X's API: the owner's own developer app, and each account * connected to the X account it sends as. One DM goes out per click, from From 07973bbdb40596d630474f083bd137797a3fc6a8 Mon Sep 17 00:00:00 2001 From: operatoruplift Date: Sun, 1 Nov 2026 16:11:17 +0000 Subject: [PATCH 4/6] Calls at their hour: each prospect's local time and calling hours - A call sheet for a prospect (the app and cold call sheet) says what time it is where they are and whether it is inside calling hours: 8am to 9pm every day unless cold.yaml's new calls: section (start, end, days) says otherwise. Outside them it says when hours open, "call from tomorrow 8:00 AM CST". - The time zone comes from a timezone column, else from both the phone's area code and the state or province: an area code inside the state narrows it, one outside it adds its zone, and calls fit every zone they may be in. The country is the last resort when it has one zone. Numbers are read in E.164, 00, the US and Canadian national forms, and the national form of a known country. One it can't place is said, never guessed (an unknown US area code is not "every US zone"). - Phone prefix zones come from Google's libphonenumber timezone map (Apache 2.0, pinned commit, regenerated with bun run data:phoneZones), down to the exchange where an area code is split. - Quiz leads it is too early or late to call drop below the ones you can call now, their Call button waits, and their push says when to call. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuUQ7RbYzBy4coULGCu --- CHANGELOG.md | 11 + README.md | 4 + THIRD_PARTY_NOTICES.md | 194 +++++++++++++++ cold.example.yaml | 8 + docs/compliance.md | 8 +- package.json | 3 +- src/cli.ts | 2 +- src/domains/call/index.ts | 13 +- src/domains/callHours/build.ts | 62 +++++ src/domains/callHours/callHours.test.ts | 127 ++++++++++ src/domains/callHours/helpers/callTime.ts | 69 ++++++ src/domains/callHours/helpers/index.ts | 2 + .../callHours/helpers/phoneZones.data.ts | 9 + src/domains/callHours/helpers/zones.ts | 231 ++++++++++++++++++ src/domains/callHours/index.ts | 2 + src/domains/callHours/types.ts | 26 ++ .../config/helpers/parseConfig.test.ts | 13 + src/domains/config/helpers/parseConfig.ts | 25 +- src/domains/config/types.ts | 13 + src/domains/leads/helpers/quizLead.ts | 20 +- src/domains/leads/index.ts | 11 +- src/domains/leads/leads.test.ts | 15 +- src/domains/leads/types.ts | 4 + src/domains/web/api/index.ts | 2 +- src/domains/web/context/index.ts | 4 +- src/domains/web/leads/index.ts | 1 + src/domains/web/leads/leads.test.ts | 15 ++ src/web/pages/calls.tsx | 27 +- 28 files changed, 890 insertions(+), 31 deletions(-) create mode 100644 src/domains/callHours/build.ts create mode 100644 src/domains/callHours/callHours.test.ts create mode 100644 src/domains/callHours/helpers/callTime.ts create mode 100644 src/domains/callHours/helpers/index.ts create mode 100644 src/domains/callHours/helpers/phoneZones.data.ts create mode 100644 src/domains/callHours/helpers/zones.ts create mode 100644 src/domains/callHours/index.ts create mode 100644 src/domains/callHours/types.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 59ea8ba..6cd83f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +- **Calls at their hour.** A call sheet for a prospect, in the app and from + `cold call sheet`, says what time it is where they are and whether it is + inside calling hours: 8am to 9pm every day unless cold.yaml's new `calls:` + says otherwise (`start`, `end`, `days`). Outside them it says when hours + open ("call from tomorrow 8:00 AM CST"). The time zone comes from a + `timezone` column, else from both their phone's area code (from Google's + libphonenumber, down to the exchange where an area code is split) and + their state or province, and an area that spans zones is called only when + it is inside hours in all of them. A number it can't place is said, never + guessed. Quiz leads it is too early or late to call drop below the ones you + can call now, their Call button waits, and their push says when to call. - **Quiz leads get called first.** offer's quiz leads (`agencyLeads` in Twenty) are a prospect source, `twenty:agencyLeads`: the person who took the quiz (never the business's decision maker, when they came through a diff --git a/README.md b/README.md index 4d0f925..34eeae5 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,10 @@ call scripts and the Do Not Contact list. someone asks for what you sell comes in as a prospect whose trigger line is the post itself, with its link. A post on X can be answered with an X DM; save a watch as a list to fill in the rest (company, email) and research it. +- **Calls at their hour.** Every call sheet says what time it is for the + prospect, from their phone's area code or their state, and when calling + hours open if they are closed (8am to 9pm by default, `calls:` in + cold.yaml). Areas that span zones are called when every zone is awake. - **Quiz leads get called first.** Leads from your [offer](https://github.com/matthewdonsemail-lab/offer) quiz funnel are a prospect source (`twenty:agencyLeads`) with their answers as variables, diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index c5f9e62..081ac29 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -54,3 +54,197 @@ with flipping copy, pop-in icons) takes after the Copyright (c) 2026 ListeningKit Contributors). No code was copied; the patterns were rewritten for COLD. +## libphonenumber + +`src/domains/callHours/helpers/phoneZones.data.ts`, the time zones of each +phone number prefix, is generated by `src/domains/callHours/build.ts` from +`resources/timezones/map_data.txt` in Google's +[libphonenumber](https://github.com/google/libphonenumber) (commit +`1ae0d623301b302f828628dfc368c983e8d27141`), Copyright (C) 2012 The +Libphonenumber Authors, under the Apache License, Version 2.0. The data was +reformatted (zone names indexed, prefixes joined into one string); the +prefixes and zones are unchanged. + +
+Apache License, Version 2.0 + +```text + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS +``` + +
diff --git a/cold.example.yaml b/cold.example.yaml index a4b7cac..fe02569 100644 --- a/cold.example.yaml +++ b/cold.example.yaml @@ -98,6 +98,14 @@ default_offer: booking-pages # TWENTY_API_KEY are set, otherwise .cold/touches.jsonl), twenty, or file. store: auto +# When to call, in each prospect's own time zone (from their timezone column, +# their phone's area code or their state). The US rule is 8am to 9pm; other +# countries set their own. Every line is optional. +# calls: +# start: "08:00" +# end: "21:00" +# days: [mon, tue, wed, thu, fri, sat, sun] + # The decision engine (docs/decision-engine.md). Every line is optional. # decide: # provider: auto # who scores: auto (Jev with TYPESAFE_API_KEY, else Claude), jev, claude diff --git a/docs/compliance.md b/docs/compliance.md index 3b6612c..bc24ab0 100644 --- a/docs/compliance.md +++ b/docs/compliance.md @@ -75,7 +75,13 @@ within 5 business days. against the Do Not Call Register. Canada: keep your own do-not-call list even for B2B calls. - **Call at reasonable hours** in the prospect's time zone. The US rule is 8am - to 9pm; other countries set their own. + to 9pm; other countries set their own. COLD shows each prospect's local time + on the call sheet (app and `cold call sheet`) and holds a quiz lead's Call + button outside the hours in cold.yaml's `calls:` (8am to 9pm by default). It + reads the time zone from a `timezone` column first, else from both the + phone's area code and the state or province, and calls only when it is + inside the hours everywhere they may be. A number it can't place says so: + check where they are before you dial. - **Say who you are and why you're calling,** and give a number they can reach you on (`{{sender_phone}}` in the voicemail scripts). - **Keep an internal do-not-call list.** That is the dialer's Do Not Contact diff --git a/package.json b/package.json index 00102e2..ea8bb34 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,8 @@ "test": "bun test", "check": "bun run typecheck && bun run test && bun run lint:messages", "twenty:schema": "bun src/cli.ts twenty:schema", - "twenty:schema:check": "bun src/cli.ts twenty:schema --dry-run" + "twenty:schema:check": "bun src/cli.ts twenty:schema --dry-run", + "data:phoneZones": "bun src/domains/callHours/build.ts" }, "dependencies": { "@anthropic-ai/sdk": "0.135.0", diff --git a/src/cli.ts b/src/cli.ts index d7cae02..92436ca 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -338,7 +338,7 @@ async function call(sub: string | undefined, id: string | undefined): Promise"); diff --git a/src/domains/call/index.ts b/src/domains/call/index.ts index 61bf3b0..2590781 100644 --- a/src/domains/call/index.ts +++ b/src/domains/call/index.ts @@ -1,4 +1,5 @@ -import type { Account, Offer } from "../config"; +import { callTime, type CallTime } from "../callHours"; +import type { Account, CallHours, Offer } from "../config"; import { bold, dim, green, indent, type Io } from "../cli/io"; import type { Library, Message } from "../messages"; import { buildVariables } from "../outreach"; @@ -68,6 +69,8 @@ export interface CallSheet { objections: Array<{ objection: string; category: string; response: string }>; /** Variables with no value, when filled for a prospect. */ missing: string[]; + /** Whether the prospect can be called now, in their own time. */ + callTime?: CallTime; } /** @@ -75,7 +78,7 @@ export interface CallSheet { * every objection with its answer in the script's voice. Filled in for a * prospect when one is given; otherwise prospect details stay as {{name}}. */ -export function callSheetData(script: Message, library: Library, account: Account, offer: Offer | undefined, prospect?: Prospect): CallSheet { +export function callSheetData(script: Message, library: Library, account: Account, offer: Offer | undefined, prospect?: Prospect, clock?: { hours: CallHours; now: Date }): CallSheet { const variables = buildVariables(account, offer, prospect); const mode = prospect ? "strict" : "partial"; const missing = new Set(); @@ -100,6 +103,7 @@ export function callSheetData(script: Message, library: Library, account: Accoun sections, objections: library.objections.map((o) => ({ objection: o.objection, category: o.category, response: fill(o[script.voice], o.objection) })), missing: [...missing], + callTime: prospect && clock ? callTime(prospect, clock.hours, clock.now) : undefined, }; } @@ -107,9 +111,10 @@ export function callSheetData(script: Message, library: Library, account: Accoun * One call script as a sheet to read from: the script, then every objection * with its answer, filled in for a prospect when one is given. */ -export function callSheet(script: Message, library: Library, account: Account, offer: Offer | undefined, prospect?: Prospect): string { - const sheet = callSheetData(script, library, account, offer, prospect); +export function callSheet(script: Message, library: Library, account: Account, offer: Offer | undefined, prospect?: Prospect, clock?: { hours: CallHours; now: Date }): string { + const sheet = callSheetData(script, library, account, offer, prospect, clock); const lines = [bold(`${sheet.title}${prospect ? ` - ${prospect.label}` : ""}`), dim(`${sheet.category} | ${sheet.voice} voice | ${sheet.when ?? ""}`), ""]; + if (sheet.callTime) lines.splice(2, 0, sheet.callTime.ok === false ? bold(`Not now: ${sheet.callTime.note}`) : sheet.callTime.ok ? `Good time to call: ${sheet.callTime.note}` : sheet.callTime.note); for (const section of sheet.sections) lines.push(section.heading.toUpperCase(), section.text, ""); lines.push(bold("OBJECTIONS")); for (const o of sheet.objections) lines.push("", `"${o.objection}"`, indent(o.response, " ")); diff --git a/src/domains/callHours/build.ts b/src/domains/callHours/build.ts new file mode 100644 index 0000000..6d47b48 --- /dev/null +++ b/src/domains/callHours/build.ts @@ -0,0 +1,62 @@ +#!/usr/bin/env bun +/** + * Writes helpers/phoneZones.data.ts: which time zones each phone number + * prefix is in, from Google's libphonenumber (resources/timezones/ + * map_data.txt, Apache License 2.0) at a pinned commit. Run it to move to a + * newer commit: `bun run data:phoneZones [commit]`. + */ +import { writeFileSync } from "node:fs"; +import path from "node:path"; + +const COMMIT = process.argv[2] ?? "1ae0d623301b302f828628dfc368c983e8d27141"; +const SOURCE = `https://raw.githubusercontent.com/google/libphonenumber/${COMMIT}/resources/timezones/map_data.txt`; +const OUT = path.join(import.meta.dir, "helpers", "phoneZones.data.ts"); + +const isZone = (name: string) => { + try { + new Intl.DateTimeFormat("en-US", { timeZone: name }); + return true; + } catch { + return false; + } +}; + +const response = await fetch(SOURCE); +if (!response.ok) throw new Error(`${SOURCE}: ${response.status}`); +const zones: string[] = []; +const index = new Map(); +const entries: string[] = []; +let dropped = 0; +for (const line of (await response.text()).split("\n")) { + const m = /^(\d+)\|(.+)$/.exec(line.trim()); + if (!m) continue; + const ids: number[] = []; + for (const zone of m[2]!.split("&")) { + if (!isZone(zone)) { + dropped++; + continue; + } + if (!index.has(zone)) { + index.set(zone, zones.length); + zones.push(zone); + } + ids.push(index.get(zone)!); + } + if (ids.length) entries.push(`${m[1]}:${ids.join(",")}`); +} +if (entries.length < 1000) throw new Error(`Only ${entries.length} prefixes: is ${SOURCE} the right file?`); + +writeFileSync( + OUT, + `// Generated by src/domains/callHours/build.ts from Google's libphonenumber, +// resources/timezones/map_data.txt at ${COMMIT} +// (Apache License 2.0, see THIRD_PARTY_NOTICES.md). Do not edit by hand. + +/** Every time zone a prefix can be in, referenced by index in PHONE_PREFIXES. */ +export const PHONE_ZONE_NAMES: readonly string[] = ${JSON.stringify(zones)}; + +/** "prefix:zone,zone" entries, ";" between them: an E.164 number's longest matching prefix gives its zones. */ +export const PHONE_PREFIXES = ${JSON.stringify(entries.join(";"))}; +`, +); +console.log(`${entries.length} prefixes, ${zones.length} zones${dropped ? `, ${dropped} unknown zone names dropped` : ""} -> ${path.relative(process.cwd(), OUT)}`); diff --git a/src/domains/callHours/callHours.test.ts b/src/domains/callHours/callHours.test.ts new file mode 100644 index 0000000..571e55f --- /dev/null +++ b/src/domains/callHours/callHours.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, test } from "bun:test"; +import { DEFAULT_CALL_HOURS, type CallHours } from "../config"; +import { callTime, localTimes, nextCallTime, offsetMinutes, phoneDigits, prospectZones, zonesForNumber, zonesForRegion } from "."; + +const CT = "America/Chicago"; +const MT = "America/Denver"; +// Monday 2 November 2026, the day after US clocks went back: 9:00 AM in Chicago. +const MONDAY_9AM_CT = new Date("2026-11-02T15:00:00Z"); +const who = (variables: Record, phone?: string) => ({ phone, variables }); + +describe("phone numbers", () => { + test("E.164, 00 and the US and Canadian national forms", () => { + expect(phoneDigits("(512) 555-0147")).toBe("15125550147"); + expect(phoneDigits("1-512-555-0147")).toBe("15125550147"); + expect(phoneDigits("+44 20 7946 0958")).toBe("442079460958"); + expect(phoneDigits("0044 20 7946 0958")).toBe("442079460958"); + expect(phoneDigits("011 44 20 7946 0958", "US")).toBe("442079460958"); + expect(phoneDigits("555-0147")).toBeUndefined(); + expect(phoneDigits(undefined)).toBeUndefined(); + }); + + test("another country's national form, when the country is known", () => { + expect(phoneDigits("020 7946 0958", "United Kingdom")).toBe("442079460958"); + expect(phoneDigits("087 123 4567", "Ireland")).toBe("353871234567"); + expect(phoneDigits("020 7946 0958")).toBeUndefined(); + expect(phoneDigits("020 7946 0958", "Narnia")).toBeUndefined(); + }); + + test("an area code gives its zones, a split one its exchange's", () => { + expect(zonesForNumber("15125550147")).toEqual([CT]); + expect(zonesForNumber("12125550147")).toEqual(["America/New_York"]); + expect(zonesForNumber("12082015555")).toEqual([MT]); + expect(zonesForNumber("12089995555")).toEqual(["America/Boise", "America/Los_Angeles"]); + expect(zonesForNumber("442079460958")).toContain("Europe/London"); + }); + + test("a state or province gives every zone it spans", () => { + expect(zonesForRegion("TX")).toEqual([CT, MT]); + expect(zonesForRegion("Texas")).toEqual([CT, MT]); + expect(zonesForRegion(" ontario ")).toEqual(["America/Toronto", "America/Winnipeg"]); + expect(zonesForRegion("Atlantis")).toBeUndefined(); + }); +}); + +describe("prospectZones", () => { + test("a timezone column says it outright", () => { + expect(prospectZones(who({ timezone: "Europe/Dublin" }, "(512) 555-0147"), MONDAY_9AM_CT)).toEqual({ zones: ["Europe/Dublin"], from: ["timezone"] }); + expect(prospectZones(who({ timezone: "Mars/Olympus" }, "(512) 555-0147"), MONDAY_9AM_CT)?.from).toEqual(["phone"]); + }); + + test("an area code inside the state narrows it; one outside it adds its zone", () => { + expect(prospectZones(who({ region: "TX" }), MONDAY_9AM_CT)).toEqual({ zones: [CT, MT], from: ["region"] }); + expect(prospectZones(who({ region: "TX" }, "(512) 555-0147"), MONDAY_9AM_CT)).toEqual({ zones: [CT], from: ["phone", "region"] }); + expect(prospectZones(who({ state: "Texas" }, "(917) 555-0147"), MONDAY_9AM_CT)).toEqual({ zones: ["America/New_York", CT, MT], from: ["phone", "region"] }); + }); + + test("a state only counts in the US and Canada", () => { + expect(prospectZones(who({ region: "WA", country: "Australia" }), MONDAY_9AM_CT)).toBeUndefined(); + expect(prospectZones(who({ region: "WA", country: "US" }), MONDAY_9AM_CT)).toEqual({ zones: ["America/Los_Angeles"], from: ["region"] }); + }); + + test("a country is the last resort, and only when it has one time zone", () => { + expect(prospectZones(who({ country: "Ireland" }), MONDAY_9AM_CT)?.from).toEqual(["country"]); + expect(prospectZones(who({ country: "Australia" }), MONDAY_9AM_CT)).toBeUndefined(); + expect(prospectZones(who({ country: "US" }), MONDAY_9AM_CT)).toBeUndefined(); + }); + + test("an unknown area code says nothing, rather than every zone in the US", () => { + expect(prospectZones(who({}, "+1 621 555 0100"), MONDAY_9AM_CT)).toBeUndefined(); + expect(prospectZones(who({}), MONDAY_9AM_CT)).toBeUndefined(); + }); +}); + +describe("callTime", () => { + test("inside calling hours: their time, and where it came from", () => { + const t = callTime(who({}, "(512) 555-0147"), DEFAULT_CALL_HOURS, MONDAY_9AM_CT); + expect(t).toMatchObject({ ok: true, local: "9:00 AM CST", zones: [CT], from: ["phone"] }); + expect(t.note).toBe("9:00 AM CST their time (from their phone number)."); + }); + + test("too late: when it opens, tomorrow", () => { + const t = callTime(who({}, "(512) 555-0147"), DEFAULT_CALL_HOURS, new Date("2026-11-03T03:30:00Z")); + expect(t.ok).toBe(false); + expect(t.local).toBe("9:30 PM CST"); + expect(t.nextAt).toBe("2026-11-03T14:00:00.000Z"); + expect(t.note).toBe("9:30 PM CST their time, outside calling hours: call from tomorrow 8:00 AM CST (from their phone number)."); + }); + + test("before hours the same morning, and a weekday-only week", () => { + expect(callTime(who({}, "(512) 555-0147"), DEFAULT_CALL_HOURS, new Date("2026-11-02T12:10:00Z")).note).toContain("call from 8:00 AM CST"); + const weekdays: CallHours = { ...DEFAULT_CALL_HOURS, days: ["mon", "tue", "wed", "thu", "fri"] }; + const saturday = callTime(who({}, "(512) 555-0147"), weekdays, new Date("2026-11-07T16:00:00Z")); + expect(saturday.ok).toBe(false); + expect(saturday.nextAt).toBe("2026-11-09T14:00:00.000Z"); + expect(saturday.note).toContain("call from Mon 8:00 AM CST"); + }); + + test("a state that spans zones is called when it is late enough in all of them", () => { + const t = callTime(who({ region: "TX" }), DEFAULT_CALL_HOURS, new Date("2026-11-02T14:30:00Z")); + expect(t.ok).toBe(false); + expect(t.local).toBe("8:30 AM CST / 7:30 AM MST"); + expect(t.nextAt).toBe("2026-11-02T15:00:00.000Z"); + expect(t.note).toContain("call from 9:00 AM CST / 8:00 AM MST (from their state or province)"); + }); + + test("zones that never share calling hours say so", () => { + const morning: CallHours = { ...DEFAULT_CALL_HOURS, start: "09:00", end: "12:00" }; + const t = callTime(who({ region: "NY" }, "(808) 555-0147"), morning, MONDAY_9AM_CT); + expect(t.ok).toBe(false); + expect(t.nextAt).toBeUndefined(); + expect(t.note).toContain("never share calling hours"); + }); + + test("an unknown time zone is said, not guessed", () => { + const t = callTime(who({}), DEFAULT_CALL_HOURS, MONDAY_9AM_CT); + expect(t.ok).toBeUndefined(); + expect(t.note).toMatch(/time zone is unknown/); + }); + + test("the clock helpers", () => { + expect(offsetMinutes(CT, MONDAY_9AM_CT)).toBe(-360); + expect(offsetMinutes(CT, new Date("2026-07-01T12:00:00Z"))).toBe(-300); + expect(offsetMinutes("Asia/Kolkata", MONDAY_9AM_CT)).toBe(330); + expect(localTimes([CT, "America/Indiana/Indianapolis", "America/New_York"], MONDAY_9AM_CT)).toBe("9:00 AM CST / 10:00 AM EST"); + expect(nextCallTime([CT], new Date("2026-11-02T13:58:00Z"), DEFAULT_CALL_HOURS)?.toISOString()).toBe("2026-11-02T14:00:00.000Z"); + }); +}); diff --git a/src/domains/callHours/helpers/callTime.ts b/src/domains/callHours/helpers/callTime.ts new file mode 100644 index 0000000..4dd499d --- /dev/null +++ b/src/domains/callHours/helpers/callTime.ts @@ -0,0 +1,69 @@ +import type { CallHours, Weekday } from "../../config"; +import type { CallTime, ZoneSource } from "../types"; +import { prospectZones } from "./zones"; + +const clocks = new Map(); + +/** The weekday and "HH:MM" in a zone at a moment. */ +function clock(zone: string, at: Date): { day: Weekday; time: string; date: string } { + let format = clocks.get(zone); + if (!format) { + format = new Intl.DateTimeFormat("en-US", { timeZone: zone, weekday: "short", year: "numeric", month: "2-digit", day: "2-digit", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }); + clocks.set(zone, format); + } + const parts = Object.fromEntries(format.formatToParts(at).map((p) => [p.type, p.value])); + return { day: String(parts.weekday).slice(0, 3).toLowerCase() as Weekday, time: `${parts.hour}:${parts.minute}`, date: `${parts.year}-${parts.month}-${parts.day}` }; +} + +/** Inside calling hours in this zone at this moment. Pure. */ +export function inCallHours(zone: string, at: Date, hours: CallHours): boolean { + const { day, time } = clock(zone, at); + return hours.days.includes(day) && time >= hours.start && time < hours.end; +} + +const STEP_MS = 5 * 60_000; +const HORIZON_MS = 8 * 86_400_000; + +/** The first moment from `from` that is inside calling hours in every zone; undefined when there is none within a week. Pure. */ +export function nextCallTime(zones: string[], from: Date, hours: CallHours): Date | undefined { + const end = from.getTime() + HORIZON_MS; + for (let t = Math.ceil(from.getTime() / STEP_MS) * STEP_MS; t <= end; t += STEP_MS) { + const at = new Date(t); + if (zones.every((z) => inCallHours(z, at, hours))) return at; + } + return undefined; +} + +/** "9:14 AM CST", and one per different local time when the zones differ: "10:14 AM EST / 9:14 AM CST". Pure. */ +export function localTimes(zones: string[], at: Date): string { + return [...new Set(zones.map((zone) => new Intl.DateTimeFormat("en-US", { timeZone: zone, hour: "numeric", minute: "2-digit", timeZoneName: "short" }).format(at)))].join(" / "); +} + +/** When calling opens, in their words: "8:00 AM CST", "tomorrow 8:00 AM CST" or "Mon 8:00 AM CST". */ +function opensAt(zones: string[], next: Date, now: Date): string { + const zone = zones[0]!; + const today = clock(zone, now).date; + const then = clock(zone, next); + const tomorrow = clock(zone, new Date(now.getTime() + 86_400_000)).date; + const day = then.date === today ? "" : then.date === tomorrow ? "tomorrow " : `${then.day[0]!.toUpperCase()}${then.day.slice(1)} `; + return `${day}${localTimes(zones, next)}`; +} + +const SOURCES: Record = { timezone: "their timezone column", phone: "their phone number", region: "their state or province", country: "their country" }; + +/** + * Whether a prospect can be called now, in their own time: their local time, + * whether it is inside calling hours in every zone they may be in, and when + * hours open next when it isn't. Pure. + */ +export function callTime(prospect: { phone?: string; variables: Record }, hours: CallHours, now: Date): CallTime { + const found = prospectZones(prospect, now); + if (!found) return { zones: [], from: [], note: "Their time zone is unknown: add their phone number with its area code, their state, or a timezone column." }; + const { zones, from } = found; + const local = localTimes(zones, now); + const source = from.map((f) => SOURCES[f]).join(" and "); + if (zones.every((z) => inCallHours(z, now, hours))) return { ok: true, local, zones, from, note: `${local} their time (from ${source}).` }; + const next = nextCallTime(zones, now, hours); + if (!next) return { ok: false, local, zones, from, note: `${local} their time, and the time zones they may be in never share calling hours: check where they are (from ${source}).` }; + return { ok: false, local, zones, from, nextAt: next.toISOString(), note: `${local} their time, outside calling hours: call from ${opensAt(zones, next, now)} (from ${source}).` }; +} diff --git a/src/domains/callHours/helpers/index.ts b/src/domains/callHours/helpers/index.ts new file mode 100644 index 0000000..f5e342a --- /dev/null +++ b/src/domains/callHours/helpers/index.ts @@ -0,0 +1,2 @@ +export * from "./callTime"; +export * from "./zones"; diff --git a/src/domains/callHours/helpers/phoneZones.data.ts b/src/domains/callHours/helpers/phoneZones.data.ts new file mode 100644 index 0000000..04e434d --- /dev/null +++ b/src/domains/callHours/helpers/phoneZones.data.ts @@ -0,0 +1,9 @@ +// Generated by src/domains/callHours/build.ts from Google's libphonenumber, +// resources/timezones/map_data.txt at 1ae0d623301b302f828628dfc368c983e8d27141 +// (Apache License 2.0, see THIRD_PARTY_NOTICES.md). Do not edit by hand. + +/** Every time zone a prefix can be in, referenced by index in PHONE_PREFIXES. */ +export const PHONE_ZONE_NAMES: readonly string[] = ["America/Adak","America/Anchorage","America/Anguilla","America/Antigua","America/Barbados","America/Boise","America/Cayman","America/Chicago","America/Denver","America/Dominica","America/Edmonton","America/Fort_Nelson","America/Grand_Turk","America/Grenada","America/Halifax","America/Jamaica","America/Juneau","America/Los_Angeles","America/Lower_Princes","America/Montserrat","America/Nassau","America/New_York","America/North_Dakota/Center","America/Phoenix","America/Port_of_Spain","America/Puerto_Rico","America/Regina","America/Santo_Domingo","America/St_Johns","America/St_Kitts","America/St_Lucia","America/St_Thomas","America/St_Vincent","America/Toronto","America/Tortola","America/Vancouver","America/Winnipeg","Atlantic/Bermuda","Pacific/Guam","Pacific/Honolulu","Pacific/Pago_Pago","Pacific/Saipan","Africa/Cairo","Africa/Nairobi","Atlantic/Canary","Europe/Paris","Africa/Tunis","Europe/Bucharest","Africa/Banjul","Africa/Dakar","Africa/Nouakchott","Africa/Bamako","Africa/Conakry","Africa/Abidjan","Africa/Ouagadougou","Africa/Niamey","Africa/Lome","Africa/Porto-Novo","Indian/Mauritius","Atlantic/Reykjavik","Africa/Freetown","Africa/Accra","Africa/Lagos","Africa/Ndjamena","Africa/Bangui","Africa/Douala","Atlantic/Cape_Verde","Africa/Sao_Tome","Africa/Malabo","Africa/Libreville","Africa/Brazzaville","Africa/Kinshasa","Africa/Lubumbashi","Africa/Luanda","Indian/Chagos","Atlantic/St_Helena","Indian/Mahe","Africa/Khartoum","Africa/Kigali","Africa/Addis_Ababa","Africa/Mogadishu","Africa/Djibouti","Africa/Dar_es_Salaam","Africa/Kampala","Africa/Bujumbura","Africa/Maputo","Africa/Lusaka","Indian/Antananarivo","Indian/Mayotte","Indian/Reunion","Africa/Harare","Africa/Windhoek","Africa/Blantyre","Africa/Maseru","Africa/Gaborone","Africa/Mbabane","Indian/Comoro","Africa/Johannesburg","Africa/Asmera","America/Aruba","Atlantic/Faeroe","America/Godthab","America/Scoresbysund","America/Thule","Europe/Athens","Europe/Amsterdam","Europe/Brussels","Europe/Madrid","Europe/Gibraltar","Atlantic/Azores","Europe/Lisbon","Europe/Luxembourg","Europe/Guernsey","Europe/Isle_of_Man","Europe/London","Europe/Tirane","Europe/Malta","Asia/Nicosia","Europe/Helsinki","Europe/Mariehamn","Europe/Sofia","Europe/Budapest","Europe/Chisinau","Asia/Yerevan","Europe/Moscow","Europe/Andorra","Europe/Monaco","Europe/San_Marino","Europe/Kyiv","Europe/Belgrade","Europe/Podgorica","Europe/Zagreb","Europe/Ljubljana","Europe/Sarajevo","Europe/Skopje","Europe/Rome","Europe/Vatican","Europe/Zurich","Europe/Prague","Europe/Bratislava","Europe/Vaduz","Europe/Vienna","Europe/Jersey","Europe/Copenhagen","Europe/Stockholm","Arctic/Longyearbyen","Europe/Oslo","Europe/Warsaw","Europe/Berlin","Atlantic/Stanley","America/Belize","America/Guatemala","America/El_Salvador","America/Tegucigalpa","America/Costa_Rica","America/Panama","America/Miquelon","America/Port-au-Prince","America/Lima","America/Cancun","America/Ciudad_Juarez","America/Matamoros","America/Mazatlan","America/Mexico_City","America/Ojinaga","America/Tijuana","America/Havana","America/Buenos_Aires","America/Manaus","America/Noronha","America/Rio_Branco","America/Sao_Paulo","America/Santiago","America/Bogota","America/Caracas","America/Guadeloupe","America/La_Paz","America/Guyana","America/Guayaquil","Pacific/Galapagos","America/Cayenne","America/Asuncion","America/Martinique","America/Paramaribo","America/Montevideo","America/Curacao","America/Kralendijk","Asia/Kuching","Australia/Adelaide","Australia/Brisbane","Australia/Eucla","Australia/Lord_Howe","Australia/Perth","Australia/Sydney","Indian/Christmas","Indian/Cocos","Asia/Jakarta","Asia/Jayapura","Asia/Makassar","Asia/Manila","Pacific/Auckland","Pacific/Chatham","Asia/Singapore","Asia/Bangkok","Asia/Dili","Pacific/Norfolk","Asia/Brunei","Pacific/Nauru","Pacific/Port_Moresby","Pacific/Tongatapu","Pacific/Guadalcanal","Pacific/Efate","Pacific/Fiji","Pacific/Palau","Pacific/Wallis","Pacific/Rarotonga","Pacific/Niue","Pacific/Apia","Pacific/Enderbury","Pacific/Kiritimati","Pacific/Tarawa","Pacific/Noumea","Pacific/Funafuti","Pacific/Gambier","Pacific/Marquesas","Pacific/Tahiti","Pacific/Fakaofo","Pacific/Kosrae","Pacific/Ponape","Pacific/Truk","Pacific/Majuro","Asia/Almaty","Asia/Anadyr","Asia/Aqtobe","Asia/Irkutsk","Asia/Kamchatka","Asia/Krasnoyarsk","Asia/Magadan","Asia/Novosibirsk","Asia/Omsk","Asia/Sakhalin","Asia/Vladivostok","Asia/Yakutsk","Asia/Yekaterinburg","Europe/Samara","Asia/Tokyo","Asia/Seoul","Asia/Ho_Chi_Minh","Asia/Hong_Kong","Asia/Shanghai","Asia/Phnom_Penh","Asia/Vientiane","Asia/Dhaka","Asia/Taipei","Europe/Istanbul","Asia/Calcutta","Asia/Karachi","Asia/Kabul","Asia/Colombo","Asia/Rangoon","Indian/Maldives","Asia/Beirut","Asia/Amman","Asia/Damascus","Asia/Baghdad","Asia/Kuwait","Asia/Riyadh","Asia/Aden","Asia/Muscat","Asia/Dubai","Asia/Jerusalem","Asia/Bahrain","Asia/Qatar","Asia/Thimphu","Asia/Choibalsan","Asia/Hovd","Asia/Ulaanbaatar","Asia/Katmandu","Asia/Tehran","Asia/Dushanbe","Asia/Ashgabat","Asia/Baku","Asia/Tbilisi","Asia/Bishkek","Asia/Tashkent"]; + +/** "prefix:zone,zone" entries, ";" between them: an E.164 number's longest matching prefix gives its zones. */ +export const PHONE_PREFIXES = "1:0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41;1201:21;1202:21;1203:21;1204:36;1205:7;1206:17;1207:21;12082:5,17;1208201:8;1208215:17;1208221:8;1208226:8;1208227:8;1208228:8;120823:8;1208241:8;1208242:8;1208245:17;1208249:8;1208250:8;1208251:8;1208253:8;1208254:8;1208255:17;1208258:8;1208262:17;1208263:17;1208265:17;1208267:17;1208278:8;120828:8;1208290:17;1208292:17;1208297:8;12083:8;120830:5,17;1208304:17;1208305:17;120831:5,17;1208320:5,17;1208325:5,17;1208328:5,17;1208329:5,17;1208330:5,17;1208332:5,17;1208335:5,17;1208341:5,17;1208347:5,17;1208348:5,17;1208349:5,17;1208355:5,17;1208358:5,17;1208361:5,17;1208363:5,17;1208364:5,17;1208368:5,17;1208369:5,17;1208370:5,17;1208372:5,17;1208374:5,17;1208379:5,17;1208380:5,17;1208386:5,17;1208387:5,17;1208393:5,17;1208394:5,17;1208395:5,17;1208396:5,17;1208399:5,17;1208400:5,17;1208401:8;1208402:8;1208403:8;1208404:5,17;1208405:5,17;1208406:5,17;1208407:8;1208408:5,17;1208409:8;120841:5,17;1208412:8;1208414:8;1208419:8;120842:8;1208421:5,17;1208427:5,17;1208428:5,17;1208430:5,17;1208431:8;1208432:5,17;1208433:8;1208434:5,17;1208435:5,17;1208436:8;1208437:17;1208438:8;1208439:5,17;1208440:8;1208441:5,17;1208442:8;1208443:17;1208444:5,17;1208445:5,17;1208446:17;1208447:5,17;1208448:17;1208449:5,17;1208450:5,17;1208451:5,17;1208452:8;1208453:8;1208454:8;1208455:8;1208456:5,17;1208457:17;1208458:5,17;1208459:8;120846:8;1208460:5,17;1208464:5,17;1208469:5,17;1208470:5,17;1208471:5,17;1208472:8;1208473:8;1208474:5,17;1208475:8;1208476:17;1208477:5,17;1208478:8;1208479:5,17;120848:5,17;1208484:8;1208489:8;120849:5,17;1208495:8;12085:5,17;1208505:8;1208514:8;1208515:8;120852:8;1208535:8;1208536:8;1208538:8;1208539:8;1208542:8;1208543:8;1208546:8;1208547:8;1208549:8;1208552:8;1208556:17;1208557:8;1208558:8;1208559:8;1208562:8;1208569:8;1208570:8;1208571:8;1208573:8;1208577:8;1208578:8;1208585:8;1208587:8;1208588:8;1208589:8;1208595:8;1208596:17;12086:5,17;1208602:8;1208608:8;1208610:17;1208622:8;1208623:17;1208624:8;1208625:17;1208628:8;1208629:8;1208631:8;1208634:8;1208637:8;1208639:8;1208640:17;1208642:8;1208651:17;1208652:8;1208656:8;1208658:8;1208659:17;120866:17;1208672:8;1208676:17;1208677:8;1208678:8;1208681:8;1208682:17;1208683:17;1208684:8;1208686:17;1208687:17;1208691:17;1208695:8;1208697:8;1208699:17;1208700:5,17;1208701:5,17;1208702:5,17;1208703:8;1208704:17;1208705:8;1208706:8;1208707:5,17;1208708:5,17;1208709:8;120871:5,17;1208713:8;1208716:8;120872:8;1208721:5,17;1208723:5,17;1208728:5,17;1208729:5,17;120873:8;1208730:5,17;1208738:5,17;1208739:5,17;120874:5,17;1208743:17;1208745:8;1208746:17;120875:5,17;1208755:17;1208756:8;1208760:5,17;1208761:8;1208762:17;1208763:5,17;1208764:5,17;1208765:17;1208766:8;1208767:5,17;1208768:5,17;1208769:17;120877:5,17;1208772:17;1208773:17;1208777:17;1208780:5,17;1208781:5,17;1208782:8;1208783:17;1208784:17;1208785:8;1208786:5,17;1208787:8;1208788:8;1208789:8;120879:5,17;1208791:17;1208794:8;1208798:17;1208799:17;120880:5,17;120881:5,17;1208814:8;1208818:17;1208819:17;120882:5,17;120883:5,17;1208830:8;1208835:17;1208837:8;120884:5,17;1208841:8;1208843:17;1208846:8;1208847:8;120885:8;1208851:5,17;1208856:5,17;1208857:5,17;1208858:5,17;120886:8;1208862:5,17;1208864:5,17;1208865:5,17;1208868:5,17;1208870:8;1208871:8;1208872:5,17;1208873:5,17;1208874:5,17;1208875:17;1208876:5,17;1208877:5,17;1208878:8;1208879:8;120888:8;1208882:17;1208883:17;1208885:17;1208889:5,17;120889:8;1208892:5,17;1208893:5,17;1208894:5,17;1208897:5,17;12089:5,17;1208901:8;1208904:8;1208906:8;1208908:8;1208917:8;1208918:8;1208921:8;1208922:8;1208926:17;1208928:8;1208934:8;1208935:17;1208936:8;1208938:8;1208939:8;1208941:8;1208944:8;1208946:17;1208947:8;1208949:8;1208954:8;1208955:8;1208962:17;1208964:17;1208983:17;1208989:8;1208991:8;1208994:8;1208995:8;1208996:8;1209:17;1210:7;1212:21;1213:17;1214:7;1215:21;1216:21;1217:7;1218:7;1219:21;1219226:7;1219227:7;1219242:7;1219261:7;1219263:7;1219285:7;1219286:7;1219299:7;1219322:7;1219324:7;1219325:7;1219326:7;1219345:7;1219362:7;1219365:7;1219369:7;1219374:7;1219392:7;1219393:7;1219395:7;1219397:7;1219398:7;1219427:7;1219440:7;1219462:7;1219464:7;1219465:7;1219472:7;1219473:7;1219474:7;1219476:7;1219477:7;1219513:7;1219531:7;1219548:7;1219558:7;1219595:7;1219659:7;1219661:7;1219662:7;1219663:7;1219690:7;1219696:7;1219707:7;1219728:7;1219733:7;1219736:7;1219738:7;1219755:7;1219756:7;1219757:7;1219759:7;1219762:7;1219763:7;1219764:7;1219766:7;1219769:7;1219778:7;1219779:7;1219785:7;1219787:7;1219791:7;1219803:7;1219836:7;1219838:7;1219844:7;1219845:7;1219852:7;1219853:7;1219864:7;1219865:7;1219866:7;1219872:7;1219873:7;1219874:7;1219878:7;1219879:7;121988:7;121992:7;121993:7;121994:7;1219956:7;1219962:7;1219963:7;1219972:7;1219977:7;1219980:7;1219983:7;1219987:7;1219989:7;1219996:7;1220:21;1223:21;1224:7;1225:7;1226:33;1227:21;1228:7;1229:21;1231:21;1234:21;1235:7;1236:35;1239:21;1240:21;1242:20;1246:4;1248:21;1249:33;1250:35;1250242:10;1250261:10;1250262:10;1250263:10;1250341:10;1250342:10;1250344:10;1250347:10;1250417:10;1250423:10;1250425:10;1250426:10;1250427:10;1250428:10;1250489:10;1250719:10;1250774:10;1250782:10;1250784:10;1250785:10;1250787:10;1250788:10;1251:7;1252:21;1253:17;1254:7;1256:7;1257:35;1260:21;1262:7;1263:33;1264:2;1267:21;1268:3;1269:21;1270:21;1270202:7;1270230:7;1270236:7;1270237:7;1270240:7;1270242:7;1270247:7;1270251:7;1270259:7;1270265:7;1270273:7;1270274:7;1270282:7;1270298:7;1270333:7;1270335:7;1270338:7;1270343:7;1270354:7;1270362:7;1270365:7;1270384:7;1270388:7;1270389:7;1270393:7;1270395:7;1270415:7;1270432:7;1270439:7;1270441:7;1270442:7;1270443:7;1270444:7;1270462:7;1270472:7;1270483:7;1270487:7;1270495:7;1270522:7;1270524:7;1270526:7;1270527:7;1270528:7;1270534:7;1270535:7;1270538:7;1270542:7;1270547:7;1270554:7;1270563:7;1270575:7;1270586:7;1270597:7;1270598:7;1270618:7;1270622:7;1270628:7;1270629:7;1270639:7;1270640:7;1270646:7;1270651:7;1270653:7;1270659:7;1270665:7;1270667:7;1270678:7;127068:7;1270691:7;1270707:7;1270725:7;1270726:7;1270745:7;1270746:7;1270753:7;1270754:7;1270755:7;1270756:7;1270759:7;1270761:7;1270762:7;1270767:7;1270773:7;1270780:7;1270781:7;1270782:7;1270783:7;1270786:7;1270792:7;1270793:7;1270796:7;1270797:7;1270798:7;1270821:7;1270824:7;1270825:7;1270826:7;1270827:7;1270830:7;1270831:7;1270842:7;1270843:7;1270846:7;1270852:7;1270864:7;1270866:7;1270879:7;1270881:7;1270885:7;1270886:7;1270887:7;1270889:7;1270898:7;1270901:7;1270904:7;1270924:7;1270926:7;1270927:7;1270928:7;1270929:7;1270932:7;1270965:7;1270988:7;1272:21;1273:33;1274:7;1276:21;1279:17;1281:7;1283:21;1284:34;1289:33;1301:21;1302:21;1303:8;1304:21;1305:21;1306:26;1307:8;1308:7;1308235:8;130824:8;1308254:8;1308262:8;1308282:8;1308284:8;1308327:8;1308352:8;1308394:8;1308423:8;1308432:8;1308436:8;1308546:8;1308623:8;1308630:8;1308632:8;1308633:8;1308635:8;1308665:8;1308762:8;1308772:8;1308874:8;1308882:8;1309:7;13102:17;13103:17;13104:17;13105:17;13106:17;13107:17;13108:17;13109:17;1312:7;1313:21;1314:7;1315:21;1316:7;1317:21;1318:7;1319:7;1320:7;1321:21;1323:17;1324:21;1325:7;1326:21;1327:7;1329:21;1330:21;1331:7;1332:21;1334:7;1336:21;1337:7;1339:21;1340:31;1341:17;1343:33;1345:6;1346:7;1347:21;1350:17;1351:21;1352:21;1353:7;1354:33;1360:17;1361:7;1363:21;1364:21;1365:33;1367:33;1368:10;1369:17;1380:21;1382:33;1385:8;1386:21;1401:21;1402:7;1403:10;1404:21;1405:7;1406:8;1407:21;1408:17;1409:7;1410:21;1412:21;1413:21;1414:7;1415:17;1416:33;1417:7;1418:33;1418937:14;1418986:14;1419:21;1423:7;1423202:21;1423209:21;1423217:21;1423218:21;1423224:21;142323:21;1423245:21;1423246:21;1423247:21;1423253:21;1423254:21;1423257:21;1423262:21;1423263:21;1423265:21;1423266:21;1423267:21;1423272:21;1423278:21;1423279:21;1423282:21;1423283:21;1423286:21;1423288:21;1423296:21;1423305:21;1423307:21;1423317:21;1423318:21;1423323:21;1423328:21;142333:21;1423343:21;1423344:21;1423345:21;1423346:21;1423349:21;1423357:21;1423365:21;1423378:21;1423391:21;1423392:21;1423396:21;1423402:21;1423421:21;1423422:21;1423424:21;1423425:21;1423431:21;1423434:21;1423439:21;1423442:21;1423443:21;1423451:21;1423453:21;1423467:21;1423468:21;142347:21;1423485:21;1423487:21;142349:21;1423503:21;1423507:21;1423508:21;1423510:21;1423521:21;1423525:21;1423531:21;1423538:21;1423542:21;1423543:21;1423547:21;1423553:21;1423559:21;1423562:21;1423566:21;1423569:21;1423570:21;1423573:21;1423578:21;1423581:21;1423585:21;1423586:21;1423587:21;1423595:21;1423602:21;1423610:21;1423613:21;1423614:21;142362:21;1423631:21;1423634:21;1423636:21;1423638:21;1423639:21;1423643:21;1423648:21;1423652:21;1423663:21;1423664:21;1423667:21;1423697:21;1423698:21;1423702:21;1423708:21;1423710:21;1423722:21;1423725:21;1423727:21;1423728:21;1423733:21;1423735:21;1423743:21;1423744:21;1423745:21;1423746:21;1423752:21;1423753:21;1423756:21;1423757:21;1423760:21;1423764:21;1423765:21;1423768:21;1423772:21;1423775:21;1423778:21;1423784:21;1423787:21;1423788:21;1423790:21;1423794:21;1423798:21;1423800:21;1423802:21;1423803:21;1423805:21;1423821:21;1423822:21;1423825:21;1423826:21;1423839:21;1423842:21;1423843:21;1423844:21;1423847:21;1423854:21;1423855:21;1423857:21;1423867:21;1423869:21;1423870:21;1423875:21;1423876:21;1423877:21;1423878:21;1423884:21;1423886:21;1423892:21;1423893:21;1423894:21;1423899:21;1423907:21;1423913:21;1423915:21;1423921:21;1423926:21;1423928:21;1423929:21;1423933:21;1423952:21;1423954:21;1423968:21;1423975:21;1423979:21;1423989:21;1424:17;1425:17;1428:14;1430:7;1431:36;1432:7;1434:21;1435:8;1437:33;1438:33;1440:21;1441:37;1442:17;1443:21;1445:21;1447:7;1448:21;1450:33;1458:17;1463:21;1464:7;1468:33;1469:7;1470:21;1472:21;1473:13;1474:36;1475:21;1478:21;1479:7;1480:23;1484:21;1501:7;1502:21;1503:17;1504:7;1505:8;1506:14;1507:7;1508:21;1509:17;1510:17;1512:7;1513:21;1514:33;1515:7;1516:21;1517:21;1518:21;1519:33;1520:23;1530:17;1531:7;1534:7;1539:7;1540:21;1541:17;1541372:8;1541473:8;1541881:8;1541889:8;1548:33;1551:21;1557:7;1559:17;1561:21;1562:17;1563:7;1564:17;1567:21;1570:21;1571:21;1572:7;1573:7;1574:21;1574772:7;1574896:7;1575:8;1579:33;1580:7;1581:33;1582:21;1584:36;1585:21;1586:21;1587:10;1601:7;1602:23;1603:21;1604:35;1605:8,22;1605201:7;1605209:8;1605217:7;1605221:7;1605223:8;1605224:7;1605225:7;1605226:7;1605229:7;1605232:7;1605234:7;1605255:8;1605256:7;1605260:7;1605261:7;1605262:7;1605271:7;1605274:7;1605275:7;1605279:8;1605284:7;1605297:7;1605310:7;1605312:7;1605321:7;1605322:7;1605323:7;1605328:7;160533:7;1605341:8;1605342:8;1605343:8;1605345:7;1605347:8;1605348:8;160535:7;1605355:8;160536:7;1605370:7;1605371:7;1605373:7;1605374:8;1605376:7;1605381:8;1605384:7;1605387:7;1605388:8;1605390:8;1605391:8;1605393:8;1605394:8;1605397:7;1605399:8;1605415:8;1605425:7;1605426:7;1605428:7;1605430:8;1605431:8;1605432:7;1605448:7;1605455:8;1605472:7;1605484:8;1605487:7;1605498:7;1605528:7;1605532:7;1605539:7;1605543:7;1605553:7;1605574:8;1605578:8;1605582:7;1605584:8;1605589:7;1605593:8;1605594:7;1605598:7;1605610:7;1605622:7;1605624:7;1605626:7;1605627:7;1605641:8;1605642:8;1605644:8;1605647:7;1605649:7;1605665:7;1605666:8;1605668:7;1605669:7;1605673:8;1605677:7;1605685:8;160569:7;1605716:8;1605717:8;1605718:8;1605719:8;1605720:8;1605721:8;1605722:8;1605723:8;1605724:7;1605725:7;1605728:7;1605734:7;1605743:7;1605745:8;1605747:7;1605753:7;1605755:8;1605759:7;1605763:7;1605765:7;1605772:7;1605773:7;1605775:7;1605778:7;1605787:8;1605791:8;1605796:7;1605823:8;1605835:7;1605837:8;1605842:7;1605845:7;1605852:7;1605853:7;1605854:7;1605856:7;1605859:8;1605867:8;1605874:7;1605878:7;1605881:7;1605882:7;1605886:7;1605892:8;1605923:8;1605925:7;1605928:7;1605929:7;1605940:7;1605941:7;1605945:7;1605946:7;1605951:7;1605964:8;1605977:7;1605983:7;1605987:7;1605988:7;1605990:7;1605995:7;1605996:7;1605997:7;1606:21;1606387:7;1607:21;1608:7;1609:21;1610:21;1612:7;1613:33;1614:21;1615:7;1616:21;1617:21;1618:7;1619:17;1620:7;1620376:8;1620384:8;1623:23;1626:17;1628:17;1629:7;1630:7;1631:21;1636:7;1639:26;1640:21;1641:7;1645:21;1646:21;1647:33;1649:12;1650:17;1651:7;1656:21;1657:17;1658:15;1659:7;1660:7;1661:17;1662:7;1664:19;1667:21;1669:17;1670:41;1671:38;1672:35;1678:21;1680:21;1681:21;1682:7;1683:33;1684:40;1686:21;1689:7;1701:8,22;1701200:7;1701205:7;1701212:7;1701214:7;170122:7;1701225:8;1701227:8;170123:7;1701241:7;1701242:7;170125:7;1701261:7;1701265:7;1701271:7;1701277:7;1701280:7;1701281:7;1701282:7;1701284:7;1701288:7;1701290:8;1701293:7;1701297:7;1701298:7;1701306:7;1701323:7;1701324:7;1701328:7;1701343:7;1701347:7;1701349:7;1701352:7;1701355:7;1701356:7;1701361:7;1701364:7;1701365:7;1701367:7;1701371:7;1701373:7;1701385:7;1701388:7;1701390:7;1701391:7;1701400:7;1701425:7;1701426:7;1701428:7;1701437:7;1701444:7;1701446:7;1701448:7;1701452:7;1701454:7;1701456:8;1701462:7;1701463:7;1701471:7;1701476:7;1701477:7;1701478:7;1701483:8;1701492:7;1701499:7;1701523:8;1701527:7;1701530:7;1701532:7;1701540:7;1701549:7;1701566:7;1701567:8;1701572:7;1701575:8;1701577:7;1701584:8;1701595:7;1701609:7;1701623:8;1701627:7;1701628:7;1701636:7;1701642:7;1701652:7;1701662:7;1701663:7;1701664:7;1701667:7;1701683:7;1701724:7;1701730:7;1701738:7;1701739:7;1701742:7;1701746:7;1701748:7;1701751:7;1701754:7;1701756:7;1701757:7;1701764:8;1701766:7;1701772:7;1701774:7;1701775:7;1701776:7;1701777:7;1701780:7;1701786:7;1701787:7;1701788:7;1701793:7;1701795:7;1701797:7;1701799:7;1701824:8;1701833:7;1701837:7;1701838:7;1701839:7;1701842:7;1701843:7;1701845:7;1701852:7;1701854:7;1701857:7;1701858:7;1701866:7;1701872:8;1701873:7;1701883:7;1701893:7;1701947:7;1701952:7;1701965:7;1701968:7;1702:17;1703:21;1704:21;1705:33;1706:21;1707:17;1708:7;1709:14,28;1712:7;1713:7;1714:17;1715:7;1716:21;1717:21;1718:21;1719:8;1720:8;1721:18;1724:21;1725:17;1726:7;1727:21;1728:21;1730:7;1731:7;1732:21;1734:21;1737:7;1738:17;1740:21;1742:33;1743:21;1747:17;1748:8;1753:33;1754:21;1757:21;1758:30;1760:17;1762:21;1763:7;1765:21;1767:9;1769:7;1770:21;1771:21;1772:21;1773:7;1774:21;1775:5,17;1775200:17;1775230:17;1775232:17;1775233:17;1775237:17;1775240:17;1775246:17;1775250:17;1775265:17;1775267:17;1775273:17;1775284:17;1775287:17;1775289:17;1775298:17;1775313:17;177532:17;177533:17;1775345:17;1775348:17;177535:17;1775360:17;1775376:17;1775384:17;1775391:17;1775392:17;1775423:17;1775424:17;1775425:17;1775428:17;1775432:17;1775445:17;1775450:17;1775453:17;1775461:17;1775463:17;1775470:17;1775473:17;1775482:17;1775525:17;1775537:17;1775544:17;1775575:17;1775577:17;1775586:17;1775588:17;1775622:17;1775623:17;1775624:17;1775625:17;1775626:17;1775635:17;1775636:17;1775657:17;1775664:8;1775673:17;1775674:17;1775677:17;1775684:17;1775686:17;1775687:17;1775688:17;1775689:17;1775720:17;1775721:17;1775722:17;1775726:17;1775727:17;1775737:17;1775738:17;1775742:17;1775746:17;1775747:17;1775750:17;1775751:17;1775752:17;1775753:17;1775762:17;1775770:17;1775771:17;1775772:17;1775777:17;1775778:17;1775782:17;1775783:17;1775784:17;1775786:17;1775787:17;1775800:17;177582:17;1775830:17;1775831:17;1775832:17;1775833:17;1775835:17;1775841:17;1775843:17;1775847:17;1775849:17;177585:17;1775867:17;1775870:17;177588:17;1775945:17;1775971:17;1775972:17;1775982:17;1778:35;1779:7;1780:10;1781:21;1782:14;1784:32;1785:7;1785852:8;1785890:8;1785899:8;1786:21;1787:25;1801:8;1802:21;1803:21;1804:21;1805:17;1806:7;1807:33;1807223:36;1807274:36;1807467:36;1807468:36;1807482:36;1807548:36;1807727:36;1807737:36;1807934:36;1808:39;1809:27;1810:21;1812:21;1812385:7;1812386:7;1812401:7;1812402:7;181242:7;1812435:7;1812437:7;1812450:7;1812464:7;181247:7;1812485:7;1812490:7;1812491:7;1812547:7;1812618:7;1812649:7;1812682:7;1812749:7;1812753:7;1812768:7;1812838:7;1812842:7;1812853:7;1812858:7;1812867:7;1812874:7;1812897:7;1812925:7;1812937:7;1812962:7;1812963:7;1812985:7;1813:21;1814:21;1815:7;1816:7;1817:7;1818:17;1819:33;1820:17;1821:21;1825:10;1826:21;1828:21;1829:27;1830:7;1831:17;1832:7;1835:21;1838:21;1839:21;1840:17;1843:21;1845:21;1847:7;1848:21;1849:27;1850:21;1850200:7;1850206:7;1850208:7;1850213:7;1850215:7;1850217:7;1850218:7;1850221:7;1850225:7;1850226:7;1850227:7;1850229:7;185023:7;1850240:7;1850243:7;1850244:7;1850248:7;1850249:7;1850250:7;1850256:7;1850258:7;1850259:7;1850261:7;1850263:7;1850265:7;1850267:7;1850269:7;1850271:7;1850276:7;1850279:7;1850291:7;1850292:7;1850301:7;1850306:7;1850314:7;1850315:7;1850319:7;1850324:7;1850327:7;1850332:7;1850341:7;1850361:7;1850362:7;1850368:7;1850376:7;1850377:7;1850380:7;1850384:7;1850387:7;1850393:7;1850396:7;1850398:7;1850416:7;1850417:7;1850420:7;1850423:7;1850424:7;1850429:7;185043:7;1850444:7;185045:7;1850460:7;1850462:7;1850466:7;1850469:7;185047:7;1850481:7;1850482:7;1850484:7;1850492:7;1850494:7;1850496:7;1850497:7;1850499:7;1850501:7;1850502:7;1850505:7;1850516:7;1850522:7;1850525:7;1850526:7;1850527:7;1850529:7;1850535:7;1850537:7;1850543:7;1850547:7;1850572:7;1850581:7;1850585:7;1850586:7;1850587:7;1850588:7;1850592:7;1850593:7;1850595:7;1850596:7;1850607:7;1850622:7;1850623:7;1850624:7;1850626:7;1850637:7;1850638:7;1850639:7;1850640:7;1850650:7;1850651:7;1850654:7;1850659:7;1850664:7;1850674:7;1850675:7;1850677:7;1850678:7;185068:7;1850696:7;1850699:7;1850708:7;1850712:7;1850722:7;1850723:7;1850729:7;1850733:7;1850747:7;1850748:7;1850763:7;1850769:7;1850784:7;1850785:7;1850791:7;1850796:7;1850797:7;1850814:7;1850819:7;1850830:7;1850832:7;1850833:7;1850835:7;1850837:7;1850855:7;1850857:7;185086:7;1850871:7;1850872:7;1850874:7;1850883:7;1850890:7;1850892:7;1850897:7;1850898:7;1850912:7;1850913:7;1850914:7;1850916:7;1850932:7;1850934:7;1850936:7;1850937:7;1850939:7;1850941:7;1850944:7;1850951:7;1850968:7;1850969:7;1850972:7;1850974:7;1850981:7;1850982:7;1850983:7;1850994:7;1850995:7;1854:21;1856:21;1857:21;1858:17;1859:21;1860:21;1862:21;1863:21;1864:21;1865:21;1867:11;1867334:35;1867393:35;1867456:35;1867536:35;1867587:10;1867633:35;1867645:36;1867667:35;1867668:35;1867669:10;1867695:10;1867777:10;1867872:10;1867873:10;1867874:10;1867920:10;1867979:33;1867983:10;1867993:35;1868:24;1869:29;1870:7;1872:7;1873:33;1876:15;1878:21;1879:25,28;1901:7;1902:14;1903:7;1904:21;1905:33;1906:21;1906265:7;1906563:7;1906663:7;1906753:7;1906774:7;1906776:7;1906779:7;1906828:7;1906863:7;1906864:7;1906875:7;1906932:7;1907:0,1;1907209:16;1907212:16;190722:16;1907230:16;1907232:16;1907235:16;190724:16;1907250:16;1907252:16;1907257:16;1907258:16;1907260:16;1907262:16;1907264:16;1907269:16;190727:16;1907283:16;1907299:16;1907301:16;1907306:16;1907317:16;190733:16;1907343:16;1907344:16;1907345:16;1907346:16;1907349:16;1907350:16;1907351:16;1907352:16;1907357:16;1907360:16;1907373:16;1907374:16;1907375:16;1907376:16;1907378:16;1907388:16;1907424:16;1907442:16;1907443:16;190745:16;1907463:16;1907465:16;1907474:16;1907479:16;1907486:16;1907487:16;1907488:16;1907490:16;1907495:16;1907522:16;1907523:16;1907543:16;1907561:16;1907562:16;1907563:16;1907567:16;1907569:16;1907580:16;1907581:39;1907586:16;1907622:16;1907631:16;1907644:16;1907646:16;1907677:16;1907683:16;1907688:16;1907694:16;1907696:16;1907714:16;1907723:16;1907727:16;1907729:16;1907733:16;1907742:16;1907743:16;1907745:16;1907746:16;1907747:16;1907766:16;1907770:16;1907772:16;1907776:16;1907780:16;1907783:16;1907786:16;1907789:16;1907790:16;1907822:16;1907826:16;1907830:16;1907835:16;1907841:16;1907842:16;1907852:16;1907868:16;1907874:16;1907883:16;1907892:16;1907895:16;1907929:16;1907966:16;1907983:16;1908:21;1909:17;1910:21;1912:21;1913:7;1914:21;1915:8;1916:17;1917:21;1918:7;1919:21;1920:7;1925:17;1928:23;1928283:8,23;1928672:8,23;1928673:8,23;1928674:8,23;1928677:8,23;1928686:8,23;1928697:8,23;1928698:8,23;1928724:8,23;1928725:8,23;1928728:8,23;1928729:8,23;1928755:8,23;1928781:8,23;1928787:8,23;1928871:8,23;1929:21;1930:21;1931:7;1934:21;1936:7;1937:21;1938:7;1939:25;1940:7;1941:21;1942:33;1943:21;1945:7;1947:21;1948:21;1949:17;1951:17;1952:7;1954:21;1956:7;1959:21;1970:8;1971:17;1972:7;1973:21;1975:7;1978:21;1979:7;1980:21;1983:8;1984:21;1985:7;1986:5,17;1989:21;20:42;211:43;212:44;213:45;216:46;218:47;220:48;221:49;222:50;223:51;224:52;225:53;226:54;227:55;228:56;229:57;230:58;231:59;232:60;233:61;234:62;235:63;236:64;237:65;238:66;239:67;240:68;241:69;242:70;243:71,72;2431:71;2432:72;2433:71;2434:71;2435:71;2436:71;2438:71,72;2439:71,72;244:73;245:59;246:74;247:75;248:76;249:77;250:78;251:79;252:80;253:81;254:43;255:82;256:83;257:84;258:85;260:86;261:87;262:88,89;262262:89;262263:89;2622688:89;2622689:88;262269:88;26263:88;26269:89;2627092:89;2627093:88;26280:88,89;26281:89;26282:89;26288:89;26289:89;2629398:88;2629399:89;2629478:88;2629479:89;2629762:89;2629763:89;2629769:88;263:90;264:91;265:92;266:93;267:94;268:95;269:96;27:97;290:75;291:98;297:99;298:100;299:101,102,103;2991:101;2992:101;2993:101;2994:101;2995:101;2996:101;2998:101;2999:101;29997:103;29999:102;30:104;31:105;32:106;33:45;34:44,107;344:107;345:107;346:107;347:107;348:107;34822:44;34827:44;34828:44;349:107;34922:44;34928:44;350:108;351:109,110;3511:109,110;3512:110;351292:109;351295:109;351296:109;3513:109,110;3516:109,110;3517:109,110;3518:109,110;3519:109,110;352:111;353:112,113,114;353539253:112,113,114;354:59;355:115;356:116;357:117;358:118,119;3581:118;35810:118,119;35818:119;3582:118;35820:118,119;35829:118,119;3583:118;35830:118,119;35839:118,119;3584:118,119;3585:118;35850:118,119;3586:118;35860:118,119;3587:118,119;3588:118;35880:118,119;3589:118;359:120;36:121;370:47;371:47;372:47;373:122;374:123;375:124;376:125;377:126;378:127;380:128;381:129;382:130;383:129;385:131;386:132;387:133;389:134;39:135,136;390:135;3906698:136;390878:135,136;391:135,136;393:135,136;394:135;395:135,136;397:135;398:135,136;40:47;41:137;420:138;421:139;423:140;43:141;44:112,113,142,114;441:114;441481:112;441534:142;441624:113;442:114;443:112,113,114;445:112,113,114;4470:112,113,114;4471:112,113,114;4472:112,113,114;4473:112,113,114;4474:112,113,114;4475:112,113,114;44760:112,113,114;44762:112,113,114;44763:112,113,114;447640:112,113,114;447641:112,113,114;447643:112,113,114;447644:112,113,114;447646:112,113,114;44765:112,113,114;44766:112,113,114;44767:112,113,114;44768:112,113,114;447693:112,113,114;447699:112,113,114;4477:112,113,114;4478:112,113,114;4479:112,113,114;448:112,113,114;449:112,113,114;45:143;46:144;47:145,146;470:146;472:146;473:146;474:146;475:146;476:146;477:146;4779:145;478:146;479:146;48:147;49:148;500:149;501:150;502:151;503:152;504:153;505:7;506:154;507:155;508:156;509:157;51:158;52:159,160,161,162,163,21,164,165;522:163;52200:163,165;52201:163,21;523:163;52311:162;52319:162;52323:162;52324:162;52325:162;52327:162;52389:162;524:163;525:163;52612:162;52613:162;52614:162;52615:162;52616:165;52618:163;5262:162;52626:164;5263:162;5264:162;52646:165;52649:162,163;52651:162;52652:162;52653:162;52656:160;52657:162;52658:165;52659:162;52660:162;52661:165;52662:162;52663:163;52664:165;52665:165;52667:162;52668:162;52669:162;52671:163;52672:162;52673:162;52674:163;52675:163;52676:163;52677:163;52686:165;52687:162;5269:162;527:163;528:163;52867:161;52877:161;52878:161;52894:161;52899:161;529:163;52983:159;52984:159;52987:159;52998:159;53:166;54:167;55:168,169,170,171;551:171;552:171;553:171;554:171;5550:171;5551:171;5553:171;5554:171;5555:171;55553682:168;5561:171;5562:171;5563:171;5564:171;5565:168;5566:168;5567:168;55673480:171;5568:170;5569:168;55693527:171;557:171;558:171;55812:169;5581300:169;55813013:169;55813016:169;55813017:169;55813018:169;55813022:169;55813024:169;55813027:169;55813029:169;55813042:169;55813043:169;55813044:169;55813047:169;55813054:169;55813056:169;55813057:169;55813058:169;5581306:169;55813070:169;55813085:169;55813098:169;558131:169;55813200:169;55813206:169;55813208:169;55813209:169;55813210:169;55813211:169;55813215:169;55813218:169;55813219:169;55813220:169;5581323:169;55813240:169;55813245:169;55813246:169;55813247:169;55813248:169;55813250:169;55813258:169;55813259:169;55813260:169;55813261:169;55813262:169;55813263:169;5581327:169;5581328:169;5581329:169;55813300:169;55813306:169;55813307:169;55813308:169;55813309:169;55813310:169;55813313:169;55813315:169;55813321:169;55813324:169;55813329:169;5581333:169;55813340:169;55813346:169;55813347:169;55813348:169;55813349:169;5581335:169;5581336:169;55813370:169;55813373:169;55813374:169;5581338:169;5581339:169;5581340:169;55813410:169;55813415:169;55813417:169;55813418:169;55813420:169;55813430:169;55813440:169;55813450:169;55813460:169;55813470:169;55813480:169;55813485:169;55813488:169;55813489:169;55813490:169;55813496:169;558135:169;5581360:169;5581361:169;55813620:169;55813623:169;55813630:169;55813632:169;55813640:169;55813650:169;55813659:169;5581366:169;55813670:169;55813672:169;55813674:169;55813677:169;55813680:169;55813690:169;55813696:169;55813697:169;55813698:169;558137:169;558138:169;558139:169;55814:169;55815:169;55817:169;55819:169;559:168;5590:171;55912121:171;55912122:171;55912123:171;55912992:171;55913004:171;5591301:171;55913028:171;55913031:171;55913032:171;55913033:171;55913038:171;55913039:171;55913072:171;55913073:171;55913075:171;5591308:171;5591311:171;55913120:171;55913121:171;55913131:171;55913181:171;55913182:171;55913184:171;55913194:171;55913199:171;559132:171;55913311:171;55913321:171;55913322:171;55913323:171;5591334:171;55913351:171;55913352:171;55913353:171;55913355:171;55913366:171;55913397:171;55913411:171;55913412:171;55913423:171;55913425:171;55913429:171;55913434:171;5591344:171;55913456:171;5591346:171;55913481:171;55913482:171;55913483:171;55913484:171;55913485:171;55913494:171;55913521:171;55913556:171;55913605:171;55913606:171;55913617:171;55913621:171;55913633:171;55913636:171;55913637:171;55913658:171;55913661:171;55913662:171;55913665:171;55913692:171;55913694:171;55913711:171;55913712:171;5591372:171;5591373:171;55913741:171;55913744:171;55913746:171;5591375:171;55913764:171;55913765:171;55913767:171;5591377:171;55913781:171;55913783:171;55913784:171;55913795:171;55913796:171;55913798:171;55913802:171;55913803:171;55913809:171;55913811:171;55913812:171;55913817:171;55913821:171;55913822:171;55913823:171;55913829:171;5591400:171;55914040:171;55914042:171;55914103:171;55914104:171;55914107:171;55914141:171;559198:171;5591991:171;5591992:171;5591993:171;55919960:171;55919961:171;55919962:171;55919963:171;55919964:171;5591998:171;5591999:171;55925009:171;55932101:171;55932122:171;55933016:171;55933017:171;55933062:171;55933063:171;55933064:171;55933067:171;55933222:171;55933502:171;55933505:171;55933512:171;55933514:171;55933515:171;55933517:171;55933518:171;5593352:171;5593353:171;5593354:171;55933552:171;55933557:171;55933558:171;55933559:171;55933563:171;55933574:171;55933582:171;55933589:171;55933593:171;55933596:171;55933597:171;55933598:171;55933735:171;55933736:171;55933737:171;55933793:171;55939810:171;55939811:171;55939812:171;55939813:171;55939840:171;55939841:171;55939880:171;55939881:171;5593991:171;55939920:171;55939921:171;55939922:171;55939923:171;55939924:171;55939965:171;55939990:171;55939991:171;55939995:171;55939997:171;55942101:171;55942103:171;55943012:171;55943013:171;55943016:171;55943018:171;55943222:171;55943301:171;55943309:171;55943311:171;55943312:171;55943315:171;55943319:171;5594332:171;55943331:171;55943332:171;55943333:171;55943335:171;55943337:171;5594334:171;5594335:171;55943364:171;55943365:171;55943366:171;55943379:171;55943382:171;55943385:171;55943386:171;55943392:171;5594342:171;55943431:171;55943432:171;55943433:171;55943434:171;55943435:171;55943491:171;55943778:171;55943779:171;55943785:171;55943786:171;55943787:171;5594981:171;55949840:171;55949841:171;55949842:171;55949880:171;55949881:171;5594991:171;5594992:171;55949930:171;55949966:171;55949990:171;55949991:171;55949992:171;55949995:171;55949997:171;5596:171;5598:171;5599:171;56:172;561:172;562:172;563:172;564:172;565:172;566:172;567:172;568:172;569:172;57:173;58:174;590:175;591:176;592:177;593:178,179;5931:178,179;5932:178;5933:178;5934:178;5935:178;59352:178,179;5936:178;5937:178;5939:178;594:180;595:181;596:182;597:183;598:184;599:185,186;5993:186;5994:186;5996:185;5997:186;5999:185;60:187;61:188,189,190,191,192,193,194,195;611:193;612:193;61255053:191;61255143:191;61255213:191;61255280:191;61255307:191;61255369:191;61255413:191;61255568:191;61255936:191;61255937:191;61265080:191;61265081:191;61265082:191;61265099:191;61265164:191;61265251:191;61265296:191;61265341:191;61265606:191;61265632:191;61265998:191;613:193;614:193;617:193;61721000:189;61721001:189;61721002:189;61721003:189;6172101:189;6172102:189;6172103:189;6172104:189;6173018:189;6173019:189;617303:189;6173040:189;6173041:189;6173046:189;617305:189;61730600:189;6173061:189;61730640:189;61730641:189;61730643:189;61730651:189;61730659:189;6173066:189;6173067:189;6173068:189;6173069:189;617307:189;617308:189;61730923:189;61730938:189;6173096:189;6173097:189;61730992:189;61731:189;617322:189;617323:189;617326:189;6173291:189;6173292:189;6173295:189;6173296:189;6173300:189;61733095:189;61733099:189;6173316:189;6173319:189;6173322:189;6173330:189;6173333:189;6173342:189;617335:189;617336:189;6173403:189;6173404:189;6173405:189;6173406:189;6173407:189;6173411:189;6173414:189;617342:189;6173434:189;6173435:189;6173443:189;6173452:189;6173455:189;6173456:189;6173457:189;6173460:189;6173493:189;61734973:189;61734974:189;61734975:189;61734976:189;617350:189;617353:189;6173540:189;6173546:189;6173547:189;6173548:189;617355:189;61735678:189;617360:189;6173613:189;617362:189;617363:189;61736404:189;61736405:189;61736406:189;61736409:189;6173646:189;6173647:189;6173648:189;6173666:189;61736677:189;61736678:189;61736679:189;61736680:189;617370:189;617371:189;617372:189;617373:189;6173748:189;6173777:189;61737788:189;6173811:189;6173815:189;6173823:189;6173828:189;61738348:189;61738359:189;6173887:189;617389:189;6173900:189;6173901:189;6173904:189;6173905:189;6173910:189;61739110:189;61739111:189;61739112:189;6173914:189;6173915:189;61739160:189;61739161:189;6173917:189;6173918:189;6173922:189;6173924:189;61739475:189;6173999:189;618510:188,192;61851001:194;61851002:195;61851030:194;61851031:195;61851059:194;61851060:195;61851087:192;61851088:194;61851089:195;6185109:192;618511:192;61851100:188,192;61851116:188,192;61851117:194;61851118:195;61851119:188,192;6185112:188,192;6185113:188,192;61851140:188,192;61851141:188,192;61851142:188,192;61851143:188,192;61851144:188,192;61851146:194;61851156:188,192;61851158:188,192;61851175:194;61851176:195;61851185:188,192;61851187:188,192;6185120:192;61851204:194;61851205:193;61851209:188,192;6185121:188,192;61851221:188,192;61851222:194;61851223:195;61851224:192;61851225:192;61851226:192;61851227:188,192;61851228:188,192;61851229:188,192;61851230:193;61851231:188,192;61851232:193;61851233:193;61851234:193;61851235:194;61851236:188,192;61851237:188,192;61851238:188,192;61851239:188,192;6185124:193;6185125:193;6185126:193;6185127:188,192;6185128:188,192;6185129:188,192;618513:193;618514:193;61855:188,192;6186:188,192;6187:188,192;6188:188,192;6189:188,192;61890393:188,190,192;61890395:188,190,192;61891006:188,192,194;61891007:188,192,194;61891008:188,192,194;61891009:188,192,194;61891010:188,192,195;61891011:188,192,195;61891012:188,192,195;61891029:188,192,195;61891122:188,192,195;61891149:188,192,194;61891150:188,192,195;61891178:188,192,194;61891179:188,192,195;61891181:188,192,194;61891182:188,192,195;61891209:188,192,194;61891210:188,192,195;61891263:188,192,194;61891264:188,192,195;61891308:188,192,195;61891312:188,192,194;61891322:188,192,195;61891326:188,192,194;61891368:188,192,195;61891375:188,192,194;61891428:188,192,195;61891456:188,192,194;61891497:188,192,194;61891498:188,192,195;6189162:195;6189164:194;61891700:188,192,194;61891701:188,192,194;61891702:188,192,195;61891703:195;61891710:188,192,194;61891711:188,192,194;61891712:188,192,194;61891958:194;61891959:188,192,195;62:196,197,198;62007:196,198;621:196,198;622:196;6231:196;6232:196;6233:196;6234:196;6235:196;6236:198;6237:198;62379:196,198;6238:198;624:198;62403:197,198;62458:196,198;62511:198;62512:198;62513:196;62515:196;62516:196,198;62517:198;62518:198;62519:196;62522:196;62525:196;62526:198;62527:198;62528:196;6253:196;62539:196,198;6254:198;6255:198;6256:196;626:196;627:196;628:196;6280:196,198;62811:196,198;62818:196,198;62819:196,198;62824:196,198;62829:196,198;62831:196,198;62833:196,198;62834:196,198;62839:196,198;62853:196,198;62868:196,198;62876:196,198;62887:196,198;6290:197;6291:197;62915:196,197;6292:197;62920:196;62951:197;62952:196,197;62955:197;62956:197;62957:197;62958:196;62962:197;62963:196;62966:197;62967:197;62969:197;62971:197;62975:197;62979:196;6298:197;63:199;64:200,201;641:200;642:200;643:200;64330:201;644:200;645:200;646:200;647:200;648:200;649:200;65:202;66:203;670:204;672:205;673:206;674:207;675:208;676:209;677:210;678:211;679:212;680:213;681:214;682:215;683:216;685:217;686:218,219,220;687:221;688:222;689:223,224,225;6894:225;68980:223,224,225;68987:223,224,225;68988:225;68989:225;690:226;691:227,228,229;6913:229;69132:228;69137:227;6918:227,228,229;6919:227,228,229;691920:228;691921:228;691924:228;691926:228;691933:229;692:230;7:231,232,233,234,235,236,237,238,239,240,241,242,243,47,124,244;7301:234;7302:242;733:243;734:243;7341:244;735:243;736:124;7381:239;7382:238;7383:238;7384:236;7385:236;7388:236;7390:236;7391:236;7394:236;7395:234;740:47;7411:242;7413:237;7415:235;7416:242;7421:241;7423:241;7424:240;7426:241;7427:232;747:124;748:124;749:124;77:231,233;771:231;7711:233;7712:233;7713:233;772:231;7729:233;779:231;7800:231,233,124;7803:124;7804:124;7808:231,233,124;7809:231,233,124;781:124;7810:231,233,124;782:124;783:124;7840:47,124;7841:124;7842:244;7843:124;7844:124;7845:124;7846:244;7847:124;7848:244;7851:244;7855:124;7856:124;7857:124;786:124;787:124;79:124;81:245;8170859:245;82:246;84:247;850:246;852:248;853:249;855:250;856:251;86:249;861:249;862:249;863:249;864:249;865:249;866:249;867:249;868:249;869:249;880:252;886:253;90:254;91:255;92:256;93:257;94:258;95:259;960:260;961:261;962:262;963:263;964:264;965:265;966:266;967:267;968:268;970:47;971:269;972:270;973:271;974:272;975:273;976:274,275,276;97611:276;97612:276;97613:276;9761362:274,276;97614:275;97615:276;976158:274;97621:276;97622:276;97623:276;9762362:274,276;97624:275;97625:276;976258:274;97626:276;9765:276;9766:276;9767:276;9768:276;9769:276;977:277;98:278;992:279;993:280;994:281;995:282;996:283;998:284"; diff --git a/src/domains/callHours/helpers/zones.ts b/src/domains/callHours/helpers/zones.ts new file mode 100644 index 0000000..7995ed5 --- /dev/null +++ b/src/domains/callHours/helpers/zones.ts @@ -0,0 +1,231 @@ +import { MAX_ZONE_SPREAD_HOURS, type ProspectZones } from "../types"; +import { PHONE_PREFIXES, PHONE_ZONE_NAMES } from "./phoneZones.data"; + +/** Is this an IANA time zone this runtime knows? */ +export function isTimeZone(name: string | undefined): name is string { + if (!name?.trim()) return false; + try { + new Intl.DateTimeFormat("en-US", { timeZone: name.trim() }); + return true; + } catch { + return false; + } +} + +let prefixes: { map: Map; longest: number } | undefined; + +/** The zones of an E.164 number (digits only, country code first), by its longest known prefix. */ +export function zonesForNumber(digits: string): string[] | undefined { + if (!prefixes) { + const map = new Map(); + let longest = 0; + for (const entry of PHONE_PREFIXES.split(";")) { + const [prefix, ids] = entry.split(":") as [string, string]; + map.set(prefix, ids.split(",").map((i) => PHONE_ZONE_NAMES[Number(i)]!)); + longest = Math.max(longest, prefix.length); + } + prefixes = { map, longest }; + } + for (let n = Math.min(digits.length, prefixes.longest); n >= 1; n--) { + const zones = prefixes.map.get(digits.slice(0, n)); + if (zones) return zones; + } + return undefined; +} + +interface Country { + /** Calling code. */ + code: string; + /** National numbers start with a 0 that the international form drops. */ + trunk: boolean; + names: string[]; +} + +/** Countries whose numbers COLD can read without a + in front, by ISO code or name. */ +const COUNTRIES: Country[] = [ + { code: "1", trunk: false, names: ["us", "usa", "united states", "united states of america", "america", "ca", "can", "canada", "pr", "puerto rico"] }, + { code: "44", trunk: true, names: ["gb", "uk", "united kingdom", "great britain", "britain", "england", "scotland", "wales", "northern ireland"] }, + { code: "353", trunk: true, names: ["ie", "irl", "ireland"] }, + { code: "61", trunk: true, names: ["au", "aus", "australia"] }, + { code: "64", trunk: true, names: ["nz", "new zealand"] }, + { code: "27", trunk: true, names: ["za", "south africa"] }, + { code: "49", trunk: true, names: ["de", "deu", "germany"] }, + { code: "33", trunk: true, names: ["fr", "fra", "france"] }, + { code: "31", trunk: true, names: ["nl", "netherlands", "holland"] }, + { code: "32", trunk: true, names: ["be", "belgium"] }, + { code: "41", trunk: true, names: ["ch", "switzerland"] }, + { code: "43", trunk: true, names: ["at", "austria"] }, + { code: "46", trunk: true, names: ["se", "sweden"] }, + { code: "358", trunk: true, names: ["fi", "finland"] }, + { code: "34", trunk: false, names: ["es", "spain"] }, + { code: "39", trunk: false, names: ["it", "italy"] }, + { code: "351", trunk: false, names: ["pt", "portugal"] }, + { code: "45", trunk: false, names: ["dk", "denmark"] }, + { code: "47", trunk: false, names: ["no", "norway"] }, + { code: "48", trunk: false, names: ["pl", "poland"] }, + { code: "52", trunk: false, names: ["mx", "mexico"] }, + { code: "91", trunk: true, names: ["in", "india"] }, + { code: "65", trunk: false, names: ["sg", "singapore"] }, + { code: "63", trunk: true, names: ["ph", "philippines"] }, + { code: "971", trunk: true, names: ["ae", "uae", "united arab emirates"] }, +]; + +function findCountry(name: string | undefined): Country | undefined { + const key = name?.trim().toLowerCase().replace(/\./g, ""); + return key ? COUNTRIES.find((c) => c.names.includes(key)) : undefined; +} + +/** + * A phone number as E.164 digits: written with + or 00 in front, or in the + * national form of the prospect's country (US and Canada when none is + * given). Undefined when it can't be read. Pure. + */ +export function phoneDigits(phone: string | undefined, countryName?: string): string | undefined { + const raw = phone?.trim() ?? ""; + const digits = raw.replace(/\D/g, ""); + if (digits.length < 7) return undefined; + if (raw.startsWith("+")) return digits; + if (raw.startsWith("00")) return digits.slice(2); + const country = findCountry(countryName); + if (countryName?.trim() && !country) return undefined; + if (!country || country.code === "1") { + if (digits.length === 10 && /^[2-9]/.test(digits)) return `1${digits}`; + if (digits.length === 11 && digits.startsWith("1")) return digits; + if (raw.startsWith("011")) return digits.slice(3); + return undefined; + } + if (country.trunk && digits.startsWith("0")) return country.code + digits.slice(1); + if (digits.startsWith(country.code) && digits.length >= country.code.length + 8) return digits; + return country.trunk ? undefined : country.code + digits; +} + +const ET = "America/New_York"; +const CT = "America/Chicago"; +const MT = "America/Denver"; +const PT = "America/Los_Angeles"; + +/** US states and territories, Canadian provinces and territories: every zone each spans. */ +const REGIONS: Array<{ names: string[]; zones: string[] }> = [ + { names: ["al", "alabama"], zones: [CT] }, + { names: ["ak", "alaska"], zones: ["America/Anchorage", "America/Adak"] }, + { names: ["az", "arizona"], zones: ["America/Phoenix", MT] }, + { names: ["ar", "arkansas"], zones: [CT] }, + { names: ["ca", "california"], zones: [PT] }, + { names: ["co", "colorado"], zones: [MT] }, + { names: ["ct", "connecticut"], zones: [ET] }, + { names: ["de", "delaware"], zones: [ET] }, + { names: ["dc", "district of columbia", "washington dc"], zones: [ET] }, + { names: ["fl", "florida"], zones: [ET, CT] }, + { names: ["ga", "georgia"], zones: [ET] }, + { names: ["hi", "hawaii"], zones: ["Pacific/Honolulu"] }, + { names: ["id", "idaho"], zones: ["America/Boise", PT] }, + { names: ["il", "illinois"], zones: [CT] }, + { names: ["in", "indiana"], zones: ["America/Indiana/Indianapolis", CT] }, + { names: ["ia", "iowa"], zones: [CT] }, + { names: ["ks", "kansas"], zones: [CT, MT] }, + { names: ["ky", "kentucky"], zones: [ET, CT] }, + { names: ["la", "louisiana"], zones: [CT] }, + { names: ["me", "maine"], zones: [ET] }, + { names: ["md", "maryland"], zones: [ET] }, + { names: ["ma", "massachusetts"], zones: [ET] }, + { names: ["mi", "michigan"], zones: ["America/Detroit", "America/Menominee"] }, + { names: ["mn", "minnesota"], zones: [CT] }, + { names: ["ms", "mississippi"], zones: [CT] }, + { names: ["mo", "missouri"], zones: [CT] }, + { names: ["mt", "montana"], zones: [MT] }, + { names: ["ne", "nebraska"], zones: [CT, MT] }, + { names: ["nv", "nevada"], zones: [PT, MT] }, + { names: ["nh", "new hampshire"], zones: [ET] }, + { names: ["nj", "new jersey"], zones: [ET] }, + { names: ["nm", "new mexico"], zones: [MT] }, + { names: ["ny", "new york"], zones: [ET] }, + { names: ["nc", "north carolina"], zones: [ET] }, + { names: ["nd", "north dakota"], zones: [CT, MT] }, + { names: ["oh", "ohio"], zones: [ET] }, + { names: ["ok", "oklahoma"], zones: [CT] }, + { names: ["or", "oregon"], zones: [PT, "America/Boise"] }, + { names: ["pa", "pennsylvania"], zones: [ET] }, + { names: ["ri", "rhode island"], zones: [ET] }, + { names: ["sc", "south carolina"], zones: [ET] }, + { names: ["sd", "south dakota"], zones: [CT, MT] }, + { names: ["tn", "tennessee"], zones: [CT, ET] }, + { names: ["tx", "texas"], zones: [CT, MT] }, + { names: ["ut", "utah"], zones: [MT] }, + { names: ["vt", "vermont"], zones: [ET] }, + { names: ["va", "virginia"], zones: [ET] }, + { names: ["wa", "washington"], zones: [PT] }, + { names: ["wv", "west virginia"], zones: [ET] }, + { names: ["wi", "wisconsin"], zones: [CT] }, + { names: ["wy", "wyoming"], zones: [MT] }, + { names: ["pr", "puerto rico"], zones: ["America/Puerto_Rico"] }, + { names: ["gu", "guam"], zones: ["Pacific/Guam"] }, + { names: ["vi", "us virgin islands", "virgin islands"], zones: ["America/St_Thomas"] }, + { names: ["as", "american samoa"], zones: ["Pacific/Pago_Pago"] }, + { names: ["mp", "northern mariana islands"], zones: ["Pacific/Saipan"] }, + { names: ["ab", "alberta"], zones: ["America/Edmonton"] }, + { names: ["bc", "british columbia"], zones: ["America/Vancouver", "America/Edmonton"] }, + { names: ["mb", "manitoba"], zones: ["America/Winnipeg"] }, + { names: ["nb", "new brunswick"], zones: ["America/Moncton"] }, + { names: ["nl", "newfoundland and labrador", "newfoundland", "labrador"], zones: ["America/St_Johns", "America/Goose_Bay"] }, + { names: ["ns", "nova scotia"], zones: ["America/Halifax"] }, + { names: ["nt", "northwest territories"], zones: ["America/Edmonton"] }, + { names: ["nu", "nunavut"], zones: ["America/Iqaluit", "America/Rankin_Inlet", "America/Cambridge_Bay"] }, + { names: ["on", "ontario"], zones: ["America/Toronto", "America/Winnipeg"] }, + { names: ["pe", "pei", "prince edward island"], zones: ["America/Halifax"] }, + { names: ["qc", "quebec", "québec"], zones: ["America/Toronto", "America/Halifax"] }, + { names: ["sk", "saskatchewan"], zones: ["America/Regina", "America/Edmonton"] }, + { names: ["yt", "yukon"], zones: ["America/Whitehorse"] }, +]; + +/** The zones of a US state or Canadian province, by code or name. Pure. */ +export function zonesForRegion(region: string | undefined): string[] | undefined { + const key = region?.trim().toLowerCase().replace(/\./g, ""); + return key ? REGIONS.find((r) => r.names.includes(key))?.zones : undefined; +} + +/** Minutes east of UTC in a zone at a moment. */ +export function offsetMinutes(zone: string, at: Date): number { + const parts = Object.fromEntries( + new Intl.DateTimeFormat("en-US", { timeZone: zone, hourCycle: "h23", year: "numeric", month: "numeric", day: "numeric", hour: "numeric", minute: "numeric" }) + .formatToParts(at) + .map((p) => [p.type, Number(p.value)]), + ) as Record; + const local = Date.UTC(parts.year!, parts.month! - 1, parts.day!, parts.hour!, parts.minute!); + return Math.round((local - Math.floor(at.getTime() / 60_000) * 60_000) / 60_000); +} + +/** Zones close enough together to say something: within a few hours of each other. */ +function narrow(zones: string[] | undefined, at: Date): string[] | undefined { + if (!zones?.length) return undefined; + const offsets = zones.map((z) => offsetMinutes(z, at)); + return Math.max(...offsets) - Math.min(...offsets) < MAX_ZONE_SPREAD_HOURS * 60 ? zones : undefined; +} + +/** + * Every time zone a prospect may be in. A timezone column says it outright. + * Otherwise their phone's area code and their state or province both count: + * an area code inside the state narrows it (a 512 number in Texas is + * Central), and one outside it adds its zone, so a Texas business on a New + * York cell is called when both are awake. The country is the last resort, + * when it has one time zone. Undefined when none of it says. + */ +export function prospectZones(prospect: { phone?: string; variables: Record }, at: Date): ProspectZones | undefined { + const v = prospect.variables; + const explicit = [v.timezone, v.time_zone, v.tz].find((z) => isTimeZone(z)); + if (explicit) return { zones: [explicit.trim()], from: ["timezone"] }; + const countryName = v.country ?? v.country_code; + const country = findCountry(countryName); + const digits = phoneDigits(prospect.phone ?? v.phone, countryName); + const byPhone = narrow(digits ? zonesForNumber(digits) : undefined, at); + const northAmerican = !countryName?.trim() || country?.code === "1"; + const byRegion = northAmerican ? zonesForRegion(v.region ?? v.state ?? v.province ?? v.state_code) : undefined; + if (byPhone && byRegion) { + const offsets = new Set(byRegion.map((z) => offsetMinutes(z, at))); + const inside = byPhone.every((z) => offsets.has(offsetMinutes(z, at))); + return { zones: inside ? byPhone : [...new Set([...byPhone, ...byRegion])], from: ["phone", "region"] }; + } + if (byPhone) return { zones: byPhone, from: ["phone"] }; + if (byRegion) return { zones: byRegion, from: ["region"] }; + const byCountry = country ? narrow(zonesForNumber(country.code), at) : undefined; + return byCountry ? { zones: byCountry, from: ["country"] } : undefined; +} diff --git a/src/domains/callHours/index.ts b/src/domains/callHours/index.ts new file mode 100644 index 0000000..1b2a09b --- /dev/null +++ b/src/domains/callHours/index.ts @@ -0,0 +1,2 @@ +export * from "./types"; +export * from "./helpers"; diff --git a/src/domains/callHours/types.ts b/src/domains/callHours/types.ts new file mode 100644 index 0000000..5e96eaf --- /dev/null +++ b/src/domains/callHours/types.ts @@ -0,0 +1,26 @@ +/** Where a prospect's time zone was read from. */ +export type ZoneSource = "timezone" | "phone" | "region" | "country"; + +/** The time zones a prospect may be in. Several when their area spans zones: calls then fit all of them. */ +export interface ProspectZones { + zones: string[]; + /** Everything that placed them: their phone's area code and their state can both count. */ + from: ZoneSource[]; +} + +/** Whether a prospect can be called now, in their own time. */ +export interface CallTime { + /** Inside calling hours in every zone they may be in; undefined when their zone is unknown. */ + ok?: boolean; + /** Their time now: "9:14 AM CST", or "9:14 AM CST / 8:14 AM MST" when their area spans zones. */ + local?: string; + zones: string[]; + from: ZoneSource[]; + /** When calling hours open next for them, when they are closed now. */ + nextAt?: string; + /** One line for people: "9:14 AM their time (from their phone number)". */ + note: string; +} + +/** A phone prefix whose zones are this many hours apart or more says nothing useful (a whole country such as the US). */ +export const MAX_ZONE_SPREAD_HOURS = 3; diff --git a/src/domains/config/helpers/parseConfig.test.ts b/src/domains/config/helpers/parseConfig.test.ts index 204e472..ec87eb2 100644 --- a/src/domains/config/helpers/parseConfig.test.ts +++ b/src/domains/config/helpers/parseConfig.test.ts @@ -29,6 +29,19 @@ describe("parseConfig", () => { ]); }); + test("the calls section is optional: 8am to 9pm every day, in the prospect's time", () => { + expect(parseConfig(example, "x").config!.calls).toEqual({ start: "08:00", end: "21:00", days: ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] }); + const tuned = parseConfig({ ...example, calls: { start: "09:00", end: "18:30", days: ["Mon", "tue", "wed", "thu", "fri"] } }, "x"); + expect(tuned.config!.calls).toEqual({ start: "09:00", end: "18:30", days: ["mon", "tue", "wed", "thu", "fri"] }); + expect(parseConfig({ ...example, calls: { end: "20:00" } }, "x").config!.calls.start).toBe("08:00"); + const { errors } = parseConfig({ ...example, calls: { start: "21:00", end: "08:00", days: ["funday"], window: 1 } }, "x"); + expect(errors).toEqual([ + "calls.window: not a calls setting (start, end, days)", + 'calls: start and end as "HH:MM" in the prospect\'s time, start before end, e.g. start: "08:00", end: "21:00"', + "calls.days: a list like [mon, tue, wed, thu, fri, sat]", + ]); + }); + test("an account without a footer gets the default for its voice", () => { const { config } = parseConfig(example, "x"); expect(config!.accounts.personal!.email!.footer).toBe(DEFAULT_FOOTERS.personal); diff --git a/src/domains/config/helpers/parseConfig.ts b/src/domains/config/helpers/parseConfig.ts index de6129f..11f1d38 100644 --- a/src/domains/config/helpers/parseConfig.ts +++ b/src/domains/config/helpers/parseConfig.ts @@ -1,6 +1,6 @@ import { PLATFORMS, VOICES, type Platform, type Voice } from "../../messages"; import { VARIABLES } from "../../render"; -import { DEFAULT_DECIDE, WEEKDAYS, type Account, type ColdConfig, type DecideSettings, type DmSettings, type EmailSettings, type Offer, type SendWindow, type StoreKind, type Weekday } from "../types"; +import { DEFAULT_CALL_HOURS, DEFAULT_DECIDE, WEEKDAYS, type Account, type CallHours, type ColdConfig, type DecideSettings, type DmSettings, type EmailSettings, type Offer, type SendWindow, type StoreKind, type Weekday } from "../types"; const ACCOUNT_VARIABLES = new Set(VARIABLES.filter((v) => v.scope === "account").map((v) => v.name)); const OFFER_VARIABLES = new Set(VARIABLES.filter((v) => v.scope === "offer").map((v) => v.name)); @@ -56,10 +56,11 @@ export function parseConfig(raw: unknown, file: string, isExample = false): { co if (!["auto", "twenty", "file"].includes(store)) errors.push("store: must be auto, twenty or file"); const decide = parseDecide(raw.decide, errors); + const calls = parseCalls(raw.calls, errors); if (errors.length) return { errors }; return { - config: { file, isExample, accounts, offers, defaultOffer, store, logFile: text(raw.log_file) || ".cold/touches.jsonl", decide }, + config: { file, isExample, accounts, offers, defaultOffer, store, logFile: text(raw.log_file) || ".cold/touches.jsonl", decide, calls }, errors, }; } @@ -167,6 +168,26 @@ function isTimezone(name: string): boolean { } } +/** The optional calls: section: when to call, in each prospect's own time. Every key has a default. */ +function parseCalls(raw: unknown, errors: string[]): CallHours { + if (raw === undefined || raw === null) return { ...DEFAULT_CALL_HOURS, days: [...DEFAULT_CALL_HOURS.days] }; + if (!isObject(raw)) { + errors.push("calls: must be a mapping (start, end, days)"); + return { ...DEFAULT_CALL_HOURS }; + } + for (const key of Object.keys(raw)) if (!["start", "end", "days"].includes(key)) errors.push(`calls.${key}: not a calls setting (start, end, days)`); + const start = text(raw.start) || DEFAULT_CALL_HOURS.start; + const end = text(raw.end) || DEFAULT_CALL_HOURS.end; + if (!TIME.test(start) || !TIME.test(end) || start >= end) errors.push('calls: start and end as "HH:MM" in the prospect\'s time, start before end, e.g. start: "08:00", end: "21:00"'); + let days = [...DEFAULT_CALL_HOURS.days]; + if (raw.days !== undefined) { + const given = Array.isArray(raw.days) ? raw.days.map((d) => text(d).toLowerCase()) : []; + if (!given.length || !given.every((d) => WEEKDAYS.includes(d as Weekday))) errors.push("calls.days: a list like [mon, tue, wed, thu, fri, sat]"); + else days = given as Weekday[]; + } + return { start, end, days }; +} + /** The optional decide: section. Every key has a default, so it can be left out entirely. */ function parseDecide(raw: unknown, errors: string[]): DecideSettings { if (raw === undefined || raw === null) return { ...DEFAULT_DECIDE }; diff --git a/src/domains/config/types.ts b/src/domains/config/types.ts index 531ead5..b7c06a2 100644 --- a/src/domains/config/types.ts +++ b/src/domains/config/types.ts @@ -14,6 +14,17 @@ export interface SendWindow { timezone: string; } +/** When to call, in each prospect's own time zone. The US rule is 8am to 9pm; other countries set their own. */ +export interface CallHours { + days: Weekday[]; + /** "08:00" */ + start: string; + /** "21:00": the first minute calls stop. */ + end: string; +} + +export const DEFAULT_CALL_HOURS: CallHours = { days: [...WEEKDAYS], start: "08:00", end: "21:00" }; + export interface EmailSettings { /** "Alex Morgan " */ from: string; @@ -80,4 +91,6 @@ export interface ColdConfig { /** The touch log when the store is a file. */ logFile: string; decide: DecideSettings; + /** When prospects may be called, in their own time. */ + calls: CallHours; } diff --git a/src/domains/leads/helpers/quizLead.ts b/src/domains/leads/helpers/quizLead.ts index 3924e76..d5fe8e7 100644 --- a/src/domains/leads/helpers/quizLead.ts +++ b/src/domains/leads/helpers/quizLead.ts @@ -1,11 +1,13 @@ +import { callTime } from "../../callHours"; +import type { CallHours } from "../../config"; import { leadNote, type Prospect } from "../../prospects"; import type { Touch } from "../../touches"; import type { NtfyMessage, QuizLead } from "../types"; const text = (v: unknown) => (typeof v === "string" && v.trim() ? v.trim() : undefined); -/** One quiz lead for the Calls page: its record in Twenty, as a prospect, and when it was last called. Pure. */ -export function quizLead(record: Record, prospect: Prospect, calledAt?: string): QuizLead { +/** One quiz lead for the Calls page: its record in Twenty, as a prospect, when it was last called, and whether it can be called now. Pure. */ +export function quizLead(record: Record, prospect: Prospect, calledAt?: string, clock?: { hours: CallHours; now: Date }): QuizLead { const v = prospect.variables; const qualification = (text(record.qualificationStatus) ?? "").toUpperCase(); const status = (text(record.status) ?? "").toUpperCase(); @@ -23,6 +25,7 @@ export function quizLead(record: Record, prospect: Prospect, ca from: v.utm_source ?? v.lead_source, calledAt, doNotContact: prospect.doNotContact, + callTime: clock ? callTime(prospect, clock.hours, clock.now) : undefined, }; } @@ -35,23 +38,22 @@ export function lastCall(prospect: Prospect, calls: Touch[]): string | undefined } /** - * Who to call first: qualified leads nobody has called yet, newest first; + * Who to call first: qualified leads nobody has called yet that can be + * called now, newest first; then the ones it's too early or late to call; * then the ones already called; then the disqualified; Do Not Contact last. * Pure. */ export function rankLeads(leads: QuizLead[]): QuizLead[] { - const group = (l: QuizLead) => (l.doNotContact ? 3 : !l.qualified ? 2 : l.calledAt ? 1 : 0); + const group = (l: QuizLead) => (l.doNotContact ? 4 : !l.qualified ? 3 : l.calledAt ? 2 : l.callTime?.ok === false ? 1 : 0); return [...leads].sort((a, b) => group(a) - group(b) || (b.createdAt ?? "").localeCompare(a.createdAt ?? "")); } /** The push for one new lead: who, their company, the number to call, the first answers. Tapping it opens the lead on the Calls page. Pure. */ export function leadPush(lead: QuizLead, appUrl?: string): Omit { const who = lead.name ?? lead.label; - const lines = [ - lead.company && lead.company !== who ? lead.company : undefined, - lead.phone ? `Call ${lead.phone}` : lead.email ? `No phone given: ${lead.email}` : undefined, - ...lead.answers.slice(0, 4), - ].filter((l): l is string => !!l); + const time = lead.callTime; + const call = !lead.phone ? undefined : time?.ok === false ? `Not yet: ${time.note}\nTheir number: ${lead.phone}` : time?.ok ? `Call ${lead.phone}: ${time.local} their time` : `Call ${lead.phone}`; + const lines = [lead.company && lead.company !== who ? lead.company : undefined, call ?? (lead.email ? `No phone given: ${lead.email}` : undefined), ...lead.answers.slice(0, 4)].filter((l): l is string => !!l); return { title: `New quiz lead: ${who}`.slice(0, 120), message: (lines.join("\n") || "They took your quiz.").slice(0, 1000), diff --git a/src/domains/leads/index.ts b/src/domains/leads/index.ts index a3cd814..2b35eec 100644 --- a/src/domains/leads/index.ts +++ b/src/domains/leads/index.ts @@ -1,3 +1,4 @@ +import { DEFAULT_CALL_HOURS, type CallHours } from "../config"; import type { Db } from "../db"; import { loadAgencyLeads } from "../prospects"; import type { Touch } from "../touches"; @@ -22,6 +23,8 @@ export interface LeadWatchOptions { target?: () => NtfyTarget | undefined; /** The app's address, for the link a push opens: "https://cold.example.com/app/". */ appUrl?: string; + /** Calling hours, so a push says when a lead can be called; cold.yaml's calls: by default 8am to 9pm. */ + hours?: () => Promise; fetch?: typeof fetch; now?: () => Date; } @@ -60,9 +63,10 @@ export class LeadWatch { let alerted = 0; let found = 0; try { + const hours = (await this.options.hours?.()) ?? DEFAULT_CALL_HOURS; const rows = await loadAgencyLeads(client, { where: `createdAt[gt]:${filterValue(seen.value)}`, limit: MAX_LEADS_PER_CHECK }); const leads = rows - .map(({ record, prospect }) => quizLead(record, prospect)) + .map(({ record, prospect }) => quizLead(record, prospect, undefined, { hours, now })) .filter((l) => l.createdAt && l.createdAt > seen.value) .sort((a, b) => a.createdAt!.localeCompare(b.createdAt!)); found = leads.length; @@ -116,9 +120,10 @@ export class LeadWatch { * The quiz leads of the last days, best to call first, each with when it was * last called. calls gives the touch log's calls since a time. */ -export async function recentLeads(client: TwentyClient, options: { now: Date; days?: number; calls?: (since: string) => Promise }): Promise { +export async function recentLeads(client: TwentyClient, options: { now: Date; days?: number; hours?: CallHours; calls?: (since: string) => Promise }): Promise { const since = new Date(options.now.getTime() - (options.days ?? LEAD_WINDOW_DAYS) * 86_400_000).toISOString(); const rows = await loadAgencyLeads(client, { where: `createdAt[gte]:${filterValue(since)}`, limit: 200 }); const calls = (await options.calls?.(since)) ?? []; - return rankLeads(rows.map(({ record, prospect }) => quizLead(record, prospect, lastCall(prospect, calls)))); + const clock = { hours: options.hours ?? DEFAULT_CALL_HOURS, now: options.now }; + return rankLeads(rows.map(({ record, prospect }) => quizLead(record, prospect, lastCall(prospect, calls), clock))); } diff --git a/src/domains/leads/leads.test.ts b/src/domains/leads/leads.test.ts index 06fe230..cf3aee3 100644 --- a/src/domains/leads/leads.test.ts +++ b/src/domains/leads/leads.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, test } from "bun:test"; +import { DEFAULT_CALL_HOURS } from "../config"; import { openDatabase, type Db } from "../db"; import { fromAgencyLead } from "../prospects"; import type { Touch } from "../touches"; @@ -92,15 +93,16 @@ describe("quiz leads", () => { expect(lastCall(prospect, [{ ...call("agencyLeads:lead-1", "2026-11-01T17:00:00Z"), channel: "EMAIL", kind: "SENT" }])).toBeUndefined(); }); - test("who to call first: uncalled qualified leads, newest first, then called, disqualified, Do Not Contact", () => { + test("who to call first: uncalled qualified leads, newest first, then the ones it's too late to call, called, disqualified, Do Not Contact", () => { const order = rankLeads([ + lead({ id: "asleep", createdAt: "2026-11-01T21:00:00Z", callTime: { ok: false, zones: ["Pacific/Honolulu"], from: ["phone"], note: "" } }), lead({ id: "dnc", doNotContact: true, createdAt: "2026-11-01T20:00:00Z" }), lead({ id: "dq", qualified: false, createdAt: "2026-11-01T19:00:00Z" }), lead({ id: "called", calledAt: "2026-11-01T18:30:00Z", createdAt: "2026-11-01T18:00:00Z" }), lead({ id: "older", createdAt: "2026-11-01T10:00:00Z" }), lead({ id: "newer", createdAt: "2026-11-01T12:00:00Z" }), ]).map((l) => l.id); - expect(order).toEqual(["newer", "older", "called", "dq", "dnc"]); + expect(order).toEqual(["newer", "older", "asleep", "called", "dq", "dnc"]); }); test("a push names the lead, the company, the number, their first answers, and opens the lead", () => { @@ -113,6 +115,15 @@ describe("quiz leads", () => { expect(leadPush(lead({ phone: undefined, answers: [] })).message).toBe("They took your quiz."); }); + test("a push says whether they can be called now, in their own time", () => { + const record = { id: "lead-1", contactName: "Sam Lee", note: note("Sam Lee"), createdAt: "2026-11-03T03:58:00.000Z" }; + const at = (iso: string) => leadPush(quizLead(record, fromAgencyLead(record), undefined, { hours: DEFAULT_CALL_HOURS, now: new Date(iso) })).message; + expect(at("2026-11-02T15:00:00Z")).toBe("Call 512 555 0147: 9:00 AM CST their time\nCars a week: 10-20"); + expect(at("2026-11-03T04:00:00Z")).toBe( + "Not yet: 10:00 PM CST their time, outside calling hours: call from tomorrow 8:00 AM CST (from their phone number).\nTheir number: 512 555 0147\nCars a week: 10-20", + ); + }); + test("many leads at once come as one push", () => { const leads = Array.from({ length: 7 }, (_, i) => lead({ id: `l${i}`, name: `Lead ${i}` })); const push = countPush(leads, "https://cold.example.com/app/"); diff --git a/src/domains/leads/types.ts b/src/domains/leads/types.ts index 998b73e..56ad3cd 100644 --- a/src/domains/leads/types.ts +++ b/src/domains/leads/types.ts @@ -1,3 +1,5 @@ +import type { CallTime } from "../callHours"; + /** Where new-lead alerts go: a topic on ntfy.sh or on your own ntfy server. */ export interface NtfyTarget { /** The server's root, where messages are published as JSON: "https://ntfy.sh/". */ @@ -41,6 +43,8 @@ export interface QuizLead { /** When they were last called, from the touch log. */ calledAt?: string; doNotContact: boolean; + /** Whether they can be called now, in their own time. */ + callTime?: CallTime; } /** What one look for new leads did. */ diff --git a/src/domains/web/api/index.ts b/src/domains/web/api/index.ts index 497b7ce..e556f9e 100644 --- a/src/domains/web/api/index.ts +++ b/src/domains/web/api/index.ts @@ -623,7 +623,7 @@ export function createApi(ctx: WebContext, extraRoutes: Route[] = []): (request: const source = url.searchParams.get("source"); const prospectKey = url.searchParams.get("prospect"); const prospect = source && prospectKey ? findProspect(await ctx.prospects(source), prospectKey) : undefined; - return json(callSheetData(script, library, account, offer, prospect)); + return json(callSheetData(script, library, account, offer, prospect, { hours: config.calls, now: ctx.now() })); }, ], [ diff --git a/src/domains/web/context/index.ts b/src/domains/web/context/index.ts index 2a5c044..8932536 100644 --- a/src/domains/web/context/index.ts +++ b/src/domains/web/context/index.ts @@ -1,7 +1,7 @@ import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; import { AiError, AiStore, aiConfigured, claudeCall, type MessagesCall } from "../../ai"; -import { APP_ROOT, configFromText, type Account, type ColdConfig, type DecideSettings } from "../../config"; +import { APP_ROOT, configFromText, DEFAULT_CALL_HOURS, type Account, type ColdConfig, type DecideSettings } from "../../config"; import type { Db } from "../../db"; import { claudeDecider, DecideStore, jevConfigured, jevProvider, subqConfigured, subqReader, type CollectOptions, type DecisionProvider, type LongReader } from "../../decide"; import { reacherFromEnv, smtpTransport, type MailTransport } from "../../email"; @@ -118,7 +118,7 @@ export class WebContext { }); this.x = new XStore(options.db, { key: usableSecretKey(), requireKey: this.storage === "database", now: () => this.now() }); this.listening = new ListenStore(options.db); - this.leads = new LeadWatch({ db: options.db, client: () => this.twenty(), appUrl: options.appUrl, fetch: options.ntfyFetch, now: () => this.now() }); + this.leads = new LeadWatch({ db: options.db, client: () => this.twenty(), appUrl: options.appUrl, hours: async () => (await this.config()).config?.calls ?? DEFAULT_CALL_HOURS, fetch: options.ntfyFetch, now: () => this.now() }); } } diff --git a/src/domains/web/leads/index.ts b/src/domains/web/leads/index.ts index 20aa4dc..83969cd 100644 --- a/src/domains/web/leads/index.ts +++ b/src/domains/web/leads/index.ts @@ -33,6 +33,7 @@ export function leadRoutes(ctx: WebContext): Route[] { response.leads = await recentLeads(client, { now: ctx.now(), days, + hours: (await ctx.config()).config?.calls, // Called or not, from the touch log; a log that can't be opened only leaves that out. calls: async (since) => { try { diff --git a/src/domains/web/leads/leads.test.ts b/src/domains/web/leads/leads.test.ts index 533a75b..5a80f58 100644 --- a/src/domains/web/leads/leads.test.ts +++ b/src/domains/web/leads/leads.test.ts @@ -6,6 +6,7 @@ import { openDatabase } from "../../db"; import { TwentyClient } from "../../twenty"; import { createApi } from "../api"; import { WebContext } from "../context"; +import type { CallSheet } from "../../call"; import type { ActionResponse, LeadAlertsResponse, LeadsResponse, SourceOption } from "../types"; import { leadRoutes } from "."; @@ -96,6 +97,20 @@ describe("lead routes", () => { expect(data.leads[0]).toMatchObject({ prospectKey: "agencyLeads:sam", phone: "512 555 0147", answers: ["Cars a week: 10-20"] }); }); + test("each lead says whether it is a time to call them", async () => { + const { data } = await call("GET", "/api/leads"); + expect(data.leads[0]!.callTime).toMatchObject({ ok: true, local: "12:00 PM CST", zones: ["America/Chicago"], from: ["phone"] }); + }); + + test("a call sheet for a lead is filled in for them, with their time", async () => { + const sheet = await call("GET", `/api/calls/sheet?${new URLSearchParams({ id: "call/personal/founder-direct", source: "twenty:agencyLeads", prospect: "agencyLeads:sam" })}`); + expect(sheet.status).toBe(200); + expect(sheet.data.sections[0]!.text).toStartWith("Hi Sam,"); + expect(sheet.data.callTime?.note).toBe("12:00 PM CST their time (from their phone number)."); + const plain = await call("GET", "/api/calls/sheet?id=call/personal/founder-direct"); + expect(plain.data.callTime).toBeUndefined(); + }); + test("the leads are a prospect source, so a call script fills in for one", async () => { process.env.TWENTY_BASE_URL = "https://crm.example"; process.env.TWENTY_API_KEY = "k"; diff --git a/src/web/pages/calls.tsx b/src/web/pages/calls.tsx index 22b0ec6..862f98b 100644 --- a/src/web/pages/calls.tsx +++ b/src/web/pages/calls.tsx @@ -1,4 +1,4 @@ -import { ListChecks, Phone, PhoneCall, RefreshCw, Search, Sparkles, X } from "lucide-react"; +import { Clock, ListChecks, Phone, PhoneCall, PhoneOff, RefreshCw, Search, Sparkles, X } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import type { CallPrep } from "../../domains/ai/tasks"; import type { CallSheet } from "../../domains/call"; @@ -57,6 +57,7 @@ export function CallsPage() { {l.phone ?? "No phone"} {l.createdAt ? `, took the quiz ${ago(l.createdAt)}` : ""} + {l.callTime?.ok === false &&
Not now: {l.callTime.local} there
} ))} {due.length > 8 &&
{due.length - 8} more waiting
} @@ -115,6 +116,11 @@ export function CallsPage() { {sheet.data && sheet.data.missing.length > 0 && This prospect has no {sheet.data.missing.map((m) => `{{${m}}}`).join(", ")}: those parts are left in braces.} + {sheet.data?.callTime && ( + + {sheet.data.callTime.note} + + )} {sheet.data && (
void }) { {lead.email ? `. ${lead.email}` : ""} {lead.phone && !lead.doNotContact && ( -
- - Call {lead.phone} - +
+ {lead.callTime?.ok === false ? ( + + ) : ( + + Call {lead.phone} + + )} + {lead.callTime?.ok && ( + + {lead.callTime.local} there + + )}
)} {lead.answers.length > 0 && ( From 339d9a80a9298d905a6fdf1814d526fa430ce900 Mon Sep 17 00:00:00 2001 From: operatoruplift Date: Sun, 1 Nov 2026 16:22:02 +0000 Subject: [PATCH 5/6] The dialer's calls come back: outcomes in the touch log and the decisions - Every call the dialer logs with an outcome becomes one touch on the business's or quiz lead's timeline (agencyProspect, else agencyLead): CALLED with the outcome (templateId dialer:), the dialer's AI score and tone, and its summary; "do not call" is an OPTED_OUT and marks the dialer's prospect Do Not Contact. A call is read once it has settled (outcome set, call over, AI reading done or 15 minutes passed) and logged once, by its id. - It runs every ten minutes in the background (the local timer and the Vercel cron, claimed in the database so one server runs it), from Log calls on the Calls page (POST /api/calls/log), or with cold call log [--days] [--dry-run]. - Talking with someone (interested, appointment set, call back, not interested) pauses their sequences for 90 days, like a reply. - The decision engine reads the calls in the history. Someone who opted out is always skipped; "not interested" on a call in the last 90 days turns reach out now into later; a wrong or bad number is no longer a way to reach them; skipping someone the dialer logged as interested goes to you to check. - Quiz leads say how their last call went. One helper claims a background run for every server (claimRun), shared with the lead watch. Toasts keep line breaks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TZhsuUQ7RbYzBy4coULGCu --- CHANGELOG.md | 15 ++ README.md | 4 + docs/compliance.md | 4 +- docs/decision-engine.md | 19 ++- src/cli.ts | 15 +- src/domains/call/helpers/callTouch.ts | 57 +++++++ src/domains/call/helpers/index.ts | 1 + src/domains/call/index.ts | 72 ++++++++- src/domains/call/outcomes.test.ts | 146 ++++++++++++++++++ src/domains/db/index.ts | 16 ++ src/domains/decide/diligence.test.ts | 5 + src/domains/decide/diligence.ts | 6 +- src/domains/decide/model.test.ts | 15 ++ src/domains/decide/model.ts | 32 ++-- src/domains/leads/helpers/quizLead.ts | 16 +- src/domains/leads/index.ts | 15 +- src/domains/leads/leads.test.ts | 2 +- src/domains/leads/types.ts | 2 + .../touches/helpers/callOutcome.test.ts | 23 +++ src/domains/touches/helpers/callOutcome.ts | 50 ++++++ src/domains/touches/helpers/index.ts | 1 + .../touches/helpers/planSequence.test.ts | 9 ++ src/domains/touches/helpers/planSequence.ts | 11 +- src/domains/web/calls/calls.test.ts | 73 +++++++++ src/domains/web/calls/index.ts | 30 ++++ src/domains/web/calls/log.ts | 54 +++++++ src/domains/web/context/index.ts | 4 + src/domains/web/server/index.ts | 5 +- src/domains/web/vercel/index.ts | 12 +- src/web/pages/calls.tsx | 37 ++++- src/web/styles.css | 2 +- 31 files changed, 699 insertions(+), 54 deletions(-) create mode 100644 src/domains/call/helpers/callTouch.ts create mode 100644 src/domains/call/outcomes.test.ts create mode 100644 src/domains/touches/helpers/callOutcome.test.ts create mode 100644 src/domains/touches/helpers/callOutcome.ts create mode 100644 src/domains/web/calls/calls.test.ts create mode 100644 src/domains/web/calls/index.ts create mode 100644 src/domains/web/calls/log.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cd83f3..07ff7fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ ## Unreleased +- **The dialer's calls come back.** Every call the dialer logs with an + outcome becomes a touch in COLD, on the business's or the quiz lead's + timeline: the outcome, the dialer's AI score and tone, and its summary, + once the call has settled. It runs every ten minutes in the background (on + your computer and on Vercel, once across servers), from **Log calls** on + Calls, or with `cold call log`, and a call is never logged twice. + - A call where you talked with them (interested, appointment set, call + back, not interested) pauses their sequences for 90 days, like a reply. + - "Do not call" is an opt-out: nothing goes to them on any channel, and the + dialer's prospect is marked Do Not Contact. + - The decision engine reads the calls in their history. Someone who opted + out is always skipped; "not interested" on a recent call turns reach out + now into later; a wrong or bad number is no longer a way to reach them; + and skipping someone who was interested goes to you to check. + - Quiz leads say how their last call went. - **Calls at their hour.** A call sheet for a prospect, in the app and from `cold call sheet`, says what time it is where they are and whether it is inside calling hours: 8am to 9pm every day unless cold.yaml's new `calls:` diff --git a/README.md b/README.md index 34eeae5..23e222c 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,10 @@ call scripts and the Do Not Contact list. - **Calls in the dialer.** One command writes the call scripts and objection answers into the dialer's `agencyScripts`, so they sit next to every live call. For a handset, `cold call sheet` prints one filled in for a prospect. + Every call the dialer makes comes back into the touch log with its outcome + and the dialer's AI summary: talking with someone pauses their sequences + like a reply, "do not call" is an opt-out on every channel, and the + decision engine reads the calls when it scores them. - **It tells you what works.** `cold stats` shows the reply rate of every message and every sequence, so you keep the winners and rewrite the rest. - **Checked like code.** `bun run check` type-checks, runs the tests (including diff --git a/docs/compliance.md b/docs/compliance.md index bc24ab0..c835bfe 100644 --- a/docs/compliance.md +++ b/docs/compliance.md @@ -85,7 +85,9 @@ within 5 business days. - **Say who you are and why you're calling,** and give a number they can reach you on (`{{sender_phone}}` in the voicemail scripts). - **Keep an internal do-not-call list.** That is the dialer's Do Not Contact - status, and `cold` respects it. + status, and `cold` respects it. A call the dialer marks "do not call" is + logged in `cold` as an opt-out (every ten minutes, or `cold call log`), so + no email or DM goes to them either. ## DMs and platform rules diff --git a/docs/decision-engine.md b/docs/decision-engine.md index c4fa1b1..6e2d3fa 100644 --- a/docs/decision-engine.md +++ b/docs/decision-engine.md @@ -160,11 +160,20 @@ it ranks well. ### The next step, and when you check -The model suggests the next step; two rules overrule it: no way to reach them -is always **research first**, and a fit under 25% is always **skip**. A call -goes to you to **check** when the model is less sure than `review_below` (75%, -the threshold TypeSafe recommends for Jev), when the score and the step -disagree, or when only the list was known about them. +The model suggests the next step; these rules overrule it: someone who opted +out (by email, or "do not call" in the dialer) is always **skip**, no way to +reach them is always **research first**, a fit under 25% is always **skip**, +and someone who said "not interested" on a call in the last 90 days is +**later**, never reach out now. A call goes to you to **check** when the +model is less sure than `review_below` (75%, the threshold TypeSafe +recommends for Jev), when the score and the step disagree, when it says skip +someone the dialer logged as interested, or when only the list was known +about them. + +The dialer's calls are part of what the model reads: each one in the history +with its outcome, the dialer's AI score and tone, and its summary. A wrong or +bad number stops counting the phone as a way to reach them, which lowers +their reach score. Anyone marked Do Not Contact, or who ever opted out, is never researched or scored. diff --git a/src/cli.ts b/src/cli.ts index 92436ca..9eb9ce9 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -4,7 +4,7 @@ * accounts, with every touch kept in Twenty. `bun run cold help` for usage. */ import { parseArgs } from "node:util"; -import { callSheet, syncCallScripts } from "./domains/call"; +import { callOutcomeLines, callSheet, syncCallOutcomes, syncCallScripts } from "./domains/call"; import { bold, cyan, dim, green, indent, red, table, terminalIo, yellow } from "./domains/cli/io"; import { getAccount, getOffer, loadConfig, loadEnvFiles, type Account, type ColdConfig } from "./domains/config"; import { runDm } from "./domains/dm"; @@ -45,6 +45,7 @@ ${bold("DMs")} ${bold("Calls")} cold call sync [--account ] [--offer ] [--campaign ] [--dry-run] cold call sheet