diff --git a/docs/analyst/crossposting-quirk-fix-ab.sql b/docs/analyst/crossposting-quirk-fix-ab.sql new file mode 100644 index 00000000..b297721a --- /dev/null +++ b/docs/analyst/crossposting-quirk-fix-ab.sql @@ -0,0 +1,55 @@ +-- Crosspost quirk-fix live A/B readout +-- (experiments/2026-11-01-crosspost-quirk-fix-ab.md). +-- One SELECT, one row per channel x arm. Each decision picks an arm at random +-- (src/crossposting/scoring.py AB_ARMS); crossposting.score_version records it: +-- control: RU v4 = 4, EN v2 = 2 fix: RU v4q = 9, EN v2q = 8 +-- zf = ln(1+forwards) - channel x MSK-week mean over posted image memes >= 48h old. +-- diff_zf / ci95 are fix minus control within the channel (Welch, normal approx). +-- changed_pct: share of fix-arm decisions where the control score would have picked +-- a different meme from the same top-N (how often the fix actually mattered). +WITH posts AS ( + SELECT cp.channel, cp.score_version, cp.views, cp.forwards, + LN(1 + cp.forwards) AS lfw, + date_trunc('week', cp.created_at + INTERVAL '3 hours') AS wk + FROM crossposting cp + JOIN meme m ON m.id = cp.meme_id + WHERE cp.channel IN ('tgchannelru', 'tgchannelen') + AND cp.score_version IN (2, 4, 8, 9) + AND cp.telegram_message_id IS NOT NULL + AND cp.views > 0 + AND cp.forwards IS NOT NULL + AND m.type = 'image' + AND cp.created_at >= '2026-11-01' + AND cp.created_at < NOW() - INTERVAL '48 hours' +), +labeled AS ( + SELECT *, lfw - AVG(lfw) OVER (PARTITION BY channel, wk) AS zf, + CASE WHEN score_version IN (8, 9) THEN 'fix' ELSE 'control' END AS arm + FROM posts +), +arms AS ( + SELECT channel, arm, COUNT(*) AS n, AVG(zf) AS mzf, VAR_SAMP(zf) AS vzf, + AVG(views) AS views, AVG(forwards::float / views) AS fwd_rate + FROM labeled GROUP BY 1, 2 +), +changed AS ( + SELECT channel, + ROUND(100.0 * AVG((candidates->0->'picks'->>'v4q' + IS DISTINCT FROM candidates->0->'picks'->>'v4')::int), 1) AS ru, + ROUND(100.0 * AVG((candidates->0->'picks'->>'v2q' + IS DISTINCT FROM candidates->0->'picks'->>'v2')::int), 1) AS en + FROM crossposting_decision_log + WHERE decided_at >= '2026-11-01' AND score_version IN (8, 9) + GROUP BY 1 +) +SELECT a.channel, a.arm, a.n, ROUND(a.mzf::numeric, 3) AS mean_zf, + ROUND(a.views::numeric) AS avg_views, ROUND((100 * a.fwd_rate)::numeric, 3) AS fwd_rate_pct, + CASE WHEN a.arm = 'fix' THEN ROUND((a.mzf - c.mzf)::numeric, 3) END AS diff_zf, + CASE WHEN a.arm = 'fix' + THEN ROUND((1.96 * SQRT(a.vzf / a.n + c.vzf / c.n))::numeric, 3) END AS ci95, + CASE WHEN a.arm = 'fix' + THEN CASE a.channel WHEN 'tgchannelru' THEN ch.ru ELSE ch.en END END AS changed_pct +FROM arms a +LEFT JOIN arms c ON c.channel = a.channel AND c.arm = 'control' +LEFT JOIN changed ch ON ch.channel = a.channel +ORDER BY a.channel, a.arm diff --git a/experiments/2026-11-01-crosspost-quirk-fix-ab.md b/experiments/2026-11-01-crosspost-quirk-fix-ab.md new file mode 100644 index 00000000..70e3a28d --- /dev/null +++ b/experiments/2026-11-01-crosspost-quirk-fix-ab.md @@ -0,0 +1,33 @@ +--- +id: crosspost_quirk_fix_ab +status: running +hypothesis: Fixing the two crosspost scoring quirks (impression penalty 0 instead of 0.8 RU / 0.5 EN; x0.5 instead of x1.0 for sources without history) lets better memes reach the channels and raises forwards per post +primary_metric: mean zf (fix - control) per channel, 95% CI +guardrails: avg views per post not lower than control; no posting gaps +unit: channel_post (random arm per decision, 50/50) +variants: control (RU v4 / EN v2, score_version 4 / 2); fix (RU v4q / EN v2q, score_version 9 / 8) +started: 2026-11-01 +check_on: 2026-11-28 +decided: +result: unknown +learning: +code: src/crossposting/scoring.py (AB_ARMS, impr_factor_q, src_quality_q) +readout: docs/analyst/crossposting-quirk-fix-ab.sql +--- + +# Experiment: Crosspost scoring quirk fix (live A/B) + +- **Created:** 2026-11-01 +- **Why:** the 2026-11-01 scoring refactor (#392) found two behaviours nobody chose: + 1. `impr_penalty` was bound as an integer, so it has always been 0, not 0.8 (RU) / 0.5 (EN). + Memes outside the top quartile of their source (`raw_impr_rank > 1`) can never be posted. + 2. `GREATEST(0.5, NULL)` gives sources without enough history x0.5, not a neutral x1.0. +- **Design:** each crosspost decision picks control or fix at random (`AB_ARMS`), and that score + does both the top-N preselect and the final pick. Both fixes ship together in one arm: + at ~5.5 posts/day per channel a 2x2 would have ~35 posts per cell by the check date. +- **Power:** ~75 posts per arm per channel by 2026-11-28; only a large effect will show. + If the CI still spans 0, decide on `changed_pct` and views: no harm → keep the fix (it is + the intended behaviour), harm → revert. +- **Overlap:** v5f shadow (`crosspost_src_fwd_v5f`) is log-only and still ranks within whichever + top-N the arm preselected. Its Spearman readout pools both arms. +- **End the test:** remove the channel from `AB_ARMS`, then set `LIVE`/`PRESELECT` to the winner. diff --git a/experiments/README.md b/experiments/README.md index e9f7c798..c7c092ae 100644 --- a/experiments/README.md +++ b/experiments/README.md @@ -55,7 +55,7 @@ Design, assignment, frozen decision rules, confounds. - YYYY-MM-DD: started (deploy sha / PR). ``` -## Open (9) +## Open (10) | check_on | experiment | status | primary metric | hypothesis | | --- | --- | --- | --- | --- | @@ -66,6 +66,7 @@ Design, assignment, frozen decision rules, confounds. | 2026-11-20 | [crosspost_content_tags_v5c](2026-10-31-crosspost-content-tags.md) | running | Gate A kappa >=0.6; Gate B refit coefficient <= -0.05 with CI excluding 0 | Relatable jokes and tweet screenshots get ~8-10% fewer RU forwards at equal source and likes; down-weighting them raises forwards per post | | 2026-11-20 | [explore_budget_v1](2026-10-30-explore-budget.md) | running | likes per active user per week, memes per session | Spend exploration only on labeling fresh memes, show fresh winners to everyone, and fill the idle-return slot from like_spread | | 2026-11-21 | [raw_impr_rank_reaction_rate](2026-10-31-raw-impr-rank-reaction-rate.md) | running | like rate of memes with raw_impr_rank = 0, 3 weeks before vs after deploy | Within-source reaction rate (reactions/views) beats views for ranking cold memes | +| 2026-11-28 | [crosspost_quirk_fix_ab](2026-11-01-crosspost-quirk-fix-ab.md) | running | mean zf (fix - control) per channel, 95% CI | Fixing the two crosspost scoring quirks (impression penalty 0 instead of 0.8 RU / 0.5 EN; x0.5 instead of x1.0 for sources without history) lets better memes reach the channels and raises forwards per post | | 2026-11-28 | [crosspost_src_fwd_v5f](2026-11-01-crosspost-src-forward-rate.md) | running | RU Spearman gain rho_v5f - rho_v5 with 95% CI excluding 0 | Memes whose source post was forwarded more than usual for that source get more forwards in our channels | | 2026-12-11 | [monthly_reminder_v1](2026-10-30-monthly-reminder.md) | running | react_30d_pct treated - holdout | One monthly HQ meme raises 30d activity of idle users who reacted before by >=3 pp at <2% blocks | diff --git a/src/crossposting/scoring.py b/src/crossposting/scoring.py index f91fb139..4c2b6f50 100644 --- a/src/crossposting/scoring.py +++ b/src/crossposting/scoring.py @@ -21,9 +21,20 @@ # impr_penalty is 0, not the intended 0.8 (RU) / 0.5 (EN): the old SQL bound it into # `CASE ... THEN 1 ELSE :impr_penalty END`, which Postgres typed integer, so asyncpg # sent 0. Production has always zeroed memes with raw_impr_rank > 1; kept as-is. +# The intended values live on in the quirk-fix A/B arm (v4q / v2q). CHANNEL_PARAMS: dict[str, dict[str, Any]] = { - "tgchannelru": {"impr_penalty": 0.0, "age_threshold": 7, "language_code": "ru"}, - "tgchannelen": {"impr_penalty": 0.0, "age_threshold": 90, "language_code": "en"}, + "tgchannelru": { + "impr_penalty": 0.0, + "impr_penalty_intended": 0.8, + "age_threshold": 7, + "language_code": "ru", + }, + "tgchannelen": { + "impr_penalty": 0.0, + "impr_penalty_intended": 0.5, + "age_threshold": 90, + "language_code": "en", + }, } @@ -61,6 +72,16 @@ def _v5_base(r: Row, channel: str) -> float | None: 1.0 if r["nmemes_sent"] <= 1 else (r["nlikes"] + r["ndislikes"]) / r["nmemes_sent"] ), "src_quality_mult": _src_quality_mult, + # quirk-fix arm (experiments/2026-11-01-crosspost-quirk-fix-ab.md): intended + # impression penalty, and a neutral 1.0 for sources without enough history + "impr_factor_q": lambda r, c: ( + 1.0 if r["raw_impr_rank"] <= 1 else CHANNEL_PARAMS[c]["impr_penalty_intended"] + ), + "src_quality_q": lambda r, c: ( + 1.0 + if r.get("src_signal") is None or not r.get("median_signal") + else _src_quality_mult(r, c) + ), "invited_boost": lambda r, c: 1.0 + min(r["invited_count"], 10) * 0.1, # v4: meme-level like volume (offline 2026-08-09) "like_volume_factor": lambda r, c: math.log(r["nlikes"] + 1.0), @@ -86,18 +107,26 @@ def _v5_base(r: Row, channel: str) -> float | None: "src_quality_mult", "invited_boost", ) +_V2Q = tuple( + {"impr_factor": "impr_factor_q", "src_quality_mult": "src_quality_q"}.get(f, f) for f in _V2 +) SCORES: dict[str, tuple[str, ...]] = { "v2": _V2, "v4": (*_V2, "like_volume_factor"), "v5": ("v5_base",), "v5c": ("v5_base", "content_mult_v1"), "v5f": ("v5_base", "src_fwd_mult"), + "v2q": _V2Q, + "v4q": (*_V2Q, "like_volume_factor"), } # crossposting.score_version / decision_log.score_version for the picking score. -SCORE_VERSION = {"v2": 2, "v4": 4, "v5": 5, "v5c": 6, "v5f": 7} +SCORE_VERSION = {"v2": 2, "v4": 4, "v5": 5, "v5c": 6, "v5f": 7, "v2q": 8, "v4q": 9} PRESELECT = {"tgchannelru": "v4", "tgchannelen": "v2"} LIVE = {"tgchannelru": "v4", "tgchannelen": "v2"} +# Live A/B: each decision picks one arm at random and uses it for both PRESELECT and +# LIVE. Remove the channel's entry to end the test (PRESELECT/LIVE apply again). +AB_ARMS = {"tgchannelru": ("v4", "v4q"), "tgchannelen": ("v2", "v2q")} def score(row: Row, channel: str, name: str) -> float | None: diff --git a/src/crossposting/service.py b/src/crossposting/service.py index 26bf24a8..a0877992 100644 --- a/src/crossposting/service.py +++ b/src/crossposting/service.py @@ -1,5 +1,6 @@ import heapq import logging +import random from typing import Any from sqlalchemy import text @@ -7,6 +8,7 @@ from src.crossposting.constants import Channel from src.crossposting.scoring import ( + AB_ARMS, CHANNEL_PARAMS, LIVE, PRESELECT, @@ -297,7 +299,9 @@ def _round(x: float | None, nd: int) -> float | None: return None if x is None else round(float(x), nd) -def _build_decision_log(channel: str, top: list[dict[str, Any]], pool_size: int) -> dict[str, Any]: +def _build_decision_log( + channel: str, top: list[dict[str, Any]], pool_size: int, live: str, pre: str +) -> dict[str, Any]: """Score the preselected top-N with every score; return log_ranker_decision kwargs.""" scored = [score_all(r, channel) for r in top] picks: dict[str, int] = {} @@ -311,7 +315,7 @@ def _build_decision_log(channel: str, top: list[dict[str, Any]], pool_size: int) ranks[i][name] = rank picks[name] = top[order[0]]["id"] # LIVE may need shadow inputs; if they failed to load, fall back to PRESELECT. - picked_by = LIVE[channel] if LIVE[channel] in picks else PRESELECT[channel] + picked_by = live if live in picks else pre log_candidates = [] for i, (row, (factors, scores)) in enumerate(zip(top, scored, strict=True)): @@ -377,12 +381,14 @@ async def get_next_meme_for_channel( rows = await fetch_all(text(_POOL_QUERY), params) if not rows: return None, None - pre = PRESELECT[channel] + pre, live = PRESELECT[channel], LIVE[channel] + if channel in AB_ARMS: + pre = live = random.choice(AB_ARMS[channel]) top = heapq.nsmallest(log_top_n, rows, key=lambda r: (-score(r, channel, pre), r["id"])) # Shadow inputs must never block posting. try: await _attach_shadow_inputs(top, channel) except Exception: logger.warning("crosspost shadow inputs failed", exc_info=True) - decision = _build_decision_log(channel, top, pool_size=len(rows)) + decision = _build_decision_log(channel, top, pool_size=len(rows), live=live, pre=pre) return await _picked_meme_dict(decision["picked_meme_id"]), decision diff --git a/tests/crossposting/test_scoring.py b/tests/crossposting/test_scoring.py index 90e56633..78bfefb3 100644 --- a/tests/crossposting/test_scoring.py +++ b/tests/crossposting/test_scoring.py @@ -2,7 +2,6 @@ import math -from src.crossposting import scoring from src.crossposting.scoring import FACTORS, score from src.crossposting.service import _build_decision_log @@ -64,11 +63,19 @@ def test_decision_log_ranks_picks_and_legacy_keys(): _cand(102, 100, theta_src=0.9, user_taste_z=0.0, src_fwd_pct=0.9), _cand(103, 100, theta_src=0.8, user_taste_z=0.0), ] - log = _build_decision_log("tgchannelru", cands, pool_size=50) + log = _build_decision_log("tgchannelru", cands, pool_size=50, live="v4", pre="v4") a, b, c = log["candidates"] assert log["picked_meme_id"] == 101 and log["score_version"] == 4 # live = v4, ties → id assert a["picked_by"] == "v4" and a["picks"]["v5"] == 101 - assert a["picks"] == {"v2": 101, "v4": 101, "v5": 101, "v5c": 102, "v5f": 102} + assert a["picks"] == { + "v2": 101, + "v4": 101, + "v5": 101, + "v5c": 102, + "v5f": 102, + "v2q": 101, + "v4q": 101, + } assert (a["ranks"]["v5f"], b["ranks"]["v5f"], c["ranks"]["v5f"]) == (3, 1, 2) # legacy flat keys used by docs/analyst readouts assert a["score_v5c"] == a["scores"]["v5c"] and a["final_score"] == a["scores"]["v4"] @@ -77,17 +84,27 @@ def test_decision_log_ranks_picks_and_legacy_keys(): assert c["src_fwd_pct"] is None and c["score_v5f"] == c["score_v5"] -def test_live_shadow_falls_back_to_preselect_without_inputs(monkeypatch): - monkeypatch.setitem(scoring.LIVE, "tgchannelen", "v5") - log = _build_decision_log("tgchannelen", [_cand(101, 100)], pool_size=1) +def test_live_shadow_falls_back_to_preselect_without_inputs(): + log = _build_decision_log("tgchannelen", [_cand(101, 100)], pool_size=1, live="v5", pre="v2") top = log["candidates"][0] assert top["picked_by"] == "v2" and log["score_version"] == 2 assert top["score_v5"] is None and "shadow_v5_rank" not in top -def test_live_shadow_picks_when_promoted(monkeypatch): - monkeypatch.setitem(scoring.LIVE, "tgchannelen", "v5") +def test_live_shadow_picks_when_promoted(): cands = [_cand(101, 100, theta_src=0.5), _cand(102, 20, theta_src=2.0)] - log = _build_decision_log("tgchannelen", cands, pool_size=2) + log = _build_decision_log("tgchannelen", cands, pool_size=2, live="v5", pre="v2") assert log["picked_meme_id"] == 102 and log["score_version"] == 5 assert log["candidates"][0]["final_score"] == log["candidates"][0]["score_v5"] + + +def test_quirk_fix_factors(): + f = FACTORS + low = _cand(1, 5, raw_impr_rank=2) + assert f["impr_factor"](low, "tgchannelru") == 0.0 + assert f["impr_factor_q"](low, "tgchannelru") == 0.8 + assert f["impr_factor_q"](low, "tgchannelen") == 0.5 + assert f["src_quality_mult"](_cand(1, 5, src_signal=None), "") == 0.5 + assert f["src_quality_q"](_cand(1, 5, src_signal=None), "") == 1.0 + assert f["src_quality_q"](_cand(1, 5, src_signal=100.0), "") == 2.0 + assert score(low, "tgchannelru", "v4") == 0.0 and score(low, "tgchannelru", "v4q") > 0 diff --git a/tests/test_crossposting_meme.py b/tests/test_crossposting_meme.py index 7860a520..c78fcf16 100644 --- a/tests/test_crossposting_meme.py +++ b/tests/test_crossposting_meme.py @@ -125,7 +125,8 @@ async def _insert_crossposting( @pytest_asyncio.fixture() -async def clean_xpost(): +async def clean_xpost(monkeypatch): + monkeypatch.setattr("src.crossposting.service.AB_ARMS", {}) async with engine.connect() as conn: await _wipe(conn) yield