diff --git a/AGENTS.md b/AGENTS.md index 107d4cd0..6adb7a47 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,8 @@ - Before designing an experiment, read [`experiments/README.md`](experiments/README.md): open checks plus every decided result and learning. - When starting or closing one, edit its file, then run `python scripts/experiments.py index` and commit the README (CI fails if it is stale). - `python scripts/experiments.py due` lists checks that are due; `readout ` prints the read-only SQL command for `$ANALYST_DATABASE_URL`. +- Metrics: use the `v_*` views and [`docs/analyst/north_star.sql`](docs/analyst/north_star.sql) (sessions, actives, shares, funnel); never redefine them ad hoc. +- Always exclude `segment = 'excluded'` (automated accounts, `v_metric_user`) from metrics and readouts. ## Operational notes - Manual upload review happens entirely inside the designated Telegram upload review chat. Keep communications and escalations there for traceability. diff --git a/SPEC.md b/SPEC.md index 854ea3df..a6f5df2e 100644 --- a/SPEC.md +++ b/SPEC.md @@ -8,6 +8,8 @@ User presses /start → meme with Like/Dislike → reaction triggers next meme. **Session north star**: session length (memes per session), not like rate alone. **Growth north star**: organic users via share deep links — see [docs/growth/virality-loop.md](docs/growth/virality-loop.md). +**Measured by**: [docs/analyst/north_star.sql](docs/analyst/north_star.sql) on the +shared `v_*` metric views (definitions: [docs/analyst/README.md](docs/analyst/README.md)). **Supply**: ETL of TG channels + VK publics → quality filters → recommendation pool. diff --git a/alembic/versions/2026-03-18_stats_perf_indexes.py b/alembic/versions/2026-03-18_stats_perf_indexes.py index 218434a6..e31340d8 100644 --- a/alembic/versions/2026-03-18_stats_perf_indexes.py +++ b/alembic/versions/2026-03-18_stats_perf_indexes.py @@ -33,10 +33,7 @@ def upgrade() -> None: # for an index. All queries that filter on reaction_id either: # - Filter within a per-user subset (uses PK) # - Filter globally (WHERE reaction_id = 1 matches 50%, seq scan is faster) - op.execute( - "DROP INDEX CONCURRENTLY IF EXISTS " - "user_meme_reaction_reaction_id_idx" - ) + op.execute("DROP INDEX CONCURRENTLY IF EXISTS user_meme_reaction_reaction_id_idx") # Tune autovacuum for user_meme_reaction (22M rows). # Default scale_factor=0.2 means vacuum triggers at 4.4M dead tuples. @@ -66,7 +63,4 @@ def downgrade() -> None: "ON user_meme_reaction (reaction_id)" ) - op.execute( - "DROP INDEX CONCURRENTLY IF EXISTS " - "ix_user_meme_reaction_reacted_at" - ) + op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_user_meme_reaction_reacted_at") diff --git a/alembic/versions/2026-03-22_chat_agent_tables.py b/alembic/versions/2026-03-22_chat_agent_tables.py index ed062a31..dea0dd46 100644 --- a/alembic/versions/2026-03-22_chat_agent_tables.py +++ b/alembic/versions/2026-03-22_chat_agent_tables.py @@ -43,9 +43,7 @@ def upgrade() -> None: server_default=sa.func.now(), nullable=False, ), - sa.UniqueConstraint( - "chat_id", "meme_id", "user_id", name="uq_chat_meme_reaction" - ), + sa.UniqueConstraint("chat_id", "meme_id", "user_id", name="uq_chat_meme_reaction"), ) op.create_table( @@ -53,15 +51,9 @@ def upgrade() -> None: sa.Column("id", sa.Integer(), sa.Identity(), primary_key=True), sa.Column("chat_id", sa.BigInteger(), nullable=False), sa.Column("user_id", sa.BigInteger(), nullable=False), - sa.Column( - "prompt_tokens", sa.Integer(), nullable=False, server_default="0" - ), - sa.Column( - "completion_tokens", sa.Integer(), nullable=False, server_default="0" - ), - sa.Column( - "tool_calls", sa.Integer(), nullable=False, server_default="0" - ), + sa.Column("prompt_tokens", sa.Integer(), nullable=False, server_default="0"), + sa.Column("completion_tokens", sa.Integer(), nullable=False, server_default="0"), + sa.Column("tool_calls", sa.Integer(), nullable=False, server_default="0"), sa.Column("response_time_ms", sa.Integer()), sa.Column("trigger_type", sa.String()), sa.Column( diff --git a/alembic/versions/2026-03-23_telegram_chat_and_wrapped.py b/alembic/versions/2026-03-23_telegram_chat_and_wrapped.py index 8f4aeb6e..c2e233cf 100644 --- a/alembic/versions/2026-03-23_telegram_chat_and_wrapped.py +++ b/alembic/versions/2026-03-23_telegram_chat_and_wrapped.py @@ -28,12 +28,8 @@ def upgrade() -> None: sa.Column("bot_status", sa.String(20)), sa.Column("bot_joined_at", sa.DateTime()), sa.Column("bot_left_at", sa.DateTime()), - sa.Column( - "created_at", sa.DateTime(), server_default=sa.func.now() - ), - sa.Column( - "updated_at", sa.DateTime(), server_default=sa.func.now() - ), + sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()), + sa.Column("updated_at", sa.DateTime(), server_default=sa.func.now()), ) op.add_column( @@ -52,9 +48,7 @@ def upgrade() -> None: ), sa.Column("data", JSONB, nullable=False), sa.Column("card_telegram_file_id", sa.String()), - sa.Column( - "created_at", sa.DateTime(), server_default=sa.func.now() - ), + sa.Column("created_at", sa.DateTime(), server_default=sa.func.now()), ) diff --git a/alembic/versions/2026-04-01_deep_link_log_index.py b/alembic/versions/2026-04-01_deep_link_log_index.py index 73252e3d..96fd699a 100644 --- a/alembic/versions/2026-04-01_deep_link_log_index.py +++ b/alembic/versions/2026-04-01_deep_link_log_index.py @@ -38,6 +38,4 @@ def upgrade() -> None: def downgrade() -> None: op.execute("COMMIT") - op.execute( - "DROP INDEX CONCURRENTLY IF EXISTS ix_user_deep_link_log_deep_link" - ) + op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_user_deep_link_log_deep_link") diff --git a/alembic/versions/2026-04-01_meme_source_id_index.py b/alembic/versions/2026-04-01_meme_source_id_index.py index cfc28c6c..76d3c816 100644 --- a/alembic/versions/2026-04-01_meme_source_id_index.py +++ b/alembic/versions/2026-04-01_meme_source_id_index.py @@ -23,15 +23,11 @@ def upgrade() -> None: # Without an index the planner does a full sequential sort on every 15-min stats run, # causing the 300s timeout. CONCURRENTLY avoids table lock on the large meme table. op.execute( - "CREATE INDEX CONCURRENTLY IF NOT EXISTS " - "ix_meme_meme_source_id " - "ON meme (meme_source_id)" + "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_meme_meme_source_id ON meme (meme_source_id)" ) def downgrade() -> None: op.execute("COMMIT") - op.execute( - "DROP INDEX CONCURRENTLY IF EXISTS ix_meme_meme_source_id" - ) + op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_meme_meme_source_id") diff --git a/alembic/versions/2026-04-13_channel_growth_measurement.py b/alembic/versions/2026-04-13_channel_growth_measurement.py index e0810a9b..487891e6 100644 --- a/alembic/versions/2026-04-13_channel_growth_measurement.py +++ b/alembic/versions/2026-04-13_channel_growth_measurement.py @@ -53,9 +53,7 @@ def upgrade() -> None: sa.Column("comments", sa.Integer()), sa.Column("reactions_detail", JSONB), sa.Column("message_text", sa.String()), - sa.Column( - "snapshot_at", sa.DateTime(), server_default=sa.func.now(), nullable=False - ), + sa.Column("snapshot_at", sa.DateTime(), server_default=sa.func.now(), nullable=False), ) op.create_index( "ix_crossposting_snapshots_channel_msg", @@ -73,9 +71,7 @@ def upgrade() -> None: sa.Column("posts_count", sa.Integer()), sa.Column("total_forwards", sa.Integer()), sa.Column("total_views", sa.Integer()), - sa.Column( - "created_at", sa.DateTime(), server_default=sa.func.now(), nullable=False - ), + sa.Column("created_at", sa.DateTime(), server_default=sa.func.now(), nullable=False), sa.UniqueConstraint("channel", "date"), ) diff --git a/alembic/versions/2026-11-02_metrics_views.py b/alembic/versions/2026-11-02_metrics_views.py new file mode 100644 index 00000000..f910f132 --- /dev/null +++ b/alembic/versions/2026-11-02_metrics_views.py @@ -0,0 +1,289 @@ +"""Shared metrics layer: analytics_excluded_user + v_* views. + +THE metric definitions for agents and experiment readouts +(docs/analyst/north_star.sql). Plain views; filter on their date columns. +Feed vs push: recommended_by LIKE 'broadcast%' is a push, everything else feed. +analytics_excluded_user is filled on prod by an operator +(docs/analyst/excluded-users-detector.sql), never by migrations. + +Revision ID: a7c3e9f1b5d2 +Revises: e5a7c9b1d3f4 +Create Date: 2026-11-02 +""" + +from alembic import op + +revision = "a7c3e9f1b5d2" +down_revision = "e5a7c9b1d3f4" +branch_labels = None +depends_on = None + +EXCLUDED_TABLE = """ +CREATE TABLE IF NOT EXISTS analytics_excluded_user ( + user_id BIGINT PRIMARY KEY, + reason TEXT NOT NULL, + created_at TIMESTAMP DEFAULT now() +) +""" + +# (name, select) in dependency order. +VIEWS = [ + ( + "v_metric_user", + """ +SELECT + U.id AS user_id, + CASE + WHEN X.user_id IS NOT NULL THEN 'excluded' + WHEN U.type IN ('moderator', 'admin', 'super_user') THEN U.type + ELSE 'user' + END AS segment, + U.created_at AS signup_at, + U.created_at::date AS signup_date, + -- First /start deep link within a day of signup; 'unknown' = no log row + -- (users older than the deep-link log). + CASE + WHEN DL.user_id IS NULL THEN 'unknown' + WHEN DL.deep_link IS NULL THEN 'organic' + WHEN DL.deep_link ~ '^[ms]_[0-9]+_[0-9]+$' THEN 'share' + WHEN DL.deep_link LIKE 'sc\\_%' THEN 'channel' + ELSE COALESCE(SUBSTRING(DL.deep_link FROM '^[A-Za-z]+'), 'other') + END AS acquisition_channel, + L.languages +FROM "user" U +LEFT JOIN analytics_excluded_user X + ON X.user_id = U.id +LEFT JOIN ( + SELECT DISTINCT ON (user_id) user_id, deep_link, created_at + FROM user_deep_link_log + ORDER BY user_id, created_at, id +) DL + ON DL.user_id = U.id + AND DL.created_at < U.created_at + INTERVAL '1 day' +LEFT JOIN LATERAL ( + SELECT STRING_AGG(UL.language_code, ',' ORDER BY UL.language_code) AS languages + FROM user_language UL + WHERE UL.user_id = U.id +) L ON TRUE +""", + ), + ( + "v_feed_session", + # A feed send starts a session when the same user had no feed send in + # the previous 30 minutes. Filter on session_start (pushes to sent_at). + """ +SELECT + S.user_id, + S.sent_at AS session_start, + X.memes_sent, + X.reactions, + X.likes, + EXISTS ( + SELECT 1 + FROM user_meme_reaction P + WHERE P.user_id = S.user_id + AND P.recommended_by LIKE 'broadcast%' + AND P.reacted_at >= S.sent_at - INTERVAL '30 minutes' + AND P.reacted_at <= S.sent_at + -- Opened soon after delivery. Idle users always have a push as the + -- last chat message, so without this every return looks push-driven. + AND P.reacted_at - P.sent_at <= INTERVAL '15 minutes' + ) AS started_by_push +FROM user_meme_reaction S +-- LATERAL ... LIMIT 1, not NOT EXISTS: keeps the planner on per-row index +-- probes instead of hashing the whole table when the date filter misestimates. +LEFT JOIN LATERAL ( + SELECT 1 AS hit + FROM user_meme_reaction P + WHERE P.user_id = S.user_id + AND P.recommended_by NOT LIKE 'broadcast%' + AND P.sent_at >= S.sent_at - INTERVAL '30 minutes' + AND P.sent_at <= S.sent_at + AND (P.sent_at < S.sent_at OR P.meme_id < S.meme_id) + LIMIT 1 +) PREV ON TRUE +LEFT JOIN LATERAL ( + -- ponytail: sessions are capped at 24h; raise the cap if a real user + -- ever scrolls longer without a 30-minute break. + SELECT + COUNT(*)::int AS memes_sent, + COUNT(G.reaction_id)::int AS reactions, + COUNT(*) FILTER (WHERE G.reaction_id = 1)::int AS likes + FROM ( + SELECT + B.reaction_id, + SUM(B.gap) OVER (ORDER BY B.sent_at, B.meme_id) AS breaks + FROM ( + SELECT + R.sent_at, + R.meme_id, + R.reaction_id, + CASE + WHEN R.sent_at - LAG(R.sent_at) OVER (ORDER BY R.sent_at, R.meme_id) + > INTERVAL '30 minutes' + THEN 1 ELSE 0 + END AS gap + FROM user_meme_reaction R + WHERE R.user_id = S.user_id + AND R.recommended_by NOT LIKE 'broadcast%' + AND R.sent_at >= S.sent_at + AND R.sent_at < S.sent_at + INTERVAL '1 day' + ) B + ) G + WHERE G.breaks = 0 +) X ON TRUE +WHERE S.recommended_by NOT LIKE 'broadcast%' + AND PREV.hit IS NULL +""", + ), + ( + "v_user_day", + # Reactions are attributed to the day the meme was SENT (sent_at), not reacted_at. + # memes_sent / reactions / likes: all sends (feed + push) by sent_at day. + # sessions: feed sessions started that day. + # shares: distinct non-self clicks (clicker x meme) on this user's + # in-bot share links (m_/s_{sharer}_{meme}), by click day. + """ +SELECT + E.user_id, + E.day, + SUM(E.memes_sent)::int AS memes_sent, + SUM(E.reactions)::int AS reactions, + SUM(E.likes)::int AS likes, + SUM(E.sessions)::int AS sessions, + SUM(E.shares)::int AS shares +FROM ( + SELECT + R.user_id, + R.sent_at::date AS day, + 1 AS memes_sent, + CASE WHEN R.reaction_id IS NOT NULL THEN 1 ELSE 0 END AS reactions, + CASE WHEN R.reaction_id = 1 THEN 1 ELSE 0 END AS likes, + 0 AS sessions, + 0 AS shares + FROM user_meme_reaction R + UNION ALL + SELECT F.user_id, F.session_start::date, 0, 0, 0, 1, 0 + FROM v_feed_session F + UNION ALL + SELECT C.sharer_id, C.day, 0, 0, 0, 0, 1 + FROM ( + SELECT DISTINCT + SPLIT_PART(D.deep_link, '_', 2)::bigint AS sharer_id, + D.user_id AS clicker_id, + SPLIT_PART(D.deep_link, '_', 3) AS meme_id, + D.created_at::date AS day + FROM user_deep_link_log D + WHERE D.deep_link ~ '^[ms]_[0-9]+_[0-9]+$' + ) C + WHERE C.sharer_id <> C.clicker_id +) E +GROUP BY E.user_id, E.day +""", + ), + ( + "v_new_user", + # One row per signup. first_tap / first_like = reaction on the first + # meme sent. d1 / d7 = any reaction 24-48h / 7-8d after signup (NULL + # until observable). blocked_within_24h uses blocked_bot_at, which only + # keeps the latest block (cleared on unblock). + """ +SELECT + MU.user_id, + MU.segment, + MU.signup_at, + MU.acquisition_channel, + F.sent_at IS NOT NULL AS got_first_meme, + COALESCE(F.reaction_id IS NOT NULL, FALSE) AS first_tap, + COALESCE(F.reaction_id = 1, FALSE) AS first_like, + CASE WHEN now() >= MU.signup_at + INTERVAL '8 days' THEN A.reactions_7d >= 5 END + AS reached_5_reactions, + CASE WHEN now() >= MU.signup_at + INTERVAL '2 days' THEN A.d1 END AS d1_return, + CASE WHEN now() >= MU.signup_at + INTERVAL '8 days' THEN A.d7 END AS d7_return, + COALESCE(U.blocked_bot_at < U.created_at + INTERVAL '24 hours', FALSE) + AS blocked_within_24h +FROM v_metric_user MU +JOIN "user" U + ON U.id = MU.user_id +LEFT JOIN LATERAL ( + SELECT R.sent_at, R.reaction_id + FROM user_meme_reaction R + WHERE R.user_id = MU.user_id + ORDER BY R.sent_at, R.meme_id + LIMIT 1 +) F ON TRUE +CROSS JOIN LATERAL ( + SELECT + COUNT(*) FILTER (WHERE R.reacted_at < MU.signup_at + INTERVAL '7 days') AS reactions_7d, + COALESCE(BOOL_OR( + R.reacted_at >= MU.signup_at + INTERVAL '1 day' + AND R.reacted_at < MU.signup_at + INTERVAL '2 days' + ), FALSE) AS d1, + COALESCE(BOOL_OR(R.reacted_at >= MU.signup_at + INTERVAL '7 days'), FALSE) AS d7 + FROM user_meme_reaction R + WHERE R.user_id = MU.user_id + AND R.reaction_id IS NOT NULL + AND R.reacted_at < MU.signup_at + INTERVAL '8 days' +) A +""", + ), + ( + "v_experiment_user", + # One row per experiment assignment; outcomes from assigned_at to now (open-ended: + # bound by the experiment end date in the readout, e.g. AND day < decided). + # reactions / likes count feed + push; sessions are feed sessions that + # started after assignment. Average these per variant (user-level means). + """ +SELECT + EA.experiment_id, + EA.variant, + EA.user_id, + EA.assigned_at, + MU.segment, + W.weeks, + S.memes_sent, + S.reactions, + S.likes, + FS.sessions, + FS.session_memes, + S.reactions / W.weeks AS reactions_per_week, + S.likes / W.weeks AS likes_per_week, + FS.session_memes::numeric / NULLIF(FS.sessions, 0) AS memes_per_session +FROM experiment_assignment EA +JOIN v_metric_user MU + ON MU.user_id = EA.user_id +CROSS JOIN LATERAL ( + SELECT GREATEST(EXTRACT(EPOCH FROM now() - EA.assigned_at) / 604800.0, 1.0 / 7) AS weeks +) W +CROSS JOIN LATERAL ( + SELECT + COUNT(*)::int AS memes_sent, + COUNT(R.reaction_id)::int AS reactions, + COUNT(*) FILTER (WHERE R.reaction_id = 1)::int AS likes + FROM user_meme_reaction R + WHERE R.user_id = EA.user_id + AND R.sent_at >= EA.assigned_at +) S +CROSS JOIN LATERAL ( + SELECT COUNT(*)::int AS sessions, COALESCE(SUM(F.memes_sent), 0)::int AS session_memes + FROM v_feed_session F + WHERE F.user_id = EA.user_id + AND F.session_start >= EA.assigned_at +) FS +""", + ), +] + + +def upgrade() -> None: + op.execute("SET lock_timeout = '5s'") + op.execute(EXCLUDED_TABLE) + for name, select in VIEWS: + op.execute(f"CREATE OR REPLACE VIEW {name} AS {select}") + + +def downgrade() -> None: + op.execute("SET lock_timeout = '5s'") + for name, _ in reversed(VIEWS): + op.execute(f"DROP VIEW IF EXISTS {name}") + op.execute("DROP TABLE IF EXISTS analytics_excluded_user") diff --git a/docs/analyst/README.md b/docs/analyst/README.md index cf53ce55..bde2b6f1 100644 --- a/docs/analyst/README.md +++ b/docs/analyst/README.md @@ -8,6 +8,19 @@ These queries and reports support product-health monitoring and experiment analy - **Statement timeout**: 30 seconds (kills runaway queries) - **Future tables**: Automatically accessible (DEFAULT PRIVILEGES granted) +## Metric definitions (use these, not ad-hoc SQL) +Sessions, actives, shares and funnel steps are defined ONCE, as DB views +(`alembic/versions/2026-11-02_metrics_views.py`). Older SQL in this folder +uses other session rules (reaction gaps, >= 2 memes); do not copy it. +- `v_metric_user` — segment (`excluded` / `moderator` / `admin` / `user`), signup, acquisition channel (first `/start` deep link), languages. +- `v_feed_session` — feed sends only (`recommended_by NOT LIKE 'broadcast%'`), new session after a 30-min gap; memes_sent, reactions, likes, started_by_push. +- `v_user_day` — per user per day: memes_sent / reactions / likes (feed + push), feed sessions, non-self share clicks on the user's links. +- `v_new_user` — per signup: got_first_meme, first_tap, first_like, reached_5_reactions (7d), d1/d7 return, blocked_within_24h. +- `v_experiment_user` — per `experiment_assignment`: outcomes since assignment, for user-level-mean readouts. +- `docs/analyst/north_star.sql` — THE north-star queries built on the views. +- Always drop `segment = 'excluded'` (automated accounts in `analytics_excluded_user`; candidates from `excluded-users-detector.sql`, inserted by an operator). Drop moderators/admins too when volume metrics matter (they work `low_sent_pool`). +- Filter views on their date columns with plain bounds (`day >= CURRENT_DATE - 28`), not `DATE_TRUNC(now())`. + ## Key Files - `docs/analyst/metrics.sql` — SQL queries organized by section (health, north star, engines, retention, etc.) - `docs/analyst/command-surface-proxies.sql` — private slash commands from `message_tg` (`chat_id > 0`) + secondary feature proxies. Notes: [`docs/product/surface-and-commands.md`](../product/surface-and-commands.md). @@ -85,4 +98,4 @@ Operational health signals: Healthy product snapshot (rough, not ship gates): new_memes_24h > 100, ok_pct ~90–96%, active reactors 24h not collapsing vs prior week, multi-engine mix still present. -Queries: `docs/analyst/metrics.sql`. +Queries: `docs/analyst/north_star.sql` (north star), `docs/analyst/metrics.sql` (operational health). diff --git a/docs/analyst/excluded-users-detector.sql b/docs/analyst/excluded-users-detector.sql new file mode 100644 index 00000000..98867cb8 --- /dev/null +++ b/docs/analyst/excluded-users-detector.sql @@ -0,0 +1,39 @@ +-- Candidate automated accounts for analytics_excluded_user (read-only SELECT). +-- Rule (2026-10-31 investigation): >= 50 push reactions and >= 50% of them in +-- ONE 10-second bin of (reacted_at - sent_at). A human opens pushes at random +-- delays; a script answers at a fixed one. +-- Push = user_meme_reaction.recommended_by LIKE 'broadcast%' +-- (broadcast_reengagement[_hq], broadcast_channel_viral, ...). +-- +-- Operator, on prod after reviewing the output (user IDs stay in the DB, +-- never in the repo): +-- INSERT INTO analytics_excluded_user (user_id, reason) +-- +-- ON CONFLICT (user_id) DO NOTHING; +WITH per_bin AS ( + SELECT + user_id, + FLOOR(EXTRACT(EPOCH FROM reacted_at - sent_at) / 10)::int AS bin, + COUNT(*) AS n, + SUM(COUNT(*)) OVER (PARTITION BY user_id) AS total + FROM user_meme_reaction + WHERE recommended_by LIKE 'broadcast%' + AND reaction_id IS NOT NULL + AND reacted_at IS NOT NULL + GROUP BY 1, 2 +), +top_bin AS ( + SELECT DISTINCT ON (user_id) user_id, bin, n, total + FROM per_bin + ORDER BY user_id, n DESC +) +SELECT + user_id, + FORMAT( + 'push-timing bot: %s of %s push reactions at %s-%ss after send', + n, total, bin * 10, bin * 10 + 10 + ) AS reason +FROM top_bin +WHERE total >= 50 + AND n >= 0.5 * total +ORDER BY user_id diff --git a/docs/analyst/north_star.sql b/docs/analyst/north_star.sql new file mode 100644 index 00000000..516cdade --- /dev/null +++ b/docs/analyst/north_star.sql @@ -0,0 +1,95 @@ +-- North star: THE metric definitions (read-only; built only on the v_* views +-- from alembic/versions/2026-11-02_metrics_views.py). +-- Window: the 28 days ending yesterday, in four 7-day buckets labeled by +-- their first day. Keep bounds as cheap date arithmetic (CURRENT_DATE - n): +-- view date filters run per row on user_meme_reaction, and DATE_TRUNC/EXTRACT +-- bounds make Q2/Q4 3-10x slower. +-- Every query drops segment = 'excluded' (automated accounts, see +-- excluded-users-detector.sql). +-- Moderators/admins are also dropped from Q2 and Q3: they work the +-- low_sent_pool review queue, which inflates reactions and session length. +-- They stay in Q1 (signups) and Q4 (their shares are real shares). +-- +-- Run: psql "$ANALYST_DATABASE_URL" -f docs/analyst/north_star.sql + +-- Q1. New users per week by acquisition channel (first /start deep link). +SELECT + CURRENT_DATE - 28 + 7 * ((signup_at::date - (CURRENT_DATE - 28)) / 7) AS week, + acquisition_channel, + COUNT(*) AS new_users +FROM v_metric_user +WHERE signup_at >= CURRENT_DATE - 28 + AND signup_at < CURRENT_DATE + AND segment <> 'excluded' +GROUP BY 1, 2 +ORDER BY 1, 3 DESC; + +-- Q2. Reactions per active user per week (active = >= 1 reaction that week). +SELECT + CURRENT_DATE - 28 + 7 * ((D.day - (CURRENT_DATE - 28)) / 7) AS week, + COUNT(DISTINCT D.user_id) AS active_users, + SUM(D.reactions) AS reactions, + ROUND(SUM(D.reactions)::numeric / COUNT(DISTINCT D.user_id), 1) AS reactions_per_active +FROM v_user_day D +JOIN v_metric_user U + ON U.user_id = D.user_id +WHERE D.day >= CURRENT_DATE - 28 + AND D.day < CURRENT_DATE + AND D.reactions > 0 + AND U.segment = 'user' +GROUP BY 1 +ORDER BY 1; + +-- Q3. Median memes per feed session (30-minute gap; push sends excluded). +SELECT + CURRENT_DATE - 28 + 7 * ((S.session_start::date - (CURRENT_DATE - 28)) / 7) AS week, + COUNT(*) AS sessions, + PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY S.memes_sent) AS median_memes_per_session, + ROUND(AVG(S.memes_sent), 1) AS avg_memes_per_session, + ROUND(AVG(S.started_by_push::int), 3) AS push_started_share +FROM v_feed_session S +JOIN v_metric_user U + ON U.user_id = S.user_id +WHERE S.session_start >= CURRENT_DATE - 28 + AND S.session_start < CURRENT_DATE + AND U.segment = 'user' +GROUP BY 1 +ORDER BY 1; + +-- Q4. Non-self share clicks and invites (new users whose first link was a +-- share link) per 1k active users per week. +WITH weekly AS ( + SELECT + CURRENT_DATE - 28 + 7 * ((D.day - (CURRENT_DATE - 28)) / 7) AS week, + COUNT(DISTINCT D.user_id) FILTER (WHERE D.reactions > 0) AS active_users, + SUM(D.shares) AS shares + FROM v_user_day D + JOIN v_metric_user U + ON U.user_id = D.user_id + WHERE D.day >= CURRENT_DATE - 28 + AND D.day < CURRENT_DATE + AND U.segment <> 'excluded' + GROUP BY 1 +), +invites AS ( + SELECT + CURRENT_DATE - 28 + 7 * ((signup_at::date - (CURRENT_DATE - 28)) / 7) AS week, + COUNT(*) AS invites + FROM v_metric_user + WHERE signup_at >= CURRENT_DATE - 28 + AND signup_at < CURRENT_DATE + AND acquisition_channel = 'share' + AND segment <> 'excluded' + GROUP BY 1 +) +SELECT + W.week, + W.active_users, + W.shares, + COALESCE(I.invites, 0) AS invites, + ROUND(1000.0 * W.shares / NULLIF(W.active_users, 0), 1) AS shares_per_1k_active, + ROUND(1000.0 * COALESCE(I.invites, 0) / NULLIF(W.active_users, 0), 1) AS invites_per_1k_active +FROM weekly W +LEFT JOIN invites I + ON I.week = W.week +ORDER BY 1; diff --git a/experiments/2026-10-30-explore-budget.md b/experiments/2026-10-30-explore-budget.md index 89c9fe76..46ec5495 100644 --- a/experiments/2026-10-30-explore-budget.md +++ b/experiments/2026-10-30-explore-budget.md @@ -130,3 +130,10 @@ Readout on 2026-11-20: - **Primary:** exclude sends of these ids from both arms. Also report the share of control low_sent sends that went to them. - **Sensitivity:** all sends. Use user-level means, not pooled. + +## Note: shared metric views (2026-11-02) +The readout SQL keeps its own session logic (frozen with the decision rule; +it also needs dedup-id exclusion and pos2/fresh_ru guardrails the views do not +have). On 2026-11-20 also report `v_experiment_user` user-level means +(`reactions_per_week`, `likes_per_week`, `memes_per_session`, segment +`<> 'excluded'`) as a cross-check; definitions in `docs/analyst/north_star.sql`. diff --git a/tests/test_metrics_views.py b/tests/test_metrics_views.py new file mode 100644 index 00000000..f22c5200 --- /dev/null +++ b/tests/test_metrics_views.py @@ -0,0 +1,139 @@ +"""Metrics views (alembic 2026-11-02_metrics_views) on a tiny fixture.""" + +from datetime import datetime, timedelta + +import pytest +from sqlalchemy import text + +from src.database import engine, experiment_assignment, user_deep_link_log, user_meme_reaction +from tests.factories import ( + cleanup_test_data, + create_meme, + create_meme_source, + create_user, + create_user_language, +) + +SHARER, FRIEND, BOT = 10501, 10502, 10503 +T0 = datetime(2026, 10, 1, 12, 0) + + +@pytest.fixture(autouse=True) +async def fixture(): + async with engine.begin() as conn: + await create_meme_source(conn, 10500) + for meme_id in range(10500, 10510): + await create_meme(conn, meme_id, 10500) + for uid in (SHARER, FRIEND, BOT): + await create_user(conn, uid) + await create_user_language(conn, uid) + await conn.execute(text('UPDATE "user" SET created_at = :t WHERE id >= 10501'), {"t": T0}) + await conn.execute( + text("INSERT INTO analytics_excluded_user (user_id, reason) VALUES (:u, 'test')"), + {"u": BOT}, + ) + await conn.execute( + user_deep_link_log.insert(), + [ + {"user_id": SHARER, "deep_link": None, "created_at": T0}, + {"user_id": FRIEND, "deep_link": f"m_{SHARER}_10500", "created_at": T0}, + {"user_id": SHARER, "deep_link": f"m_{SHARER}_10500", "created_at": T0}, # self + ], + ) + # SHARER: session 1 = 3 feed memes (2 likes), 2h gap, push reacted, + # session 2 = 2 feed memes; day 2 one more feed meme. + sends = [ + (10500, "lr_smoothed", T0, 1), + (10501, "lr_smoothed", T0 + timedelta(minutes=1), 1), + (10502, "lr_smoothed", T0 + timedelta(minutes=2), 2), + # push opened 10 min after delivery -> started_by_push + (10503, "broadcast_reengagement_hq", T0 + timedelta(hours=2, minutes=50), 1), + (10504, "recently_liked", T0 + timedelta(hours=3), None), + (10505, "recently_liked", T0 + timedelta(hours=3, minutes=1), 1), + (10506, "lr_smoothed", T0 + timedelta(days=1, hours=1), 1), + ] + await conn.execute( + user_meme_reaction.insert(), + [ + { + "user_id": SHARER, + "meme_id": m, + "recommended_by": rb, + "sent_at": t, + "reaction_id": r, + "reacted_at": (T0 + timedelta(hours=3) if rb.startswith("broadcast") else t) + if r + else None, + } + for m, rb, t, r in sends + ], + ) + await conn.execute( + experiment_assignment.insert(), + { + "experiment_id": "metrics_test", + "user_id": SHARER, + "variant": "control", + "assigned_at": T0 - timedelta(minutes=1), + }, + ) + yield + async with engine.begin() as conn: + await conn.execute( + experiment_assignment.delete().where( + experiment_assignment.c.experiment_id == "metrics_test" + ) + ) + await conn.execute(text("DELETE FROM analytics_excluded_user WHERE user_id >= 10500")) + await conn.execute(user_meme_reaction.delete().where(user_meme_reaction.c.user_id >= 10500)) + await cleanup_test_data(conn) + + +async def _rows(sql: str) -> list[dict]: + async with engine.connect() as conn: + return [dict(r._mapping) for r in await conn.execute(text(sql))] + + +@pytest.mark.asyncio +async def test_metric_user_segments_and_channels(): + rows = await _rows("SELECT * FROM v_metric_user WHERE user_id >= 10501") + rows = {r["user_id"]: r for r in rows} + assert rows[SHARER]["segment"] == "user" + assert rows[SHARER]["acquisition_channel"] == "organic" + assert rows[FRIEND]["acquisition_channel"] == "share" + assert rows[BOT]["segment"] == "excluded" + assert rows[BOT]["acquisition_channel"] == "unknown" + assert rows[SHARER]["languages"] == "ru" + + +@pytest.mark.asyncio +async def test_feed_sessions_split_on_30_min_gap(): + rows = await _rows( + f"SELECT * FROM v_feed_session WHERE user_id = {SHARER} ORDER BY session_start" + ) + assert [(r["memes_sent"], r["reactions"], r["likes"], r["started_by_push"]) for r in rows] == [ + (3, 3, 2, False), + (2, 1, 1, True), + (1, 1, 1, False), + ] + + +@pytest.mark.asyncio +async def test_user_day_counts_non_self_shares(): + rows = await _rows(f"SELECT * FROM v_user_day WHERE user_id = {SHARER} ORDER BY day") + day1 = rows[0] + assert (day1["memes_sent"], day1["reactions"], day1["likes"]) == (6, 5, 4) + assert (day1["sessions"], day1["shares"]) == (2, 1) + assert rows[1]["sessions"] == 1 + + +@pytest.mark.asyncio +async def test_new_user_and_experiment_user(): + nu = (await _rows(f"SELECT * FROM v_new_user WHERE user_id = {SHARER}"))[0] + assert nu["got_first_meme"] and nu["first_tap"] and nu["first_like"] + assert nu["reached_5_reactions"] and nu["d1_return"] and not nu["d7_return"] + assert not nu["blocked_within_24h"] + + eu = (await _rows("SELECT * FROM v_experiment_user WHERE experiment_id = 'metrics_test'"))[0] + assert (eu["memes_sent"], eu["reactions"], eu["likes"], eu["sessions"]) == (7, 6, 5, 3) + assert eu["memes_per_session"] == 2