diff --git a/docs/feature-flags.md b/docs/feature-flags.md index 47f7a8e04..5fb4acc95 100644 --- a/docs/feature-flags.md +++ b/docs/feature-flags.md @@ -57,7 +57,7 @@ it can never ship a flag that silently resolves to `undefined`. > **Every `example-*` key in this document is illustrative, not a real flag.** > Grep for one and you will find only this file. The registry holds exactly the > flags that gate live behaviour — today four rollout gates, all plain booleans -> (`insights-visual` also declares a 50/50 experiment that is not running): +> (`insights-visual` also runs a 50/50 experiment): > `retention-policies`, `lore-explorer-instruments`, > `lore-explorer-duplicate-clusters` and `insights-visual`. It has also held entries since removed: the > rollout gate for the `/insights` route (deleted once that page became @@ -260,7 +260,7 @@ nothing is the correct, expected symptom of that gap, not a broken query. Per-user splits need no storage: a signed-in user is bucketed by their Supabase user id (see [Targeting key resolution](#targeting-key-resolution)), so they see the same arm on every device for as long as the weights stand still. -`insights-visual` is declared this way and is the worked example. +`insights-visual` runs this way and is the worked example. ### 1. Declare the arms, dark diff --git a/packages/feature-flags/generated/flags.manifest.json b/packages/feature-flags/generated/flags.manifest.json index 5d3266310..95995b991 100644 --- a/packages/feature-flags/generated/flags.manifest.json +++ b/packages/feature-flags/generated/flags.manifest.json @@ -57,7 +57,7 @@ }, { "key": "insights-visual", - "description": "Insights — the visual first version of `/insights`: ONE page-wide window, the Learn → Recall → Promote → Prune loop strip, the Pulse timeline (lessons written and records read on a shared time axis, chosen vs bulk), the recall flow (client → tool → scope type), and runs drawn as a weekday × hour rhythm plus swimlanes — with the existing 'earning its place' grid, runs list and operational health kept below. Gates the whole page body through the `InsightsView` resolver; with it off `/insights` renders exactly as before. Reads only routes that already ship (`/memories/activity`, `/read-activity`, `/usage`, `/usage/runs`, `/utility`), so there is no backend prerequisite and flipping it is a pure UI decision. Declared as a 50/50 experiment that is NOT running (`enabled: false` resolves every user to `off`): launching it is flipping that one boolean, and `on` is listed first so ramping its weight up never moves a user from `on` back to `off`. Exposure is measured by `feature_flag.exposed`, emitted by the `InsightsView` resolver — see docs/feature-flags.md § Running an experiment.", + "description": "Insights — the visual first version of `/insights`: ONE page-wide window, the Learn → Recall → Promote → Prune loop strip, the Pulse timeline (lessons written and records read on a shared time axis, chosen vs bulk), the recall flow (client → tool → scope type), and runs drawn as a weekday × hour rhythm plus swimlanes — with the existing 'earning its place' grid, runs list and operational health kept below. Gates the whole page body through the `InsightsView` resolver; with it off `/insights` renders exactly as before. Reads only routes that already ship (`/memories/activity`, `/read-activity`, `/usage`, `/usage/runs`, `/utility`), so there is no backend prerequisite and flipping it is a pure UI decision. RUNNING as a 50/50 experiment: each signed-in user is assigned an arm by a hash of their user id (the same arm on every device), a developer override still wins, and setting `enabled: false` sends everyone back to `off`. `on` is listed first so ramping its weight up never moves a user from `on` back to `off`; lowering it or reordering the arms reassigns users mid-experiment. Exposure is measured by `feature_flag.exposed`, emitted by the `InsightsView` resolver — see docs/feature-flags.md § Running an experiment.", "type": "boolean", "variants": { "off": false, @@ -65,7 +65,7 @@ }, "defaultVariant": "off", "experiment": { - "enabled": false, + "enabled": true, "variants": [ { "key": "on", diff --git a/packages/feature-flags/src/client.spec.ts b/packages/feature-flags/src/client.spec.ts index 372920a93..c0fb1f286 100644 --- a/packages/feature-flags/src/client.spec.ts +++ b/packages/feature-flags/src/client.spec.ts @@ -65,6 +65,26 @@ describe('evaluateFlagDetails', () => { expect(details).toMatchObject({ value: false, variant: 'off', reason: 'OVERRIDE' }); }); + it('splits every running experiment by targeting key, near its declared weights', async () => { + // Derived from the registry, like the dark-experiment case below: a launch + // must change who sees what, report SPLIT, and honour the weights it declares. + const running = FLAG_REGISTRY.filter((flag) => flag.experiment?.enabled); + const users = Array.from({ length: 2_000 }, (_, i) => `user-${i}`); + for (const flag of running) { + const counts = new Map(); + for (const targetingKey of users) { + const details = await evaluateFlagDetails(flag.key as FlagKey, { targetingKey }); + expect(details.reason).toBe('SPLIT'); + counts.set(details.variant ?? '', (counts.get(details.variant ?? '') ?? 0) + 1); + } + for (const arm of flag.experiment?.variants ?? []) { + // ±5 points at n = 2,000 is more than three standard deviations for any weight. + expect(((counts.get(arm.key) ?? 0) / users.length) * 100).toBeGreaterThan(arm.weight - 5); + expect(((counts.get(arm.key) ?? 0) / users.length) * 100).toBeLessThan(arm.weight + 5); + } + } + }); + it('keeps every declared-but-disabled experiment on its default for every user', async () => { // A dark experiment must change nothing: no user lands on another arm and // none is reported as SPLIT. Derived from the registry rather than naming a diff --git a/packages/feature-flags/src/provider.spec.ts b/packages/feature-flags/src/provider.spec.ts index 1cefe23be..27768933f 100644 --- a/packages/feature-flags/src/provider.spec.ts +++ b/packages/feature-flags/src/provider.spec.ts @@ -139,6 +139,27 @@ describe('LoreKitFlagProvider', () => { expect(['control', 'treatment']).toContain(result.variant); }); + it('resolves a disabled experiment to its default variant with reason STATIC, whatever the targeting key', async () => { + // The rollback path: `enabled: false` must send every user back to the default and + // never report SPLIT. A fixture, so it holds whatever the live registry declares. + const darkFlag: FlagDefinition = { + ...experimentFlag, + key: 'dark-flag', + experiment: { + enabled: false, + variants: [ + { key: 'control', weight: 50 }, + { key: 'treatment', weight: 50 }, + ], + }, + }; + const provider = new LoreKitFlagProvider([darkFlag]); + for (const targetingKey of ['user-a', 'user-b', 'user-c', 'user-d', 'user-e']) { + const result = await provider.resolveBooleanEvaluation('dark-flag', true, { targetingKey }, NOOP_LOGGER); + expect(result).toMatchObject({ value: false, variant: 'control', reason: StandardResolutionReasons.STATIC }); + } + }); + it('resolves a string flag correctly', async () => { const provider = new LoreKitFlagProvider([stringFlag]); const result = await provider.resolveStringEvaluation('string-flag', '', {}, NOOP_LOGGER); diff --git a/packages/feature-flags/src/registry.ts b/packages/feature-flags/src/registry.ts index 51f600e0c..6b0c0e755 100644 --- a/packages/feature-flags/src/registry.ts +++ b/packages/feature-flags/src/registry.ts @@ -81,12 +81,12 @@ const REGISTRY_INPUT = [ { key: 'insights-visual', description: - "Insights — the visual first version of `/insights`: ONE page-wide window, the Learn → Recall → Promote → Prune loop strip, the Pulse timeline (lessons written and records read on a shared time axis, chosen vs bulk), the recall flow (client → tool → scope type), and runs drawn as a weekday × hour rhythm plus swimlanes — with the existing 'earning its place' grid, runs list and operational health kept below. Gates the whole page body through the `InsightsView` resolver; with it off `/insights` renders exactly as before. Reads only routes that already ship (`/memories/activity`, `/read-activity`, `/usage`, `/usage/runs`, `/utility`), so there is no backend prerequisite and flipping it is a pure UI decision. Declared as a 50/50 experiment that is NOT running (`enabled: false` resolves every user to `off`): launching it is flipping that one boolean, and `on` is listed first so ramping its weight up never moves a user from `on` back to `off`. Exposure is measured by `feature_flag.exposed`, emitted by the `InsightsView` resolver — see docs/feature-flags.md § Running an experiment.", + "Insights — the visual first version of `/insights`: ONE page-wide window, the Learn → Recall → Promote → Prune loop strip, the Pulse timeline (lessons written and records read on a shared time axis, chosen vs bulk), the recall flow (client → tool → scope type), and runs drawn as a weekday × hour rhythm plus swimlanes — with the existing 'earning its place' grid, runs list and operational health kept below. Gates the whole page body through the `InsightsView` resolver; with it off `/insights` renders exactly as before. Reads only routes that already ship (`/memories/activity`, `/read-activity`, `/usage`, `/usage/runs`, `/utility`), so there is no backend prerequisite and flipping it is a pure UI decision. RUNNING as a 50/50 experiment: each signed-in user is assigned an arm by a hash of their user id (the same arm on every device), a developer override still wins, and setting `enabled: false` sends everyone back to `off`. `on` is listed first so ramping its weight up never moves a user from `on` back to `off`; lowering it or reordering the arms reassigns users mid-experiment. Exposure is measured by `feature_flag.exposed`, emitted by the `InsightsView` resolver — see docs/feature-flags.md § Running an experiment.", type: 'boolean', variants: { off: false, on: true }, defaultVariant: 'off', experiment: { - enabled: false, + enabled: true, variants: [ { key: 'on', weight: 50, description: 'The visual page' }, { key: 'off', weight: 50, description: 'The original page (control)' },