diff --git a/.sampo/changesets/experiment-exposure.md b/.sampo/changesets/experiment-exposure.md new file mode 100644 index 000000000..622898ef0 --- /dev/null +++ b/.sampo/changesets/experiment-exposure.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: minor +--- + +Send `$experiment_exposure` for config version 2 experiment splits, and add rule attribution to `$feature_flag_called` for config version 2 flags. A read that sends `$feature_flag_called` now also sends `$experiment_exposure` when the flag was evaluated remotely and its config version 2 record is a complete experiment split: reason code `experiment_split`, rule type `experiment`, a rule ID and a variant key. This applies to `get_feature_flag()`, `feature_enabled()`, `get_feature_flag_result()`, and the `is_enabled()` and `get_flag()` reads of an `evaluate_flags()` snapshot. The exposure carries `$feature_flag`, the same `$feature_flag_response` as its `$feature_flag_called`, `$feature_flag_variant`, `$feature_flag_rule_id` and `locally_evaluated: false`, plus the flag ID, flag version, request ID and evaluation time when the response has them. It does not carry the config version, reason code or rule type, which stay on `$feature_flag_called`. Minimal-event mode does not apply to the exposure. The SDK sends one exposure per distinct ID, groups, flag, rule and variant. The value, request ID and evaluation time are not part of this dedupe, a flags reload does not reset it, and it is separate from the `$feature_flag_called` dedupe. No other outcome sends an exposure: not a targeting match, rollout miss, paused experiment, holdout or failed record, and not a config version 1 flag, a flag from a server without typed values or a locally evaluated flag. A config version 2 `$feature_flag_called` carries `$feature_flag_config_version` 2 and `$feature_flag_reason_code`, which replaces `$feature_flag_reason`. A complete experiment split also adds `$feature_flag_rule_id`, `$feature_flag_rule_type` and `$feature_flag_variant`. A targeting match (rule type `targeted_release` or `percentage_rollout`), a rollout miss (`percentage_rollout` or `experiment`), a paused experiment or a holdout (`experiment`) adds `$feature_flag_rule_id` and `$feature_flag_rule_type`. Other reasons and failed records add no rule properties. The rule ID and rule type are sent as a pair or not at all: when a record has only one of them, a rule type its reason does not allow, or a rule ID that is not a hyphenated UUID, the event drops both and keeps the config version and reason code. A split that is missing its rule type, rule ID or variant key sends no exposure and no rule properties. A reason code the SDK does not know is sent unchanged with no rule properties, and a record without a reason code still sends its config version. None of this changes the value a read returns. The `$feature_flag_called` dedupe of a config version 2 flag includes its attribution, so equal-valued arms, successive experiments of one flag and a holdout after a variant are each reported. Minimal `$feature_flag_called` events keep the attribution. The SDK tries the two events separately. If capture of one raises or drops it (a full or closed queue, opt-out or a disabled client), the other is still sent, and the failed event keeps no dedupe entry, so a later read tries it again. An event that `before_send` drops counts as handled: it keeps its dedupe entry, and later reads do not offer it again. `get_feature_flag_payload()` sends no flag events by default; a call with `match_value` reports the caller's value, so it sends no exposure. `send_feature_flag_events=False` suppresses both events. Events of config version 1 flags, flags from servers without typed values and locally evaluated flags do not change, including their `$feature_flag_reason`, and `$feature_flag_response` does not change on any event. diff --git a/posthog/args.py b/posthog/args.py index 42083b923..aeaf5a1d8 100644 --- a/posthog/args.py +++ b/posthog/args.py @@ -5,6 +5,9 @@ Any, Dict, FrozenSet, + List, + Mapping, + NamedTuple, Union, Tuple, Type, @@ -65,6 +68,27 @@ class OptionalCaptureArgs(TypedDict): _property_allowlist: NotRequired[ Optional[FrozenSet[str]] ] # Internal: strict allowlist applied to the fully-enriched event properties. Used by minimal $feature_flag_called events. + _pinned_properties: NotRequired[ + Optional["_PinnedProperties"] + ] # Internal: properties fixed after super properties are merged. Used by config version 2 flag events. + _before_send_drops: NotRequired[ + Optional[List[str]] + ] # Internal: gets the event name when before_send drops the event. Used by flag event dedupe. + + +class _PinnedProperties(NamedTuple): + """Internal: event properties that super properties cannot change. + + Applied to the fully-enriched properties, after super properties are merged and + before any allowlist: ``removed`` names are dropped, then ``values`` are set. + """ + + removed: FrozenSet[str] + values: Mapping[str, Any] + + def apply(self, properties: Dict[str, Any]) -> Dict[str, Any]: + kept = {k: v for k, v in properties.items() if k not in self.removed} + return {**kept, **self.values} class OptionalSetArgs(TypedDict): diff --git a/posthog/async_client.py b/posthog/async_client.py index 72f7cd2fe..10bdad53e 100644 --- a/posthog/async_client.py +++ b/posthog/async_client.py @@ -38,8 +38,8 @@ from .capture_mode import CaptureMode, _resolve_capture_mode from .client import ( MAX_DICT_SIZE as _MAX_DICT_SIZE, - _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES, Client as _SyncClient, + _capture_flag_events, add_context_tags as _add_context_tags, get_identity_state as _get_identity_state, stringify_id as _stringify_id, @@ -77,6 +77,7 @@ ) from .release_id import _resolve_release_id from .request import QuotaLimitError, determine_server_host, normalize_host +from .types import FeatureFlag as _FeatureFlag from .types import ( FlagValue, _has_untyped_flag_records, @@ -192,6 +193,9 @@ def __init__( 0, feature_flags_request_max_retries ) self.distinct_ids_feature_flags_reported = SizeLimitedDict(_MAX_DICT_SIZE, set) + # $experiment_exposure dedupe, kept apart from $feature_flag_called's so that a + # reported call never suppresses its exposure. Same bound and eviction. + self._experiment_exposures_reported = SizeLimitedDict(_MAX_DICT_SIZE, set) self._feature_flag_evaluations_host_cache: Optional[ _FeatureFlagEvaluationsHost ] = None @@ -463,6 +467,7 @@ def _prepare_event( msg: dict[str, Any], disable_geoip: Optional[bool], property_allowlist=None, + pinned_properties=None, ) -> tuple[Optional[dict[str, Any]], Optional[str]]: if self.disabled or not self._accepting: return None, None @@ -494,6 +499,10 @@ def _prepare_event( msg["properties"].setdefault("$release_id", self._release_id) if self.is_server: msg["properties"]["$is_server"] = True + # After every enrichment step, so a super property never changes or adds + # config version 2 flag attribution. + if pinned_properties is not None: + msg["properties"] = pinned_properties.apply(msg["properties"]) if property_allowlist is not None: msg["properties"] = { key: value @@ -526,7 +535,7 @@ async def _process_event(self, msg: dict[str, Any]) -> Optional[dict[str, Any]]: def _build_capture_event( self, event: str, kwargs: OptionalCaptureArgs - ) -> tuple[dict[str, Any], Optional[bool], Any]: + ) -> tuple[dict[str, Any], Optional[bool], Any, Any]: properties = {**(kwargs.get("properties") or {}), **system_context()} if self.capture_trace_context: properties = {**_get_current_otel_span_properties(), **properties} @@ -562,6 +571,7 @@ def _build_capture_event( }, kwargs.get("disable_geoip"), kwargs.get("_property_allowlist"), + kwargs.get("_pinned_properties"), ) def capture( @@ -569,11 +579,11 @@ def capture( ) -> Optional[str]: """Queue an event without blocking for network delivery.""" try: - msg, disable_geoip, property_allowlist = self._build_capture_event( - event, kwargs + msg, disable_geoip, property_allowlist, pinned_properties = ( + self._build_capture_event(event, kwargs) ) prepared, sent_uuid = self._prepare_event( - msg, disable_geoip, property_allowlist + msg, disable_geoip, property_allowlist, pinned_properties ) if prepared is None or sent_uuid is None: return None @@ -617,11 +627,11 @@ async def capture_immediate( self._immediate_callers[current] = self._immediate_callers.get(current, 0) + 1 error_batch: list[dict[str, Any]] = [] try: - msg, disable_geoip, property_allowlist = self._build_capture_event( - event, kwargs + msg, disable_geoip, property_allowlist, pinned_properties = ( + self._build_capture_event(event, kwargs) ) prepared, sent_uuid = self._prepare_event( - msg, disable_geoip, property_allowlist + msg, disable_geoip, property_allowlist, pinned_properties ) if prepared is None or sent_uuid is None: return None @@ -1085,38 +1095,29 @@ def _capture_feature_flag_called_if_needed( disable_geoip: Optional[bool] = None, has_experiment: Optional[bool] = None, minimal_flag_called_events: bool = False, + details: Optional[_FeatureFlag] = None, + value_from_details: bool = False, ) -> None: - groups_key = ( - tuple(sorted((str(k), str(v)) for k, v in groups.items())) if groups else () - ) - reported_key = (key, response, groups_key) - reported_flags = self.distinct_ids_feature_flags_reported.get(distinct_id) - if reported_flags is None: - reported_flags = set() - self.distinct_ids_feature_flags_reported[distinct_id] = reported_flags - if reported_key in reported_flags: - return - - if has_experiment is not None: - properties["$feature_flag_has_experiment"] = has_experiment - capture_kwargs: dict[str, Any] = {} - if minimal_flag_called_events and has_experiment is False: - capture_kwargs["_property_allowlist"] = ( - _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES - ) capture_groups = { str(group_type): _stringify_id(group_key) for group_type, group_key in (groups or {}).items() } - self.capture( - "$feature_flag_called", + _capture_flag_events( + self.capture, + self.distinct_ids_feature_flags_reported, + self._experiment_exposures_reported, distinct_id=distinct_id, + key=key, + response=response, properties=properties, - groups=capture_groups, + groups=groups, + capture_groups=capture_groups, disable_geoip=disable_geoip, - **capture_kwargs, + has_experiment=has_experiment, + minimal_flag_called_events=minimal_flag_called_events, + details=details, + value_from_details=value_from_details, ) - reported_flags.add(reported_key) def _pending_queue_items(self) -> int: return int(getattr(self._queue, "_unfinished_tasks", self._queue.qsize())) @@ -1278,6 +1279,7 @@ async def shutdown(self) -> None: errors.append(error) self.distinct_ids_feature_flags_reported.clear() + self._experiment_exposures_reported.clear() self._closed = True if errors and self.debug: raise errors[0] diff --git a/posthog/client.py b/posthog/client.py index ce5eb3352..7fbce7629 100644 --- a/posthog/client.py +++ b/posthog/client.py @@ -4,6 +4,7 @@ import json import logging import os +import re import sys import threading import time @@ -11,14 +12,33 @@ import weakref from contextvars import ContextVar from datetime import datetime, timedelta, timezone -from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union, cast +from typing import ( + Any, + Callable, + Dict, + FrozenSet, + List, + Mapping, + NamedTuple, + Optional, + Set, + Tuple, + Union, + cast, +) from uuid import UUID, uuid4 from typing_extensions import Unpack from posthog._async_utils import _BackgroundEventLoopRunner from posthog._disabled_lane_queue import _DisabledLaneQueue -from posthog.args import ID_TYPES, ExceptionArg, OptionalCaptureArgs, OptionalSetArgs +from posthog.args import ( + ID_TYPES, + ExceptionArg, + OptionalCaptureArgs, + OptionalSetArgs, + _PinnedProperties, +) from posthog.metrics_capture import ( _DEFAULT_FLUSH_INTERVAL_SECONDS, PostHogMetrics, @@ -353,6 +373,20 @@ def wrapper(self, *args, **kwargs): } ) +# A config version 2 event also keeps its attribution when minimized. Only those +# events use this wider allowlist, so a config version 1 event is trimmed as before, +# even when super properties happen to use these names. +_MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES: frozenset[str] = ( + _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES + | { + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_id", + "$feature_flag_rule_type", + "$feature_flag_variant", + } +) + def _parse_has_experiment(value: Any) -> Optional[bool]: """Server-reported experiment linkage; anything but an explicit bool means unknown.""" @@ -365,6 +399,341 @@ def _metadata_has_experiment(metadata: Any) -> Optional[bool]: return metadata.has_experiment if isinstance(metadata, FlagMetadata) else None +# The reason code and rule type of the one outcome that sends ``$experiment_exposure``: +# a variant assigned by an experiment rule. +_EXPERIMENT_SPLIT_REASON = "experiment_split" +_EXPERIMENT_RULE_TYPE = "experiment" + +# The reason codes of a config version 2 record that report the rule that produced +# the result, and the rule types each one allows. Any other reason code, and any +# failed record, reports no rule, even if the record names one. +_RULE_TYPES_BY_REASON_CODE: Dict[str, FrozenSet[str]] = { + "targeting_match": frozenset({"targeted_release", "percentage_rollout"}), + _EXPERIMENT_SPLIT_REASON: frozenset({_EXPERIMENT_RULE_TYPE}), + "rollout_miss": frozenset({"percentage_rollout", _EXPERIMENT_RULE_TYPE}), + "experiment_paused": frozenset({_EXPERIMENT_RULE_TYPE}), + "holdout": frozenset({_EXPERIMENT_RULE_TYPE}), +} + +# Properties a super property must not add to, or change on, a config version 2 +# ``$feature_flag_called``: the attribution, which is set again from the record +# after super properties are merged, and the fields such an event never carries. +_V2_FLAG_CALLED_PINNED = frozenset( + { + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_id", + "$feature_flag_rule_type", + "$feature_flag_variant", + "$feature_flag_reason", + "$feature_flag_forced_variant", + "$feature_flag_experiment_id", + } +) + +# Properties an ``$experiment_exposure`` never carries, even from a super property. +_EXPOSURE_FORBIDDEN = frozenset( + { + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$experiment_exposure_source", + "$feature_flag_experiment_id", + "$feature_flag_forced_variant", + } +) + +# ``$experiment_exposure`` copies these from the ``$feature_flag_called`` of the same +# read, when it has them. +_EXPOSURE_PROPERTIES_FROM_FLAG_CALLED = ( + "$feature_flag_id", + "$feature_flag_version", + "$feature_flag_request_id", + "$feature_flag_evaluated_at", +) + + +def _v4_field(record: Any, name: str) -> Any: + """Read a ``/flags?v=4`` field of a parsed flag record or its metadata, such as + ``failed`` or ``config_version``. + + These fields stay private, as ``_failed``, ``_config_version`` and so on, until + the typed flag accessors make them public under their wire names. Either spelling + is read, so the two changes can land in either order. + """ + return getattr(record, name, getattr(record, f"_{name}", None)) + + +# The rule ID's wire format: a hyphenated UUID, as the contract's event schemas +# require. +_UUID_PATTERN = re.compile( + r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" +) + + +def _is_uuid(value: Any) -> bool: + return isinstance(value, str) and _UUID_PATTERN.fullmatch(value) is not None + + +class _RuleAttribution(NamedTuple): + """What a config version 2 flag record reports on its flag events.""" + + reason_code: Optional[str] + rule_id: Optional[str] + rule_type: Optional[str] + # Set only for a complete experiment split, the outcome that sends an exposure. + variant_key: Optional[str] + + def flag_called_properties(self) -> Dict[str, Any]: + properties: Dict[str, Any] = {"$feature_flag_config_version": 2} + if self.reason_code is not None: + properties["$feature_flag_reason_code"] = self.reason_code + if self.rule_id is not None: + properties["$feature_flag_rule_id"] = self.rule_id + if self.rule_type is not None: + properties["$feature_flag_rule_type"] = self.rule_type + if self.variant_key is not None: + properties["$feature_flag_variant"] = self.variant_key + return properties + + +def _rule_attribution(details: Optional[FeatureFlag]) -> Optional[_RuleAttribution]: + """Read the event attribution of a config version 2 flag record. + + Returns ``None`` for anything else: no record, an unreadable record, a config + version 1 record, or a record from a server without typed values. Only the + evaluation details are read, never the value a getter returned. + + The config version and reason code are always reported. The rule ID and rule + type are reported together or not at all, only for a targeting match, a rollout + miss, a paused experiment, a holdout and a complete experiment split, and only + with a rule type the reason code allows (``_RULE_TYPES_BY_REASON_CODE``). A + record with no rule ID, no rule type or another rule type, such as a targeting + match that names an experiment rule, reports neither. A rule ID that is not a + UUID counts as missing. So does a failed record or any other reason code, + including one this SDK does not know, and a record with no reason code. The + variant key is reported only for a complete experiment split: reason + ``experiment_split``, rule type ``experiment``, a rule ID and a variant key (any + string, including an empty one). A split without all of them is malformed. A + record that reports no rule keeps its reason, reports no rule ID, rule type or + variant, and sends no exposure. Its identity is never guessed from the value, + the key or an earlier result. + """ + if details is None or not isinstance(details.metadata, FlagMetadata): + return None + metadata = details.metadata + if _v4_field(metadata, "config_version") != 2: + return None + reason_code = (details.reason.code if details.reason else None) or None + allowed_rule_types = _RULE_TYPES_BY_REASON_CODE.get(reason_code or "") + if _v4_field(details, "failed") is True or allowed_rule_types is None: + return _RuleAttribution(reason_code, None, None, None) + rule_id = _v4_field(metadata, "rule_id") + rule_type = _v4_field(metadata, "rule_type") + if not _is_uuid(rule_id) or rule_type not in allowed_rule_types: + return _RuleAttribution(reason_code, None, None, None) + if reason_code != _EXPERIMENT_SPLIT_REASON: + # Only an experiment split assigns a variant. + return _RuleAttribution(reason_code, rule_id, rule_type, None) + variant_key = _v4_field(metadata, "variant_key") + if not isinstance(variant_key, str): + return _RuleAttribution(reason_code, None, None, None) + return _RuleAttribution(reason_code, rule_id, rule_type, variant_key) + + +_flag_events_log = logging.getLogger("posthog") + + +def _capture_once( + capture: Callable[..., Any], + reported: Set[Any], + dedupe_key: Any, + event: str, + **kwargs: Any, +) -> None: + """Capture one flag event and keep its dedupe key unless capture failed. + + The key is reserved before the capture, so concurrent reads do not send the + same event twice. It is released when capture raises, or returns ``None`` + because the event could not be queued (a full or closed queue, opt-out or a + disabled client), so a later read tries again. An event that ``before_send`` + drops keeps its key: the application chose to drop it, and every later read + would be dropped the same way. A failure is logged and never reaches the read. + """ + reported.add(dedupe_key) + before_send_drops: List[str] = [] + try: + accepted = capture( + event, _before_send_drops=before_send_drops, **kwargs + ) is not None or bool(before_send_drops) + except Exception: + _flag_events_log.exception("[FEATURE FLAGS] Failed to capture %s", event) + accepted = False + if not accepted: + reported.discard(dedupe_key) + + +def _capture_flag_events( + capture: Callable[..., Any], + flag_calls_reported: Dict[Any, Any], + exposures_reported: Dict[Any, Any], + *, + distinct_id: ID_TYPES, + key: str, + response: Optional[FlagValue], + properties: Dict[str, Any], + groups: Optional[Mapping[str, Union[str, int]]], + capture_groups: Mapping[str, Any], + disable_geoip: Optional[bool], + has_experiment: Optional[bool], + minimal_flag_called_events: bool, + details: Optional[FeatureFlag], + value_from_details: bool, +) -> None: + """Send the flag events of one flag read: ``$feature_flag_called`` and, for an + experiment split, ``$experiment_exposure``. + + Every read that sends flag events comes here, on both clients: the single-flag + getters and the reads of an ``evaluate_flags()`` snapshot. Both events are built + from ``details``, the parsed ``/flags`` record that supplied the value, and the + call's ``properties``; the flag is never evaluated again. + + ``$feature_flag_called`` is sent if the (distinct_id, flag, response, groups) + tuple hasn't already been reported on this client. Group context is included so + that group-scoped flags fire a separate event for each group a user is evaluated + under. A config version 2 record adds its attribution to the event and to that + tuple, so equal-valued A/A arms, successive experiments of one flag, and + holdouts are each reported. Its reason code replaces the human-readable + ``$feature_flag_reason``. A config version 1 event and tuple are unchanged. + + ``has_experiment`` is the server-reported signal for whether the flag is linked + to an experiment (``None`` when the server did not report it). When the + server-controlled gate is on and the flag is known non-experiment, the event is + trimmed to a strict allowlist; any missing signal sends the full legacy shape, + and experiment-linked flags keep the full set for exposure analysis. + + ``minimal_flag_called_events`` is the gate as observed by the evaluation that + produced ``response``, not a fresh read of client-wide state: both callers + resolve it themselves (the snapshot pins it at construction; the single-flag + path reads it from the specific local/remote source that produced the value) so + a concurrent poller refresh or another ``/flags`` call can't reshape an event + after the fact. + + ``$experiment_exposure`` is sent for a complete config version 2 experiment + split whose record supplied the reported value (``value_from_details``), once + per distinct ID, groups, flag, rule and variant. It has its own dedupe, so a + ``$feature_flag_called`` already reported never suppresses it. + + Both config version 2 events are attempted independently: a failed capture of + one does not stop the other, and keeps no dedupe key, so a later read tries + again. An event that ``before_send`` drops keeps its key. Their attribution is + pinned after super properties are merged, so a super property can neither change + it nor add a field the event never carries. + Config version 1 events are sent and deduped as before. + """ + groups_key = ( + tuple(sorted((str(k), str(v)) for k, v in groups.items())) if groups else () + ) + attribution = _rule_attribution(details) + feature_flag_reported_key: Tuple[Any, ...] = (key, response, groups_key) + if attribution is not None: + feature_flag_reported_key += (2, *attribution) + + reported_flags = flag_calls_reported.get(distinct_id) + if reported_flags is None: + reported_flags = set() + flag_calls_reported[distinct_id] = reported_flags + + if feature_flag_reported_key not in reported_flags: + if attribution is not None: + properties.pop("$feature_flag_reason", None) + properties.update(attribution.flag_called_properties()) + # Record the server's experiment signal when known, so minimization's impact + # can be measured by segmenting on it. + if has_experiment is not None: + properties["$feature_flag_has_experiment"] = has_experiment + + # Minimize iff the server-controlled gate is on AND the flag is known to have + # no linked experiment. Any missing signal (gate absent, has_experiment + # missing) fails safe to the full legacy shape. + should_minimize = minimal_flag_called_events and has_experiment is False + # Only thread the internal allowlist through when minimizing, so the + # full-property path's capture() call signature stays unchanged. + extra_capture_kwargs: Dict[str, Any] = {} + if should_minimize: + extra_capture_kwargs["_property_allowlist"] = ( + _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES + if attribution is None + else _MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES + ) + + if attribution is None: + # Config version 1 events are sent and deduped as before. + capture( + "$feature_flag_called", + distinct_id=distinct_id, + properties=properties, + groups=capture_groups, + disable_geoip=disable_geoip, + **extra_capture_kwargs, + ) + reported_flags.add(feature_flag_reported_key) + else: + _capture_once( + capture, + reported_flags, + feature_flag_reported_key, + "$feature_flag_called", + distinct_id=distinct_id, + properties=properties, + groups=capture_groups, + disable_geoip=disable_geoip, + _pinned_properties=_PinnedProperties( + _V2_FLAG_CALLED_PINNED, attribution.flag_called_properties() + ), + **extra_capture_kwargs, + ) + + if attribution is None or attribution.variant_key is None or not value_from_details: + return + + # The value, request ID, evaluation time and evaluation source are left out, so + # one assignment is one exposure however often it is read. + exposure_key = (key, groups_key, 2, attribution.rule_id, attribution.variant_key) + normalized_distinct_id = str(distinct_id) + reported_exposures = exposures_reported.get(normalized_distinct_id) + if reported_exposures is None: + reported_exposures = set() + exposures_reported[normalized_distinct_id] = reported_exposures + if exposure_key in reported_exposures: + return + + exposure_properties: Dict[str, Any] = { + name: properties[name] + for name in _EXPOSURE_PROPERTIES_FROM_FLAG_CALLED + if name in properties + } + exposure_identity = { + "$feature_flag": key, + "$feature_flag_response": response, + "$feature_flag_variant": attribution.variant_key, + "$feature_flag_rule_id": attribution.rule_id, + "locally_evaluated": properties.get("locally_evaluated") is True, + } + exposure_properties.update(exposure_identity) + _capture_once( + capture, + reported_exposures, + exposure_key, + "$experiment_exposure", + distinct_id=distinct_id, + properties=exposure_properties, + groups=capture_groups, + disable_geoip=disable_geoip, + _pinned_properties=_PinnedProperties(_EXPOSURE_FORBIDDEN, exposure_identity), + ) + + class _Lane: """A capture lane: a queue drained by its own consumer pool, posting to one endpoint. @@ -941,6 +1310,9 @@ def __init__( ) self.poller: Optional[Poller] = None self.distinct_ids_feature_flags_reported = SizeLimitedDict(MAX_DICT_SIZE, set) + # $experiment_exposure dedupe, kept apart from $feature_flag_called's so that a + # reported call never suppresses its exposure. Same bound and eviction. + self._experiment_exposures_reported = SizeLimitedDict(MAX_DICT_SIZE, set) self.flag_fallback_cache_url = flag_fallback_cache_url self.flag_cache = self._initialize_flag_cache(flag_fallback_cache_url) self.flag_definition_version = 0 @@ -1837,6 +2209,8 @@ def _capture( # Internal, set for minimal $feature_flag_called events: a strict allowlist # applied to the fully-enriched properties dict just before enqueueing. property_allowlist = kwargs.get("_property_allowlist", None) + pinned_properties = kwargs.get("_pinned_properties", None) + before_send_drops = kwargs.get("_before_send_drops", None) properties = {**(properties or {}), **system_context()} @@ -1954,7 +2328,12 @@ def _capture( msg["properties"] = properties return self._enqueue( - msg, disable_geoip, lane, property_allowlist=property_allowlist + msg, + disable_geoip, + lane, + property_allowlist=property_allowlist, + pinned_properties=pinned_properties, + before_send_drops=before_send_drops, ) def _parse_send_feature_flags(self, send_feature_flags) -> SendFeatureFlagsOptions: @@ -2563,7 +2942,15 @@ def _normalize_event_uuid(self, msg): # Always send a uuid, so we can always return one msg["uuid"] = stringify_id(uuid4()) - def _enqueue(self, msg, disable_geoip, lane=None, property_allowlist=None): + def _enqueue( + self, + msg, + disable_geoip, + lane=None, + property_allowlist=None, + pinned_properties=None, + before_send_drops=None, + ): # type: (...) -> Optional[str] """Push a new `msg` onto a lane's queue (analytics when unspecified), return the event uuid or None.""" @@ -2615,6 +3002,11 @@ def _enqueue(self, msg, disable_geoip, lane=None, property_allowlist=None): # Applied after every enrichment step (system context, context tags, super # properties, $lib/$lib_version) so the final event shape is exactly the # allowlist regardless of where a property came from. + # After every enrichment step, so a super property never changes or adds + # config version 2 flag attribution. + if pinned_properties is not None: + msg["properties"] = pinned_properties.apply(msg["properties"]) + if property_allowlist is not None: msg["properties"] = { k: v for k, v in msg["properties"].items() if k in property_allowlist @@ -2629,6 +3021,8 @@ def _enqueue(self, msg, disable_geoip, lane=None, property_allowlist=None): modified_msg = self.before_send(msg) if modified_msg is None: self.log.debug("Event dropped by before_send callback") + if before_send_drops is not None: + before_send_drops.append(msg["event"]) return None if not isinstance(modified_msg, dict): raise TypeError("before_send must return a dict or None") @@ -3226,6 +3620,11 @@ def _shutdown_once(self, errors: list[Exception]) -> None: self.distinct_ids_feature_flags_reported.clear, errors, ) + self._run_lifecycle_cleanup( + "Failed to clear experiment exposure deduplication state on shutdown", + self._experiment_exposures_reported.clear, + errors, + ) if self.exception_capture: self._run_lifecycle_cleanup( @@ -4094,6 +4493,9 @@ def _get_feature_flag_result( flag_result = None flag_details = None + # Whether flag_details supplied the value this read reports. Only then may the + # read send $experiment_exposure. + value_from_details = False request_id = None evaluated_at = None feature_flag_error: Optional[str] = None @@ -4207,6 +4609,11 @@ def _get_feature_flag_result( self.log.debug( f"Successfully computed flag remotely: #{key} -> #{flag_result}" ) + # A payload lookup for a caller's match_value reports that value, not + # the record's. A failure above falls back to a stale cached value. + value_from_details = ( + flag_details is not None and override_match_value is None + ) except QuotaLimitError as e: self.log.warning(f"[FEATURE FLAGS] Quota limit exceeded: {e}") feature_flag_error = FeatureFlagError.QUOTA_LIMITED @@ -4263,6 +4670,7 @@ def _get_feature_flag_result( request_id, evaluated_at, flag_details, + value_from_details, feature_flag_error, has_experiment, minimal_flag_called_events, @@ -4289,6 +4697,9 @@ def get_feature_flag_result( Get a FeatureFlagResult object which contains the flag result and payload for a key by evaluating locally or remotely depending on whether local evaluation is enabled and the flag can be locally evaluated. This also captures the `$feature_flag_called` event unless `send_feature_flag_events` is `False`. + When the flag was evaluated remotely and its config version 2 record is a complete experiment + split (an experiment rule with a rule ID and a variant key), it also captures the + `$experiment_exposure` event, once per distinct ID, groups, flag, rule and variant. Examples: ```python @@ -4554,6 +4965,7 @@ def _capture_feature_flag_called( request_id: Optional[str], evaluated_at: Optional[int], flag_details: Optional[FeatureFlag], + value_from_details: bool, feature_flag_error: Optional[str] = None, has_experiment: Optional[bool] = None, minimal_flag_called_events: bool = False, @@ -4599,6 +5011,8 @@ def _capture_feature_flag_called( disable_geoip=disable_geoip, has_experiment=has_experiment, minimal_flag_called_events=minimal_flag_called_events, + details=flag_details, + value_from_details=value_from_details, ) def _capture_feature_flag_called_if_needed( @@ -4612,66 +5026,34 @@ def _capture_feature_flag_called_if_needed( disable_geoip: Optional[bool] = None, has_experiment: Optional[bool] = None, minimal_flag_called_events: bool = False, + details: Optional[FeatureFlag] = None, + value_from_details: bool = False, ) -> None: - """Fire a ``$feature_flag_called`` event if the (distinct_id, flag, response, - groups) tuple hasn't already been reported on this client. Group context is - included so that group-scoped flags fire a separate event for each group a user - is evaluated under. Shared by the single-flag evaluation path and - ``FeatureFlagEvaluations.is_enabled() / get_flag()`` so both paths dedupe - identically. - - ``has_experiment`` is the server-reported signal for whether the flag is linked - to an experiment (``None`` when the server did not report it). When the - server-controlled gate is on and the flag is known non-experiment, the event is - trimmed to a strict allowlist; any missing signal sends the full legacy shape, - and experiment-linked flags keep the full set for exposure analysis. - - ``minimal_flag_called_events`` is the gate as observed by the evaluation that - produced ``response``, not a fresh read of client-wide state: both callers - resolve it themselves (the snapshot pins it at construction; the single-flag - path reads it from the specific local/remote source that produced the value) - so a concurrent poller refresh or another ``/flags`` call can't reshape an - event after the fact. + """Send the flag events of a flag read: ``$feature_flag_called`` and, for an + experiment split, ``$experiment_exposure``. See ``_capture_flag_events``. + + Shared by the single-flag evaluation path and the reads of a + ``FeatureFlagEvaluations`` snapshot, so every read reports and dedupes + identically. ``details`` is the parsed ``/flags`` record of the read, or + ``None`` for a locally evaluated flag. ``value_from_details`` says whether that + record supplied the reported value; only then can the read send an exposure. """ - groups_key = ( - tuple(sorted((str(k), str(v)) for k, v in groups.items())) if groups else () - ) - feature_flag_reported_key = (key, response, groups_key) - - reported_flags = self.distinct_ids_feature_flags_reported.get(distinct_id) - if reported_flags is None: - reported_flags = set() - self.distinct_ids_feature_flags_reported[distinct_id] = reported_flags - - if feature_flag_reported_key in reported_flags: - return - - # Record the server's experiment signal when known, so minimization's impact - # can be measured by segmenting on it. - if has_experiment is not None: - properties["$feature_flag_has_experiment"] = has_experiment - - # Minimize iff the server-controlled gate is on AND the flag is known to have - # no linked experiment. Any missing signal (gate absent, has_experiment - # missing) fails safe to the full legacy shape. - should_minimize = minimal_flag_called_events and has_experiment is False - # Only thread the internal allowlist through when minimizing, so the - # full-property path's capture() call signature stays unchanged. - extra_capture_kwargs: dict[str, Any] = {} - if should_minimize: - extra_capture_kwargs["_property_allowlist"] = ( - _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES - ) - - self.capture( - "$feature_flag_called", + _capture_flag_events( + self.capture, + self.distinct_ids_feature_flags_reported, + self._experiment_exposures_reported, distinct_id=distinct_id, + key=key, + response=response, properties=properties, - groups=groups or {}, + groups=groups, + capture_groups=groups or {}, disable_geoip=disable_geoip, - **extra_capture_kwargs, + has_experiment=has_experiment, + minimal_flag_called_events=minimal_flag_called_events, + details=details, + value_from_details=value_from_details, ) - reported_flags.add(feature_flag_reported_key) def get_remote_config_payload(self, key: str): """ diff --git a/posthog/feature_flag_evaluations.py b/posthog/feature_flag_evaluations.py index 28e5bf31e..ee179d4bc 100644 --- a/posthog/feature_flag_evaluations.py +++ b/posthog/feature_flag_evaluations.py @@ -125,6 +125,13 @@ class FeatureFlagEvaluations: To narrow the set of flags that get attached to a captured event, use the in-memory helpers :meth:`only` and :meth:`only_accessed`. To narrow the set of flags requested from the server in the first place, pass ``flag_keys`` to :meth:`Client.evaluate_flags`. + + Reading a flag sends ``$feature_flag_called``, deduplicated as :meth:`is_enabled` + describes. When the flag was evaluated remotely and its config version 2 record is + a complete experiment split (an experiment rule with a rule ID and a variant key), + the read also sends ``$experiment_exposure``. The SDK sends one exposure per + distinct ID, groups, flag, rule and variant. ``send_feature_flag_events=False`` + suppresses both events. """ def __init__( @@ -387,4 +394,8 @@ def _record_access(self, key: str, send_feature_flag_events: bool) -> None: properties=properties, has_experiment=flag.has_experiment if flag else None, minimal_flag_called_events=self._minimal_flag_called_events, + # The record that supplied the value. A remotely evaluated experiment + # split also sends $experiment_exposure, whatever this read returned. + details=flag.details if isinstance(flag, _RemoteFlagRecord) else None, + value_from_details=True, ) diff --git a/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json b/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json index df7f1c7e0..d9b4518a7 100644 --- a/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json +++ b/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json @@ -1,14 +1,14 @@ { "description": "Copied unchanged from the released feature flag rules v2 contract in PostHog/posthog-sdk-test-harness. Do not edit these files; copy a newer release instead.", "repository": "https://github.com/PostHog/posthog-sdk-test-harness", - "release": "1.22.0", - "revision": "17d861f8e230fe943934d70e3eebf5cff6bf3aa4", + "release": "1.28.0", + "revision": "ffa6cc38dbc4efa64b474d2d81a6c6abad4cd07a", "contract_directory": "contracts/feature_flag_rules_v2", - "contract_version": "7.0.0", + "contract_version": "9.0.0", "files": { "readers.json": { "source_path": "fixtures/wire/readers.json", - "sha256": "fd8edd89f827b732c4659ce7246eab534b0106b96a465a4a2d98cfa0a17d7827" + "sha256": "e75abb9bf6656989a147dda4c6e5232e7b544b657d7d540164dce65688641501" }, "responses.json": { "source_path": "fixtures/wire/responses.json", @@ -21,6 +21,14 @@ "v2_value_evaluation.json": { "source_path": "corpus/v2_value_evaluation.json", "sha256": "aaddd05e47fc5a4ae07ea42ada5a659798f4b5fc8d0c0827f6a68e4daf68c128" + }, + "feature_flag_called_context.schema.json": { + "source_path": "schemas/feature_flag_called_context.schema.json", + "sha256": "8d151f067bfcd2c69a9ce0193f0666a4f9af58276a7ff724787518e04480fcc1" + }, + "experiment_exposure_properties.schema.json": { + "source_path": "schemas/experiment_exposure_properties.schema.json", + "sha256": "6d33707de086537abb90bb6b649da49bd4aca359fcf2c7877eceffb0a86dee63" } } } diff --git a/posthog/test/fixtures/feature_flag_rules_v2/experiment_exposure_properties.schema.json b/posthog/test/fixtures/feature_flag_rules_v2/experiment_exposure_properties.schema.json new file mode 100644 index 000000000..9f50ebe79 --- /dev/null +++ b/posthog/test/fixtures/feature_flag_rules_v2/experiment_exposure_properties.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://posthog.com/contracts/feature_flag_rules_v2/experiment_exposure_properties.schema.json", + "title": "SDK-sent $experiment_exposure properties", + "type": "object", + "additionalProperties": false, + "required": [ + "$feature_flag", + "$feature_flag_response", + "$feature_flag_variant", + "$feature_flag_rule_id", + "locally_evaluated" + ], + "properties": { + "$feature_flag": { + "type": "string", + "minLength": 1 + }, + "$feature_flag_response": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "$feature_flag_variant": { + "type": "string" + }, + "$feature_flag_rule_id": { + "type": "string", + "format": "uuid" + }, + "locally_evaluated": { + "type": "boolean" + }, + "$feature_flag_forced_variant": { + "const": true + } + }, + "description": "Closed projection of the properties of an SDK-sent exposure. The SDK sends it only for a config version 2 experiment split, so it carries no config version, rule type, reason code or source marker; the rule ID marks it as SDK-sent. Experiment ID and forced-variant context are forbidden. Capture identity and existing optional transport properties are outside this projection. Scan the entire event recursively for assignment seeds.", + "not": { + "required": [ + "$feature_flag_forced_variant" + ] + } +} diff --git a/posthog/test/fixtures/feature_flag_rules_v2/feature_flag_called_context.schema.json b/posthog/test/fixtures/feature_flag_rules_v2/feature_flag_called_context.schema.json new file mode 100644 index 000000000..7bad83d87 --- /dev/null +++ b/posthog/test/fixtures/feature_flag_rules_v2/feature_flag_called_context.schema.json @@ -0,0 +1,692 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://posthog.com/contracts/feature_flag_rules_v2/feature_flag_called_context.schema.json", + "title": "$feature_flag_called context properties", + "type": "object", + "additionalProperties": false, + "required": [ + "$feature_flag", + "$feature_flag_response" + ], + "properties": { + "$feature_flag": { + "type": "string", + "minLength": 1 + }, + "$feature_flag_response": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "$feature_flag_config_version": { + "enum": [ + 1, + 2 + ] + }, + "$feature_flag_variant": { + "type": "string" + }, + "$feature_flag_rule_id": { + "type": "string", + "format": "uuid" + }, + "$feature_flag_rule_type": { + "enum": [ + "targeted_release", + "percentage_rollout", + "experiment" + ] + }, + "$feature_flag_experiment_id": { + "type": "integer" + }, + "$feature_flag_reason_code": { + "type": "string", + "minLength": 1 + }, + "$feature_flag_reason": {}, + "$feature_flag_forced_variant": { + "const": true + } + }, + "description": "Closed projection of the flag-access context properties. Other existing capture fields are outside this projection. Complete v2 producer context follows the terminal outcome. A tolerant reader drops the whole attribution of a record that breaks its reason's rows (a rule ID without its rule type or the reverse, a rule type the reason does not allow, or a malformed split): for every reason that names a rule, a call may keep the config version and reason code and omit all rule, variant and experiment attribution, while a partly populated or mismatched tuple is rejected. A config version 2 call whose reason code is missing or not registered keeps the raw code, if any, and carries no attribution; a registered code only matches its own reason's branches. A config version 2 call never carries $feature_flag_reason, the reason description that only version 1 calls carry.", + "allOf": [ + { + "if": { + "required": [ + "$feature_flag_config_version" + ], + "properties": { + "$feature_flag_config_version": { + "const": 2 + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_forced_variant" + ] + }, + { + "required": [ + "$feature_flag_reason" + ] + } + ] + }, + "oneOf": [ + { + "properties": { + "$feature_flag_reason_code": { + "const": "targeting_match" + }, + "$feature_flag_rule_type": { + "const": "targeted_release" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "targeting_match" + }, + "$feature_flag_rule_type": { + "const": "percentage_rollout" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "experiment_split" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id", + "$feature_flag_variant" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "rollout_miss" + }, + "$feature_flag_rule_type": { + "const": "percentage_rollout" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "rollout_miss" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "experiment_paused" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "holdout" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "no_rule_match" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "missing_group_key" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "dependency_error" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "error" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "targeting_match" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "experiment_split" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "rollout_miss" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "experiment_paused" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "holdout" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "not": { + "enum": [ + "targeting_match", + "experiment_split", + "rollout_miss", + "experiment_paused", + "holdout", + "no_rule_match", + "missing_group_key", + "dependency_error", + "error", + "flag_disabled" + ] + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + }, + { + "description": "A local-only outcome with no matrix row: the server never returns flag_disabled, and the call carries no rule or experiment attribution.", + "properties": { + "$feature_flag_reason_code": { + "const": "flag_disabled" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant" + ] + } + ] + } + } + ] + }, + "else": { + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_rule_type" + ] + } + ] + } + } + } + ] +} diff --git a/posthog/test/fixtures/feature_flag_rules_v2/readers.json b/posthog/test/fixtures/feature_flag_rules_v2/readers.json index 7ed8c252b..add97e172 100644 --- a/posthog/test/fixtures/feature_flag_rules_v2/readers.json +++ b/posthog/test/fixtures/feature_flag_rules_v2/readers.json @@ -992,7 +992,10 @@ "value": "compact", "config_version": 1, "experiment_exposure": false, - "error_code": null + "error_code": null, + "legacy_payload": { + "columns": 2 + } }, "producer_expected": "valid" }, @@ -1018,7 +1021,8 @@ "value": true, "config_version": 1, "experiment_exposure": false, - "error_code": "GENERAL" + "error_code": "GENERAL", + "legacy_payload": null }, "producer_expected": "valid" }, diff --git a/posthog/test/released_flags_scenarios.py b/posthog/test/released_flags_scenarios.py index b3cacdac1..dfbf629bc 100644 --- a/posthog/test/released_flags_scenarios.py +++ b/posthog/test/released_flags_scenarios.py @@ -31,6 +31,7 @@ ) API_KEY = "phc_released_flags_scenarios" DISTINCT_ID = "user-1" +ACCEPTED_UUID = "70000000-0000-4000-8000-000000000000" REQUEST_ID = "60000000-0000-4000-8000-000000000000" EVALUATED_AT = 1800000000000 MATCHED = { @@ -170,7 +171,8 @@ def flags(*args: Any, **kwargs: Any) -> Any: def enqueue(message: Dict[str, Any], *args: Any, **kwargs: Any) -> Optional[str]: events.append([message["event"], _flag_event_properties(message)]) - return None + # Accept the event, as the real queue does, by returning an event UUID. + return ACCEPTED_UUID try: with ( diff --git a/posthog/test/test_experiment_exposure.py b/posthog/test/test_experiment_exposure.py new file mode 100644 index 000000000..5ad3db960 --- /dev/null +++ b/posthog/test/test_experiment_exposure.py @@ -0,0 +1,1638 @@ +"""``$experiment_exposure`` and the attribution of ``$feature_flag_called``. + +A remotely evaluated config version 2 record that assigns an experiment variant sends +``$experiment_exposure`` next to its ``$feature_flag_called``, and every config version +2 record adds its rule attribution to ``$feature_flag_called``. The reader fixtures and +event schemas are copied unchanged from a released ``PostHog/posthog-sdk-test-harness`` +contract; ``fixtures/feature_flag_rules_v2/SOURCE.json`` records which. +""" + +from __future__ import annotations + +import copy +import json +import warnings +from dataclasses import dataclass +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional +from unittest import mock + +import jsonschema +import pytest + +from posthog import AsyncPosthog +from posthog.client import ( + _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES, + _MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES, + Client, +) +from posthog.test.test_flags_v4_response import FIXTURES, READERS, _build +from posthog.test.test_utils import FAKE_TEST_API_KEY +from posthog.types import FeatureFlagResult + +FLAG = "pricing-page" +DISTINCT_ID = "visitor-42" +RULE = "7f3c2a10-8d4e-4b6a-9c21-5e0f1a2b3c4d" +OTHER_RULE = "2b9d4e61-0c7a-4f35-8e12-6a4b3c2d1e0f" +REQUEST_ID = "5d1e2f30-4a5b-4c6d-8e7f-90a1b2c3d4e5" +OTHER_REQUEST_ID = "6e2f3a41-5b6c-4d7e-9f80-a1b2c3d4e5f6" +EVALUATED_AT = 1_800_000_000_000 + +EXPOSURE = "$experiment_exposure" +FLAG_CALLED = "$feature_flag_called" +ATTRIBUTION_PROPERTIES = ( + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_id", + "$feature_flag_rule_type", + "$feature_flag_variant", +) + + +# $experiment_exposure leaves these out. The config version, rule type and reason code +# stay on the $feature_flag_called of the same read. The schema projection drops +# undeclared properties, so their absence is asserted separately. +EXPOSURE_LEAVES_OUT = ( + "$feature_flag_config_version", + "$feature_flag_rule_type", + "$feature_flag_reason_code", + "$experiment_exposure_source", +) + + +def _schema(name: str) -> jsonschema.Draft202012Validator: + schema = json.loads((FIXTURES / name).read_text(encoding="utf-8")) + return jsonschema.Draft202012Validator(schema) + + +FLAG_CALLED_SCHEMA = _schema("feature_flag_called_context.schema.json") +EXPOSURE_SCHEMA = _schema("experiment_exposure_properties.schema.json") + + +def _projection( + validator: jsonschema.Draft202012Validator, properties: Mapping[str, Any] +) -> Dict[str, Any]: + """The properties a closed contract projection covers.""" + names = validator.schema["properties"] + return {key: value for key, value in properties.items() if key in names} + + +# --- Responses ---------------------------------------------------------------- + + +def _record( + code: str, + value: Any = True, + *, + key: str = FLAG, + rule_type: Optional[str] = None, + rule_id: Optional[str] = None, + variant_key: Optional[str] = None, + config_version: int = 2, + has_experiment: bool = False, + description: Optional[str] = None, + failed: bool = False, +) -> Dict[str, Any]: + metadata: Dict[str, Any] = { + "id": 77, + "version": 3, + "config_version": config_version, + "description": None, + "payload": None, + "has_experiment": has_experiment, + } + for field, field_value in ( + ("rule_type", rule_type), + ("rule_id", rule_id), + ("variant_key", variant_key), + ): + if field_value is not None: + metadata[field] = field_value + reason: Dict[str, Any] = { + "code": code, + "condition_index": 0 if rule_id else None, + } + if description is not None: + reason["description"] = description + record = {"key": key, "value": value, "reason": reason, "metadata": metadata} + if failed: + record["failed"] = True + return record + + +def _split( + value: Any = "annual", + variant_key: str = "annual-first", + rule_id: str = RULE, + **overrides: Any, +) -> Dict[str, Any]: + fields: Dict[str, Any] = { + "rule_type": "experiment", + "rule_id": rule_id, + "variant_key": variant_key, + "has_experiment": True, + **overrides, + } + return _record("experiment_split", value, **fields) + + +def _response( + *records: Dict[str, Any], + request_id: str = REQUEST_ID, + evaluated_at: int = EVALUATED_AT, + **envelope: Any, +) -> Dict[str, Any]: + return { + "flags": {record["key"]: record for record in records}, + "errorsWhileComputingFlags": False, + "requestId": request_id, + "evaluatedAt": evaluated_at, + **envelope, + } + + +# --- Recording events ----------------------------------------------------------- + + +@dataclass +class _Event: + name: str + distinct_id: Any + properties: Dict[str, Any] + groups: Any + property_allowlist: Any + uuid: Optional[str] + + +def _record_events(client: Any) -> tuple[Any, List[_Event]]: + """Patch ``client.capture`` to record each call, then capture as usual.""" + events: List[_Event] = [] + capture = client.capture + + def record(event: str, **kwargs: Any) -> Optional[str]: + uuid = capture(event, **kwargs) + events.append( + _Event( + name=event, + distinct_id=kwargs.get("distinct_id"), + properties=copy.deepcopy(kwargs.get("properties") or {}), + groups=kwargs.get("groups"), + property_allowlist=kwargs.get("_property_allowlist"), + uuid=uuid, + ) + ) + return uuid + + return mock.patch.object(client, "capture", side_effect=record), events + + +def _named(events: List[_Event], name: str, key: str = FLAG) -> List[_Event]: + return [ + event + for event in events + if event.name == name and event.properties.get("$feature_flag") == key + ] + + +def _attribution(properties: Mapping[str, Any]) -> Dict[str, Any]: + return { + name: properties[name] for name in ATTRIBUTION_PROPERTIES if name in properties + } + + +# --- Reading through each path ------------------------------------------------------ + +# Every sync read that sends flag events. Each takes the client, the key, the +# distinct ID and the groups. +SYNC_READS: Dict[str, Callable[..., Any]] = { + "get_feature_flag": lambda client, key, distinct_id, groups: ( + client.get_feature_flag(key, distinct_id, groups=groups) + ), + "feature_enabled": lambda client, key, distinct_id, groups: ( + client.feature_enabled(key, distinct_id, groups=groups) + ), + "get_feature_flag_result": lambda client, key, distinct_id, groups: ( + client.get_feature_flag_result(key, distinct_id, groups=groups) + ), + "snapshot.get_flag": lambda client, key, distinct_id, groups: ( + client.evaluate_flags(distinct_id, groups=groups).get_flag(key) + ), + "snapshot.is_enabled": lambda client, key, distinct_id, groups: ( + client.evaluate_flags(distinct_id, groups=groups).is_enabled(key) + ), +} +SINGLE_FLAG_READS = ["get_feature_flag", "feature_enabled", "get_feature_flag_result"] +SNAPSHOT_READS = ["snapshot.get_flag", "snapshot.is_enabled"] + + +@pytest.fixture +def client() -> Iterator[Client]: + client = Client(FAKE_TEST_API_KEY, send=False) + yield client + client.shutdown() + + +def _serving(*responses: Dict[str, Any]) -> Any: + """Patch ``/flags`` to answer with each response in turn, the last one after that.""" + remaining = list(responses) + + def answer(*args: Any, **kwargs: Any) -> Dict[str, Any]: + response = remaining.pop(0) if len(remaining) > 1 else remaining[0] + return copy.deepcopy(response) + + return mock.patch("posthog.client.flags", side_effect=answer) + + +def _read( + client: Client, + path: str, + key: str = FLAG, + distinct_id: Any = DISTINCT_ID, + groups: Optional[Dict[str, Any]] = None, +) -> Any: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return SYNC_READS[path](client, key, distinct_id, groups or {}) + + +def _read_events( + path: str, + *responses: Dict[str, Any], + reads: int = 1, + client: Optional[Client] = None, + **read_kwargs: Any, +) -> List[_Event]: + """Read ``FLAG`` ``reads`` times through ``path``, one response per read.""" + own_client = client is None + client = client or Client(FAKE_TEST_API_KEY, send=False) + recording, events = _record_events(client) + try: + with _serving(*responses), recording: + for _ in range(reads): + _read(client, path, **read_kwargs) + finally: + if own_client: + client.shutdown() + return events + + +async def _read_async_events( + responses: List[Dict[str, Any]], + reads: List[Callable[[Any], Any]], +) -> List[_Event]: + """Run the reads on one async client, answering each evaluation with the next + response, and the last one after that.""" + remaining = list(responses) + + def answer(*args: Any, **kwargs: Any) -> Dict[str, Any]: + response = remaining.pop(0) if len(remaining) > 1 else remaining[0] + return copy.deepcopy(response) + + with mock.patch( + "posthog.async_client._async_flags", new=mock.AsyncMock(side_effect=answer) + ): + client = AsyncPosthog("project-key", send=False) + recording, events = _record_events(client) + with recording: + for read in reads: + await read(client) + await client.shutdown() + return events + + +# --- The contract's reader cases -------------------------------------------------------- + +READER_CASES = [case for case in READERS["cases"] if case["expected"] == "reader"] + + +def _check_reader_events(case: Dict[str, Any], events: List[_Event]) -> None: + reader = case["reader"] + expectations = {reader["requested_key"]: reader} + expectations.update(case.get("sibling_expectations", {})) + for key, expected in expectations.items(): + exposures = _named(events, EXPOSURE, key) + assert len(exposures) == (1 if expected["experiment_exposure"] else 0), key + [called] = _named(events, FLAG_CALLED, key) + config_version = expected.get("config_version") + if expected.get("error_code") == "PARSE_ERROR": + config_version = None + assert called.properties.get("$feature_flag_config_version") == ( + 2 if config_version == 2 else None + ), key + if called.properties["$feature_flag_response"] is not None: + FLAG_CALLED_SCHEMA.validate( + _projection(FLAG_CALLED_SCHEMA, called.properties) + ) + for exposure in exposures: + EXPOSURE_SCHEMA.validate(_projection(EXPOSURE_SCHEMA, exposure.properties)) + assert not set(EXPOSURE_LEAVES_OUT) & set(exposure.properties), key + assert ( + exposure.properties["$feature_flag_response"] + == (called.properties["$feature_flag_response"]) + ) + assert exposure.uuid is not None and exposure.uuid != called.uuid + for event in [called, *exposures]: + # A flag may be named anything, so `$feature/` is left out. + for name in event.properties: + if not name.startswith("$feature/"): + for part in ("experiment_id", "holdout", "seed"): + assert part not in name, name + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +@pytest.mark.parametrize("case", READER_CASES, ids=lambda case: case["id"]) +def test_reader_cases_send_an_exposure_exactly_when_the_contract_expects(case, path): + response = _build(READERS, case) + client = Client(FAKE_TEST_API_KEY, send=False) + recording, events = _record_events(client) + try: + with _serving(response), recording: + _read(client, path, case["reader"]["requested_key"]) + for sibling in case.get("sibling_expectations", {}): + _read(client, path, sibling) + finally: + client.shutdown() + + _check_reader_events(case, events) + + +@pytest.mark.parametrize("method", ["get_flag", "is_enabled"]) +@pytest.mark.parametrize("case", READER_CASES, ids=lambda case: case["id"]) +async def test_async_reader_cases_send_an_exposure_exactly_when_the_contract_expects( + case, method +): + keys = [case["reader"]["requested_key"], *case.get("sibling_expectations", {})] + + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags(DISTINCT_ID) + for key in keys: + getattr(snapshot, method)(key) + + events = await _read_async_events([_build(READERS, case)], [read]) + + _check_reader_events(case, events) + + +# --- Attribution on $feature_flag_called -------------------------------------------------- + +_RULE_ID = "10000000-0000-4000-8000-000000000001" +ATTRIBUTION_CASES = [ + # (reader template or case id, attribution on $feature_flag_called) + ( + "experiment_split", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_split", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + "$feature_flag_variant": "arm_a", + }, + ), + ( + "targeting_match_targeted_release", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "targeting_match", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "targeted_release", + }, + ), + ( + "targeting_match_percentage_rollout", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "targeting_match", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "percentage_rollout", + }, + ), + ( + "rollout_miss_percentage_rollout", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "rollout_miss", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "percentage_rollout", + }, + ), + ( + "rollout_miss_experiment", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "rollout_miss", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + ( + "experiment_paused", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_paused", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + ( + "holdout", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "holdout", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + # A holdout assigns no variant, even when its record names one. + ( + "readers.holdout_variant_key", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "holdout", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + ( + "no_rule_match", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "no_rule_match", + }, + ), + ( + "missing_group_key", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "missing_group_key", + }, + ), + ( + "dependency_error", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "dependency_error", + }, + ), + ( + "error", + {"$feature_flag_config_version": 2, "$feature_flag_reason_code": "error"}, + ), + # A malformed split keeps its config version and reason, and loses the whole rule + # tuple. + *( + ( + case_id, + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_split", + }, + ) + for case_id in ( + "readers.partial_split_missing_rule_type", + "readers.partial_split_missing_rule_id", + "readers.partial_split_missing_variant_key", + "readers.wrong_rule_type_literal", + "readers.partial_split_string_value", + ) + ), + # Config version 1, from either kind of server, is unchanged. + ("v1", {}), + ("old_server", {}), + ("readers.partial_split_missing_config_version", {}), +] + + +def _attribution_response(source: str) -> Dict[str, Any]: + if source.startswith("readers."): + [case] = [case for case in READERS["cases"] if case["id"] == source] + return _build(READERS, case) + return _build(READERS, {"template": source}) + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize( + "source, expected", ATTRIBUTION_CASES, ids=[c[0] for c in ATTRIBUTION_CASES] +) +def test_flag_called_attribution_follows_the_record(source, expected, path): + events = _read_events(path, _attribution_response(source), key="garden-layout") + + [called] = _named(events, FLAG_CALLED, "garden-layout") + assert _attribution(called.properties) == expected + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + exposures = _named(events, EXPOSURE, "garden-layout") + assert len(exposures) == (1 if "$feature_flag_variant" in expected else 0) + + +# A record whose reason names no rule, or that failed, reports no rule even when it +# carries one. Neither sends an exposure. The last element says whether the record is +# a valid server outcome, which the contract's event schema then accepts. +STRAY_RULE_CASES = [ + ("no_rule_match", {}, True), + ("missing_group_key", {}, True), + ("flag_disabled", {}, True), + ("dependency_error", {"failed": True, "value": None}, True), + ("error", {"failed": True, "value": None}, True), + # A failed record reports no rule, whatever its reason code. + ( + "experiment_split", + {"failed": True, "value": None, "variant_key": "annual-first"}, + True, + ), + ("holdout", {"failed": True, "value": None}, False), + # A reason code this SDK does not know. + ("future_reason", {}, False), +] + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize( + "code, fields, valid_outcome", + STRAY_RULE_CASES, + ids=[f"{c}-{sorted(f)}" for c, f, _ in STRAY_RULE_CASES], +) +def test_rule_fields_are_sent_only_for_reasons_that_name_a_rule( + code, fields, valid_outcome, path +): + fields = {"value": False, **fields} + record = _record( + code, rule_type="experiment", rule_id=RULE, has_experiment=True, **fields + ) + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": code, + } + assert _named(events, EXPOSURE) == [] + if valid_outcome: + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize( + "code, rule_type", + [ + ("targeting_match", "targeted_release"), + ("targeting_match", "percentage_rollout"), + ("rollout_miss", "percentage_rollout"), + ("rollout_miss", "experiment"), + ("experiment_paused", "experiment"), + ("holdout", "experiment"), + ], +) +def test_rule_matches_misses_pauses_and_holdouts_name_their_rule(code, rule_type, path): + record = _record(code, False, rule_type=rule_type, rule_id=RULE) + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": code, + "$feature_flag_rule_id": RULE, + "$feature_flag_rule_type": rule_type, + } + assert _named(events, EXPOSURE) == [] + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +# The rule ID and rule type are sent together, with a rule type the reason allows, or +# not at all. A call that drops a broken pair keeps its config version and reason code. +BROKEN_RULE_PAIR_CASES = [ + # Half pairs. + ("targeting_match", {"rule_id": RULE}), + ("rollout_miss", {"rule_id": RULE}), + ("experiment_paused", {"rule_id": RULE}), + ("holdout", {"rule_id": RULE}), + ("targeting_match", {"rule_type": "targeted_release"}), + ("rollout_miss", {"rule_type": "percentage_rollout"}), + ("experiment_paused", {"rule_type": "experiment"}), + ("holdout", {"rule_type": "experiment"}), + # Rule types the reason code does not allow. + ("targeting_match", {"rule_id": RULE, "rule_type": "experiment"}), + ("rollout_miss", {"rule_id": RULE, "rule_type": "targeted_release"}), + ("experiment_paused", {"rule_id": RULE, "rule_type": "percentage_rollout"}), + ("holdout", {"rule_id": RULE, "rule_type": "targeted_release"}), + # A rule type this SDK does not know. + ("targeting_match", {"rule_id": RULE, "rule_type": "future_rule_type"}), +] + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize( + "code, fields", + BROKEN_RULE_PAIR_CASES, + ids=[f"{c}-{sorted(f.items())}" for c, f in BROKEN_RULE_PAIR_CASES], +) +def test_a_half_or_disallowed_rule_pair_is_not_sent(code, fields, path): + record = _record(code, False, **fields) + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": code, + } + assert _named(events, EXPOSURE) == [] + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_targeting_match_that_names_an_experiment_rule_sends_no_exposure(path): + # Even with a variant key and a linked experiment, a targeting match assigns no + # arm, so it reports no rule and sends no exposure. + record = _record( + "targeting_match", + "annual", + rule_type="experiment", + rule_id=RULE, + variant_key="annual-first", + has_experiment=True, + ) + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "targeting_match", + } + assert _named(events, EXPOSURE) == [] + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +def test_an_older_server_record_reports_no_attribution(): + # A record without a typed value comes from a server before /flags?v=4. Its + # metadata is not read as config version 2, whatever it says. + response = _response( + { + "key": FLAG, + "enabled": True, + "variant": "annual", + "reason": {"code": "condition_match", "condition_index": 0}, + "metadata": { + "id": 77, + "version": 3, + "config_version": 2, + "rule_type": "experiment", + "rule_id": RULE, + "variant_key": "annual-first", + }, + } + ) + for path in SYNC_READS: + events = _read_events(path, response) + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == {} + assert _named(events, EXPOSURE) == [] + + +# --- $feature_flag_reason ------------------------------------------------------------------ + +REASON_DESCRIPTION = "Matched rule 1" + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +@pytest.mark.parametrize( + "record", + [ + _record( + "targeting_match", + rule_type="targeted_release", + rule_id=RULE, + description=REASON_DESCRIPTION, + ), + _record("no_rule_match", False, description=REASON_DESCRIPTION), + _record("error", None, failed=True, description=REASON_DESCRIPTION), + _split(description=REASON_DESCRIPTION), + ], + ids=["targeting_match", "no_rule_match", "failed", "experiment_split"], +) +def test_a_config_version_2_call_reports_its_reason_by_code_only(record, path): + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert called.properties["$feature_flag_reason_code"] == record["reason"]["code"] + assert "$feature_flag_reason" not in called.properties + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +@pytest.mark.parametrize( + "record", + [ + _record( + "condition_match", + "annual", + config_version=1, + description=REASON_DESCRIPTION, + ), + # A record from a server before /flags?v=4, which has no config version. + { + "key": FLAG, + "enabled": True, + "variant": "annual", + "reason": { + "code": "condition_match", + "condition_index": 0, + "description": REASON_DESCRIPTION, + }, + "metadata": {"id": 77, "version": 3}, + }, + ], + ids=["config_version_1", "older_server"], +) +def test_a_call_without_config_version_2_keeps_its_reason_description(record, path): + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert called.properties["$feature_flag_reason"] == REASON_DESCRIPTION + assert _attribution(called.properties) == {} + + +@pytest.mark.parametrize( + "config_version, code, reason", + [(1, "condition_match", REASON_DESCRIPTION), (2, "targeting_match", None)], +) +def test_minimal_calls_keep_the_reason_description_only_without_config_version_2( + config_version, code, reason +): + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(message) + return message + + client = Client(FAKE_TEST_API_KEY, send=False, before_send=before_send) + record = _record( + code, config_version=config_version, description=REASON_DESCRIPTION + ) + try: + with _serving(_response(record, minimalFlagCalledEvents=True)): + client.evaluate_flags(DISTINCT_ID).get_flag(FLAG) + client.get_feature_flag_result(FLAG, "visitor-43") + finally: + client.shutdown() + + allowlist = ( + _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES + if config_version == 1 + else _MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES + ) + assert [message["event"] for message in messages] == [FLAG_CALLED, FLAG_CALLED] + for message in messages: + assert set(message["properties"]) <= allowlist + assert message["properties"].get("$feature_flag_reason") == reason + + +def test_minimal_config_version_1_calls_drop_super_properties_named_like_attribution(): + attribution_names = ( + _MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES - _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES + ) + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(message) + return message + + client = Client( + FAKE_TEST_API_KEY, + send=False, + before_send=before_send, + super_properties={name: "set-by-the-app" for name in attribution_names}, + ) + record = _record( + "condition_match", config_version=1, description=REASON_DESCRIPTION + ) + try: + with _serving(_response(record, minimalFlagCalledEvents=True)): + client.get_feature_flag_result(FLAG, DISTINCT_ID) + finally: + client.shutdown() + + [message] = messages + assert not attribution_names & set(message["properties"]) + + +# --- The exact events of a split --------------------------------------------------------- + + +# The attribution of a split's $feature_flag_called. +EXPECTED_ATTRIBUTION = { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_split", + "$feature_flag_rule_id": RULE, + "$feature_flag_rule_type": "experiment", + "$feature_flag_variant": "annual-first", +} +EXPECTED_EXPOSURE = { + "$feature_flag": FLAG, + "$feature_flag_response": "annual", + "$feature_flag_variant": "annual-first", + "$feature_flag_rule_id": RULE, + "locally_evaluated": False, + "$feature_flag_id": 77, + "$feature_flag_version": 3, + "$feature_flag_request_id": REQUEST_ID, + "$feature_flag_evaluated_at": EVALUATED_AT, +} + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_split_sends_both_events_with_exact_properties(path): + response = _response(_split(description="Assigned variant 'annual-first'")) + + events = _read_events(path, response, groups={"company": "company-7"}) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + called, exposure = events + assert called.properties == { + "$feature_flag": FLAG, + "$feature_flag_response": "annual", + "locally_evaluated": False, + f"$feature/{FLAG}": "annual", + "$feature_flag_request_id": REQUEST_ID, + "$feature_flag_evaluated_at": EVALUATED_AT, + # The reason code replaces the record's reason description. + "$feature_flag_version": 3, + "$feature_flag_id": 77, + "$feature_flag_has_experiment": True, + **EXPECTED_ATTRIBUTION, + } + assert exposure.properties == EXPECTED_EXPOSURE + assert exposure.distinct_id == called.distinct_id == DISTINCT_ID + assert exposure.groups == called.groups == {"company": "company-7"} + assert exposure.property_allowlist is None + + +async def test_async_split_sends_both_events_with_exact_properties(): + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags( + DISTINCT_ID, groups={"company": 7}, disable_geoip=True + ) + snapshot.get_flag(FLAG) + + response = _response(_split(description="Assigned variant 'annual-first'")) + events = await _read_async_events([response], [read]) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + called, exposure = events + assert exposure.properties == EXPECTED_EXPOSURE + assert _attribution(called.properties) == EXPECTED_ATTRIBUTION + assert "$feature_flag_reason" not in called.properties + # The async client sends group keys as strings. + assert exposure.groups == called.groups == {"company": "7"} + assert exposure.distinct_id == called.distinct_id == DISTINCT_ID + + +def test_the_pair_shares_capture_identity_but_not_its_uuid(): + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(message) + return message + + client = Client(FAKE_TEST_API_KEY, send=False, before_send=before_send) + try: + with _serving(_response(_split())): + snapshot = client.evaluate_flags( + DISTINCT_ID, groups={"company": "company-7"}, disable_geoip=True + ) + assert snapshot.get_flag(FLAG) == "annual" + finally: + client.shutdown() + + called, exposure = messages + assert (called["event"], exposure["event"]) == (FLAG_CALLED, EXPOSURE) + assert called["uuid"] and exposure["uuid"] and called["uuid"] != exposure["uuid"] + assert exposure["distinct_id"] == called["distinct_id"] == DISTINCT_ID + for name in ("$groups", "$lib", "$lib_version", "$geoip_disable"): + assert exposure["properties"][name] == called["properties"][name], name + assert exposure["properties"]["$groups"] == {"company": "company-7"} + + +@pytest.mark.parametrize("path", SNAPSHOT_READS) +def test_an_assignment_is_reported_whatever_the_read_returns(path): + # An experiment arm whose value is false still assigned the visitor to that arm. + response = _response(_split(value=False, variant_key="no-banner")) + + events = _read_events(path, response) + + [exposure] = _named(events, EXPOSURE) + assert exposure.properties["$feature_flag_response"] is False + assert exposure.properties["$feature_flag_variant"] == "no-banner" + + +# --- What sends no exposure ------------------------------------------------------------ + + +@pytest.mark.parametrize("path", SINGLE_FLAG_READS) +def test_opting_out_of_flag_events_suppresses_both(client, path): + recording, events = _record_events(client) + with _serving(_response(_split())), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + getattr(client, path)(FLAG, DISTINCT_ID, send_feature_flag_events=False) + + assert events == [] + + +def test_a_payload_read_sends_no_flag_events(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + client.get_feature_flag_payload(FLAG, DISTINCT_ID) + snapshot = client.evaluate_flags(DISTINCT_ID) + snapshot.get_flag_payload(FLAG) + + assert events == [] + + +def test_a_payload_read_that_sends_flag_events_also_sends_the_exposure(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + client.get_feature_flag_payload( + FLAG, DISTINCT_ID, send_feature_flag_events=True + ) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + + +MATCH_VALUE_CASES = [ + ("another value", _split(value="annual"), "monthly"), + ("the record's own value", _split(value="annual"), "annual"), + # Object and number arms both read as True, so equal values prove nothing about + # where a value came from. + ("an object arm", _split(value={"layout": "grid"}), True), + ("a number arm", _split(value=12.5), True), +] + + +@pytest.mark.parametrize("later_read", list(SYNC_READS)) +@pytest.mark.parametrize( + "record, match_value", + [case[1:] for case in MATCH_VALUE_CASES], + ids=[case[0] for case in MATCH_VALUE_CASES], +) +def test_a_payload_lookup_for_a_match_value_sends_no_exposure( + client, record, match_value, later_read +): + # The lookup reports the caller's match_value, not the record's value, so it + # sends no exposure, whatever the value. A later read of the record does. + recording, events = _record_events(client) + with _serving(_response(record)), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + client.get_feature_flag_payload( + FLAG, DISTINCT_ID, match_value=match_value, send_feature_flag_events=True + ) + [called] = events + assert called.name == FLAG_CALLED + assert called.properties["$feature_flag_response"] == match_value + # The call still carries the record's attribution. + assert _attribution(called.properties) == EXPECTED_ATTRIBUTION + + _read(client, later_read) + + [exposure] = _named(events, EXPOSURE) + assert exposure.properties["$feature_flag_variant"] == "annual-first" + + +def test_a_stale_cached_value_sends_no_exposure(client): + # The record arrived, but caching it failed, so the read reports the stale cached + # value instead. That value did not come from the record, even though it is equal. + cache = mock.Mock() + cache._set_cached_remote_flag.side_effect = RuntimeError("cache unavailable") + cache.get_stale_cached_flag.return_value = FeatureFlagResult( + key=FLAG, enabled=True, variant="annual", payload=None, reason=None + ) + client.flag_cache = cache + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + result = client.get_feature_flag_result(FLAG, DISTINCT_ID) + + assert result is not None and result.get_value() == "annual" + cache.get_stale_cached_flag.assert_called_once() + assert [event.name for event in events] == [FLAG_CALLED] + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_locally_evaluated_flag_is_unchanged(path): + client = Client(FAKE_TEST_API_KEY, send=False, secret_key="secret-key") + client.feature_flags = [ + { + "id": 9, + "name": "Pricing page", + "key": FLAG, + "active": True, + "filters": { + "groups": [{"properties": [], "rollout_percentage": 100}], + "multivariate": { + "variants": [{"key": "annual", "rollout_percentage": 100}] + }, + }, + } + ] + recording, events = _record_events(client) + try: + with _serving(_response(_split())) as remote, recording: + _read(client, path) + finally: + client.shutdown() + + assert remote.call_count == 0 + [called] = events + assert called.name == FLAG_CALLED + assert called.properties["locally_evaluated"] is True + assert _attribution(called.properties) == {} + + +def test_an_empty_snapshot_sends_nothing(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + snapshot = client.evaluate_flags() + snapshot.get_flag(FLAG) + snapshot.is_enabled(FLAG) + + assert events == [] + + +def test_a_failed_exposure_capture_does_not_change_the_read(): + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + if message["event"] == EXPOSURE: + raise ValueError("rejected") + return message + + client = Client(FAKE_TEST_API_KEY, send=False, before_send=before_send) + try: + with _serving(_response(_split())), warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + assert client.get_feature_flag(FLAG, DISTINCT_ID) == "annual" + assert client.evaluate_flags(DISTINCT_ID).get_flag(FLAG) == "annual" + finally: + client.shutdown() + + +# --- Dedupe ------------------------------------------------------------------------ + + +@pytest.mark.parametrize( + "first, second", + [ + ("get_feature_flag", "get_feature_flag"), + ("snapshot.get_flag", "snapshot.is_enabled"), + ("feature_enabled", "snapshot.get_flag"), + ("snapshot.get_flag", "get_feature_flag_result"), + ], +) +def test_a_repeated_read_sends_nothing_new(client, first, second): + # The second evaluation has a new request ID and time, which neither event's + # dedupe reads. + later = _response( + _split(), request_id=OTHER_REQUEST_ID, evaluated_at=EVALUATED_AT + 1 + ) + recording, events = _record_events(client) + with _serving(_response(_split()), later), recording: + _read(client, first) + _read(client, second) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_equal_valued_arms_are_each_reported(path): + # An A/A experiment: both arms return the same value. + events = _read_events( + path, + _response(_split(value=True, variant_key="control")), + _response(_split(value=True, variant_key="test")), + reads=2, + ) + + exposures = _named(events, EXPOSURE) + assert [e.properties["$feature_flag_variant"] for e in exposures] == [ + "control", + "test", + ] + assert len(_named(events, FLAG_CALLED)) == 2 + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_later_experiment_on_the_flag_is_reported_again(path): + events = _read_events( + path, + _response(_split(rule_id=RULE)), + _response(_split(rule_id=OTHER_RULE)), + reads=2, + ) + + exposures = _named(events, EXPOSURE) + assert [e.properties["$feature_flag_rule_id"] for e in exposures] == [ + RULE, + OTHER_RULE, + ] + assert len(_named(events, FLAG_CALLED)) == 2 + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_the_exposure_dedupe_leaves_out_the_value(path): + # The arm's value changed, but the visitor is still in the same arm. + events = _read_events( + path, + _response(_split(value="annual")), + _response(_split(value="annual-v2")), + reads=2, + ) + + assert len(_named(events, FLAG_CALLED)) == 2 + assert len(_named(events, EXPOSURE)) == 1 + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_holdout_and_a_split_of_one_rule_are_both_reported(path): + holdout = _record( + "holdout", False, rule_type="experiment", rule_id=RULE, has_experiment=True + ) + events = _read_events( + path, _response(holdout), _response(_split(value=False)), reads=2 + ) + + called = _named(events, FLAG_CALLED) + assert [e.properties["$feature_flag_reason_code"] for e in called] == [ + "holdout", + "experiment_split", + ] + assert len(_named(events, EXPOSURE)) == 1 + + +def test_groups_are_part_of_the_exposure_identity(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + _read(client, "snapshot.get_flag", groups={"company": "a", "team": "b"}) + _read(client, "get_feature_flag", groups={"team": "b", "company": "a"}) + _read(client, "snapshot.get_flag", groups={"company": "c"}) + + exposures = _named(events, EXPOSURE) + assert [e.groups for e in exposures] == [ + {"company": "a", "team": "b"}, + {"company": "c"}, + ] + + +def test_the_exposure_dedupe_normalizes_the_distinct_id(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + _read(client, "get_feature_flag_result", distinct_id=42) + _read(client, "snapshot.get_flag", distinct_id="42") + + assert len(_named(events, EXPOSURE)) == 1 + + +def test_a_reported_call_never_suppresses_its_exposure(client): + # A read whose value the record did not supply reports the call without an + # exposure. A later read of the same record still sends the exposure, although + # its call is a duplicate. + with _serving(_response(_split())): + details = client.evaluate_flags(DISTINCT_ID)._flags[FLAG].details + recording, events = _record_events(client) + with recording: + for value_from_details in (False, True): + client._capture_feature_flag_called_if_needed( + distinct_id=DISTINCT_ID, + key=FLAG, + response="annual", + properties={"$feature_flag": FLAG, "$feature_flag_response": "annual"}, + details=details, + value_from_details=value_from_details, + ) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + + +async def test_async_a_reported_call_never_suppresses_its_exposure(): + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags(DISTINCT_ID) + details = snapshot._flags[FLAG].details + for value_from_details in (False, True): + client._capture_feature_flag_called_if_needed( + distinct_id=DISTINCT_ID, + key=FLAG, + response="annual", + properties={"$feature_flag": FLAG, "$feature_flag_response": "annual"}, + details=details, + value_from_details=value_from_details, + ) + + events = await _read_async_events([_response(_split())], [read]) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + + +async def test_async_snapshots_dedupe_both_events(): + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags(DISTINCT_ID) + snapshot.get_flag(FLAG) + snapshot.is_enabled(FLAG) + snapshot.only([FLAG]).get_flag(FLAG) + + events = await _read_async_events( + [ + _response(_split(variant_key="control", value=True)), + _response(_split(variant_key="control", value=True)), + _response(_split(variant_key="test", value=True)), + ], + [read, read, read], + ) + + assert [e.properties["$feature_flag_variant"] for e in events] == [ + "control", + "control", + "test", + "test", + ] + assert [e.name for e in events] == [FLAG_CALLED, EXPOSURE] * 2 + + +def test_the_v1_dedupe_key_is_unchanged(client): + with _serving(_build(READERS, {"template": "v1"})): + _read(client, "get_feature_flag", "garden-layout") + + assert client.distinct_ids_feature_flags_reported[DISTINCT_ID] == { + ("garden-layout", "compact", ()) + } + assert len(client._experiment_exposures_reported) == 0 + + +def test_shutdown_clears_the_exposure_dedupe(): + client = Client(FAKE_TEST_API_KEY, send=False) + with _serving(_response(_split())): + _read(client, "snapshot.get_flag") + assert len(client._experiment_exposures_reported) == 1 + + client.shutdown() + + assert len(client._experiment_exposures_reported) == 0 + + +async def test_async_shutdown_clears_the_exposure_dedupe(): + with mock.patch( + "posthog.async_client._async_flags", + new=mock.AsyncMock(return_value=_response(_split())), + ): + client = AsyncPosthog("project-key", send=False) + (await client.evaluate_flags(DISTINCT_ID)).get_flag(FLAG) + assert len(client._experiment_exposures_reported) == 1 + await client.shutdown() + + assert len(client._experiment_exposures_reported) == 0 + + +# --- Minimal events ------------------------------------------------------------------- + + +def test_minimal_flag_called_events_keep_the_attribution(): + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(message) + return message + + client = Client( + FAKE_TEST_API_KEY, + send=False, + before_send=before_send, + super_properties={"plan": "team"}, + ) + # A split of an experiment rule no experiment links to: the flag reports no + # experiment, so its $feature_flag_called is minimal. + response = _response(_split(has_experiment=False), minimalFlagCalledEvents=True) + try: + with _serving(response): + client.evaluate_flags(DISTINCT_ID).get_flag(FLAG) + finally: + client.shutdown() + + called, exposure = messages + assert set(called["properties"]) <= _MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES + assert "plan" not in called["properties"] + assert _attribution(called["properties"]) == EXPECTED_ATTRIBUTION + # The exposure is a full event. + assert exposure["event"] == EXPOSURE + assert exposure["properties"]["plan"] == "team" + assert _attribution(exposure["properties"]) == _attribution(EXPECTED_EXPOSURE) + + +# --- Reason codes, rule IDs and variant keys --------------------------------------------- + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize("code", ["future_reason", None], ids=["unknown", "missing"]) +def test_an_unknown_or_missing_reason_code_still_reports_config_version_2(code, path): + record = _split() + if code is None: + del record["reason"] + else: + record["reason"]["code"] = code + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + expected: Dict[str, Any] = {"$feature_flag_config_version": 2} + if code is not None: + expected["$feature_flag_reason_code"] = code + assert _attribution(called.properties) == expected + assert "$feature_flag_reason" not in called.properties + assert _named(events, EXPOSURE) == [] + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +NOT_A_UUID = ["rule-1", "", "7F3C2A108D4E4B6A9C215E0F1A2B3C4D", "{" + RULE + "}"] + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize("rule_id", NOT_A_UUID) +def test_a_rule_id_that_is_not_a_uuid_counts_as_missing(rule_id, path): + for record in ( + _split(rule_id=rule_id), + _record("targeting_match", rule_type="targeted_release", rule_id=rule_id), + ): + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": record["reason"]["code"], + } + assert _named(events, EXPOSURE) == [] + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +def test_an_upper_case_uuid_rule_id_is_kept(): + events = _read_events( + "get_feature_flag_result", _response(_split(rule_id=RULE.upper())) + ) + + [exposure] = _named(events, EXPOSURE) + assert exposure.properties["$feature_flag_rule_id"] == RULE.upper() + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_an_empty_variant_key_is_a_complete_split(path): + events = _read_events(path, _response(_split(variant_key=""))) + + [called] = _named(events, FLAG_CALLED) + assert called.properties["$feature_flag_variant"] == "" + [exposure] = _named(events, EXPOSURE) + assert exposure.properties["$feature_flag_variant"] == "" + EXPOSURE_SCHEMA.validate(_projection(EXPOSURE_SCHEMA, exposure.properties)) + + +# --- Capture failures ---------------------------------------------------------------------- + + +def _dropping(event: str) -> Callable[[Dict[str, Any]], Optional[Dict[str, Any]]]: + """A before_send hook that drops the first ``event`` and keeps everything else.""" + dropped: List[str] = [] + + def before_send(message: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if message["event"] == event and not dropped: + dropped.append(event) + return None + return message + + return before_send + + +def _failing_once(client: Client, event: str, failure: str) -> Any: + """Patch ``client.capture`` to fail for the first ``event``, then capture as usual. + + ``failure`` is ``"raise"`` for a raising capture, or ``"unqueued"`` for a capture + that returns ``None`` without ``before_send`` dropping the event, as a full or + closed queue, opt-out or a disabled client does. + """ + capture = client.capture + failed: List[str] = [] + + def maybe_fail(name: str, **kwargs: Any) -> Optional[str]: + if name == event and not failed: + failed.append(name) + if failure == "raise": + raise RuntimeError("capture failed") + return None + return capture(name, **kwargs) + + return mock.patch.object(client, "capture", side_effect=maybe_fail) + + +def _sent(client: Client, path: str, reads: int = 2) -> List[str]: + """The events that reached the queue over ``reads`` reads of a complete split.""" + sent: List[str] = [] + enqueue = client._enqueue + + def record(msg: Dict[str, Any], *args: Any, **kwargs: Any) -> Optional[str]: + uuid = enqueue(msg, *args, **kwargs) + if uuid is not None: + sent.append(msg["event"]) + return uuid + + with ( + _serving(_response(_split())), + mock.patch.object(client, "_enqueue", side_effect=record), + ): + for _ in range(reads): + assert _read(client, path) == "annual" + return sent + + +@pytest.mark.parametrize("path", ["get_feature_flag", "snapshot.get_flag"]) +@pytest.mark.parametrize("event", [FLAG_CALLED, EXPOSURE]) +def test_an_event_dropped_by_before_send_keeps_its_dedupe_key(event, path): + client = Client(FAKE_TEST_API_KEY, send=False, before_send=_dropping(event)) + try: + sent = _sent(client, path) + finally: + client.shutdown() + + # The application dropped the event, so the second read does not send it again. + # The other event of the first read is still sent, once. + assert sent == [EXPOSURE if event == FLAG_CALLED else FLAG_CALLED] + + +async def test_async_an_event_dropped_by_before_send_keeps_its_dedupe_key(): + seen: List[str] = [] + batches: List[Any] = [] + + def before_send(message: Dict[str, Any]) -> Optional[Dict[str, Any]]: + seen.append(message["event"]) + return None if message["event"] == EXPOSURE else message + + async def batch_post(*args: Any, **kwargs: Any) -> None: + batches.append(kwargs["batch"]) + + with ( + mock.patch( + "posthog.async_client._async_flags", + new=mock.AsyncMock(return_value=_response(_split())), + ), + mock.patch("posthog._async_consumer.async_batch_post", side_effect=batch_post), + ): + client = AsyncPosthog("project-key", before_send=before_send, flush_interval=30) + for _ in range(2): + snapshot = await client.evaluate_flags(DISTINCT_ID) + assert snapshot.get_flag(FLAG) == "annual" + await client.flush(timeout_seconds=1) + await client.shutdown() + + # before_send saw each event once: the dropped exposure kept its key, so the + # second read sent neither event again. + assert seen == [FLAG_CALLED, EXPOSURE] + sent = [event["event"] for batch in batches for event in batch] + assert sent == [FLAG_CALLED] + + +@pytest.mark.parametrize("path", ["get_feature_flag", "snapshot.get_flag"]) +@pytest.mark.parametrize("event", [FLAG_CALLED, EXPOSURE]) +@pytest.mark.parametrize("failure", ["raise", "unqueued"]) +def test_a_failed_capture_keeps_no_dedupe_key(failure, event, path, client): + with _failing_once(client, event, failure): + sent = _sent(client, path) + + # The other event of the first read is still sent, and the failed one is sent + # by the second read, once. + assert sorted(sent) == sorted([FLAG_CALLED, EXPOSURE]) + + +def test_an_unqueued_config_version_1_call_is_deduped_as_before(client): + record = _record("condition_match", "annual", config_version=1) + calls: List[str] = [] + + def unqueued(name: str, **kwargs: Any) -> None: + calls.append(name) + return None + + with ( + _serving(_response(record)), + mock.patch.object(client, "capture", side_effect=unqueued), + ): + _read(client, "get_feature_flag") + _read(client, "get_feature_flag") + + # Config version 1 dedupe is unchanged: the attempt takes the key. + assert calls == [FLAG_CALLED] + + +def test_a_flags_reload_does_not_repeat_an_exposure(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + _read(client, "get_feature_flag") + client._update_flag_state( + {"flags": [], "group_type_mapping": {}, "cohorts": {}} + ) + _read(client, "get_feature_flag") + client.evaluate_flags(DISTINCT_ID).get_flag(FLAG) + + assert len(_named(events, EXPOSURE)) == 1 + + +# --- Super properties ------------------------------------------------------------------------ + +SPOOFED_SUPER_PROPERTIES = { + "$feature_flag_config_version": 7, + "$feature_flag_reason_code": "spoofed", + "$feature_flag_rule_type": "spoofed", + "$feature_flag_rule_id": OTHER_RULE, + "$feature_flag_variant": "spoofed", + "$feature_flag_reason": "spoofed", + "$feature_flag_forced_variant": True, + "$feature_flag_experiment_id": 301, + "$experiment_exposure_source": "spoofed", +} + + +def _messages_with_super_properties(record: Dict[str, Any]) -> List[Dict[str, Any]]: + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(copy.deepcopy(message)) + return message + + client = Client( + FAKE_TEST_API_KEY, + send=False, + before_send=before_send, + super_properties=dict(SPOOFED_SUPER_PROPERTIES), + ) + try: + with _serving(_response(record)): + _read(client, "get_feature_flag") + finally: + client.shutdown() + return messages + + +def test_super_properties_cannot_change_config_version_2_attribution(): + messages = _messages_with_super_properties(_split()) + + [called] = [m for m in messages if m["event"] == FLAG_CALLED] + assert _attribution(called["properties"]) == EXPECTED_ATTRIBUTION + for name in ( + "$feature_flag_reason", + "$feature_flag_forced_variant", + "$feature_flag_experiment_id", + ): + assert name not in called["properties"], name + [exposure] = [m for m in messages if m["event"] == EXPOSURE] + assert _attribution(exposure["properties"]) == _attribution(EXPECTED_EXPOSURE) + assert not set(EXPOSURE_LEAVES_OUT) & set(exposure["properties"]) + for name in ("$feature_flag_experiment_id", "$feature_flag_forced_variant"): + assert name not in exposure["properties"], name + EXPOSURE_SCHEMA.validate(_projection(EXPOSURE_SCHEMA, exposure["properties"])) + + +def test_super_properties_cannot_add_rule_fields_a_config_version_2_call_omits(): + messages = _messages_with_super_properties(_record("no_rule_match", False)) + + [called] = messages + assert _attribution(called["properties"]) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "no_rule_match", + } + + +def test_super_properties_still_apply_to_config_version_1_calls(): + record = _record("condition_match", "annual", config_version=1) + [called] = _messages_with_super_properties(record) + + for name, value in SPOOFED_SUPER_PROPERTIES.items(): + assert called["properties"][name] == value, name + + +async def test_async_super_properties_cannot_change_config_version_2_attribution(): + messages: List[Dict[str, Any]] = [] + with mock.patch( + "posthog.async_client._async_flags", + new=mock.AsyncMock(return_value=_response(_split())), + ): + client = AsyncPosthog( + "project-key", + send=False, + super_properties=dict(SPOOFED_SUPER_PROPERTIES), + ) + prepare = client._prepare_event + + def record(*args: Any, **kwargs: Any) -> Any: + prepared, uuid = prepare(*args, **kwargs) + messages.append(copy.deepcopy(prepared)) + return prepared, uuid + + with mock.patch.object(client, "_prepare_event", side_effect=record): + snapshot = await client.evaluate_flags(DISTINCT_ID) + snapshot.get_flag(FLAG) + await client.shutdown() + + [called] = [m for m in messages if m["event"] == FLAG_CALLED] + assert _attribution(called["properties"]) == EXPECTED_ATTRIBUTION + assert "$feature_flag_reason" not in called["properties"] + [exposure] = [m for m in messages if m["event"] == EXPOSURE] + assert _attribution(exposure["properties"]) == _attribution(EXPECTED_EXPOSURE) + assert not set(EXPOSURE_LEAVES_OUT) & set(exposure["properties"]) diff --git a/posthog/test/test_flags_released_behaviour.py b/posthog/test/test_flags_released_behaviour.py index 7ad8022f8..368f417cd 100644 --- a/posthog/test/test_flags_released_behaviour.py +++ b/posthog/test/test_flags_released_behaviour.py @@ -30,7 +30,7 @@ def results(request: Any) -> Dict[str, Any]: return scenarios.run_all(request.param) -def test_every_legacy_getter_and_event_matches_7_82_1(results): +def test_every_legacy_getter_and_event_matches_7_82_1(request, results): expected = copy.deepcopy(RELEASED["getters"]) # A record without a flag ID or version reads them as absent (None), where 7.82.1 # read 0. Events already left out both, so only the parsed record differs. @@ -38,6 +38,21 @@ def test_every_legacy_getter_and_event_matches_7_82_1(results): assert (sparse["id"], sparse["version"]) == (0, 0) sparse["id"] = sparse["version"] = None + if request.node.callspec.params["results"] == "v4": + # A $feature_flag_called for a config version 2 record carries its config + # version and reason code in place of the reason text. Only the v4 shape says + # a record is config version 2; config version 1 events are unchanged. + for name, properties in expected["events"]: + if name == "$feature_flag_called" and properties[ + "$feature_flag" + ].startswith("v2-"): + assert ( + properties.pop("$feature_flag_reason") + == scenarios.MATCHED["description"] + ) + properties["$feature_flag_config_version"] = 2 + properties["$feature_flag_reason_code"] = scenarios.MATCHED["code"] + assert results["getters"] == expected diff --git a/posthog/test/test_flags_v4_response.py b/posthog/test/test_flags_v4_response.py index 662011dce..c48a2936f 100644 --- a/posthog/test/test_flags_v4_response.py +++ b/posthog/test/test_flags_v4_response.py @@ -83,8 +83,8 @@ def _evaluate(client: Client, response: Dict[str, Any]) -> FeatureFlagEvaluation def test_vendored_fixtures_match_the_recorded_release_digests(): source = _load("SOURCE.json") - assert source["release"] == "1.22.0" - assert source["contract_version"] == "7.0.0" + assert source["release"] == "1.28.0" + assert source["contract_version"] == "9.0.0" for name, entry in source["files"].items(): digest = hashlib.sha256((FIXTURES / name).read_bytes()).hexdigest() assert digest == entry["sha256"], name @@ -1009,11 +1009,27 @@ def _parity_response(shape: str) -> Dict[str, Any]: } +# A config version 2 record adds these to its flag events, and leaves out +# $feature_flag_reason. A v2-shaped response has no config version 2 records, so they +# are left out of the comparison of the v2-* flags and checked in +# test_experiment_exposure.py. +_CONFIG_VERSION_2_EVENT_PROPERTIES = { + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_id", + "$feature_flag_rule_type", + "$feature_flag_variant", +} + + def _flag_event_properties(message: Dict[str, Any]) -> Dict[str, Any]: + left_out = set(_CONFIG_VERSION_2_EVENT_PROPERTIES) + if str(message["properties"].get("$feature_flag", "")).startswith("v2-"): + left_out.add("$feature_flag_reason") return { key: value for key, value in message["properties"].items() - if key.startswith("$feature") + if (key.startswith("$feature") and key not in left_out) or key in ("$active_feature_flags", "locally_evaluated") } @@ -1029,7 +1045,8 @@ def response(*args: Any, **kwargs: Any) -> Dict[str, Any]: def enqueue(message: Dict[str, Any], *args: Any, **kwargs: Any) -> Optional[str]: events.append((message["event"], _flag_event_properties(message))) - return None + # Accept the event, as the real queue does, by returning an event UUID. + return "70000000-0000-4000-8000-000000000000" try: with (