From 484c4155d14ba72eaa24c3c835ddafd5acd4e393 Mon Sep 17 00:00:00 2001 From: Radu Raicea Date: Mon, 19 Oct 2026 14:08:38 -0400 Subject: [PATCH] feat(llma): support scorer and experiment tags --- .sampo/changesets/evaluation-tags.md | 5 + .../example-offline-evaluations/README.md | 37 ++++++++ posthog/ai/evaluations/_client.py | 50 ++++++++++ posthog/ai/evaluations/_scorer_types.py | 2 + posthog/ai/evaluations/_scorers.py | 67 ++++++++++--- posthog/ai/evaluations/_tags.py | 28 ++++++ posthog/test/ai/test_evaluations.py | 95 +++++++++++++++++++ posthog/test/ai/test_evaluations_scorers.py | 92 ++++++++++++++++++ references/public_api_snapshot.txt | 19 ++-- 9 files changed, 376 insertions(+), 19 deletions(-) create mode 100644 .sampo/changesets/evaluation-tags.md create mode 100644 posthog/ai/evaluations/_tags.py diff --git a/.sampo/changesets/evaluation-tags.md b/.sampo/changesets/evaluation-tags.md new file mode 100644 index 000000000..ea734ed30 --- /dev/null +++ b/.sampo/changesets/evaluation-tags.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: minor +--- + +Support tags in synchronous and asynchronous AI observability scorer creation, metadata updates, responses, and list filters. Add `get_tags()` and `set_tags()` to offline experiment handles for reading, replacing, and clearing experiment tags independently of uploads and completion. diff --git a/examples/example-offline-evaluations/README.md b/examples/example-offline-evaluations/README.md index c1ec2207f..279fe4e70 100644 --- a/examples/example-offline-evaluations/README.md +++ b/examples/example-offline-evaluations/README.md @@ -11,6 +11,8 @@ Offline evaluations must be enabled for your PostHog project. Use the project's | Create experiments, upload results, complete/fail | Personal API key or project secret key with `offline_evaluation_ingestion:write` | | Fetch scorers and versions | Personal API key with `llm_analytics:read` | | Create/update scorers and create versions | Personal API key with `llm_analytics:write` | +| Read experiment tags | Personal API key with `evaluation:read` | +| Replace experiment tags | Personal API key with `evaluation:write` | A personal key can cover both scorer management and uploads. Project secret keys can upload using a known scorer-version UUID without scorer read access. All operations use the explicit project/environment; scorer names are not unique. @@ -147,6 +149,41 @@ version = evaluations.scorers.get_version(scorer.id, updated.current_version_id) Scorer creation, version creation, and metadata updates are never automatically retried. If a response is lost, inspect server state before repeating a mutation. Version creation is not made idempotent by `base_version`. +## Tags + +Assign tags when creating a scorer, or replace them with a metadata update: + +```python +scorer = evaluations.scorers.create( + name="Answer relevance", + kind="numeric", + config=NumericScorerConfig(min=0, max=1), + tags=["quality", "production"], +) +print(scorer.tags) # ("production", "quality") +scorer = evaluations.scorers.update(scorer.id, tags=["quality"]) +page = evaluations.scorers.list(tags=["quality", "production"], tag_match="all") +evaluations.scorers.update(scorer.id, tags=[]) # Remove all tags. +``` + +Omitting `tags` preserves existing tags on updates. `tags=[]` clears them; `None` is not a valid replacement. Lists and tuples are accepted. The server trims and lowercases names, removes duplicates, and sorts them. Returned tags are immutable tuples. Writes accept up to 100 names, each nonblank and at most 255 characters after trimming. + +List filters accept up to 20 tags. `tag_match="any"` (the default) matches at least one selected tag; `"all"` requires every selected tag. An empty filter disables tag filtering. Preserve these filters when requesting subsequent pages. + +Experiment tags are separate from the creation/upload declaration. Set them on a newly created or resumed handle, including after completion: + +```python +experiment = evaluations.resume_experiment(experiment_id) +tags = experiment.set_tags(["quality", "baseline"]) +print(tags) # ("baseline", "quality") +print(experiment.get_tags()) +experiment.set_tags([]) # Remove all tags. +``` + +Reading and writing experiment tags require the personal-key scopes listed above; an upload-only project secret key cannot manage tags. Tag updates do not change the saved `submission`, lifecycle `receipt`, or scorer versions. They are not retried automatically: after a lost acknowledgment, read the current tags before deciding whether to repeat the replacement, since another caller may have edited them. + +The async client exposes the same API: await scorer methods and `experiment.get_tags()` / `experiment.set_tags(...)`. + ## Receipts, errors, and recovery Upload receipts contain persisted item/result UUIDs, original acceptance times, and `created` flags. `created=False` acknowledges an exact duplicate. A bulk upload returns `chunks`; each chunk commits independently. A malformed or incomplete acknowledgment raises rather than reporting success. diff --git a/posthog/ai/evaluations/_client.py b/posthog/ai/evaluations/_client.py index 9e7c75c56..69fc622cb 100644 --- a/posthog/ai/evaluations/_client.py +++ b/posthog/ai/evaluations/_client.py @@ -23,6 +23,7 @@ _timestamp, _uuid, ) +from ._tags import _response_tags, _tag_names from ._transport import AsyncTransport, SyncTransport from ._types import ( BulkUploadError, @@ -38,6 +39,20 @@ ) +def _experiment_tags(data: dict[str, Any], identity: str) -> tuple[str, ...]: + try: + if _uuid(data["id"]) != identity: + raise ValueError("Mismatched experiment ID.") + return _response_tags(data["tags"]) + except (KeyError, TypeError, ValueError): + raise EvaluationAPIError( + code="invalid_response", + detail="The experiment API returned invalid tag metadata.", + response=data, + persistence="unknown", + ) from None + + def _experiment_body(name: str, options: ExperimentOptions) -> bytes: allowed = set(ExperimentOptions.__annotations__) if set(options) - allowed: @@ -179,6 +194,26 @@ def submission(self) -> dict[str, Any] | None: """Copy of original creation fields; unavailable on a resumed handle.""" return None if self._submission is None else json.loads(self._submission) + def get_tags(self) -> tuple[str, ...]: + """Fetch current tags using a personal key with ``evaluation:read``.""" + with self._lock: + data = self._transport.request("GET", self._path) + return _experiment_tags(data, self.id) + + def set_tags(self, tags: Sequence[str]) -> tuple[str, ...]: + """Replace tags and return the server's normalized names; [] clears them. + + Requires a personal key with ``evaluation:write``. Tags can change after + completion and are separate from the immutable creation ``submission``. + Updates are not retried automatically: inspect tags after a lost response. + """ + body = _encode({"tags": _tag_names(tags)}) + with self._lock: + data = self._transport.request( + "PATCH", self._path, body=body, retry_safe=False + ) + return _experiment_tags(data, self.id) + def upload_result( self, *, @@ -289,6 +324,21 @@ def submission(self) -> dict[str, Any] | None: """Copy of original creation fields; unavailable on a resumed handle.""" return None if self._submission is None else json.loads(self._submission) + async def get_tags(self) -> tuple[str, ...]: + """Fetch current tags. See Experiment.get_tags for key requirements.""" + async with self._lock: + data = await self._transport.request("GET", self._path) + return _experiment_tags(data, self.id) + + async def set_tags(self, tags: Sequence[str]) -> tuple[str, ...]: + """Replace tags. See Experiment.set_tags for permissions and recovery.""" + body = _encode({"tags": _tag_names(tags)}) + async with self._lock: + data = await self._transport.request( + "PATCH", self._path, body=body, retry_safe=False + ) + return _experiment_tags(data, self.id) + async def upload_result( self, *, diff --git a/posthog/ai/evaluations/_scorer_types.py b/posthog/ai/evaluations/_scorer_types.py index 6dd204541..e7aedd95b 100644 --- a/posthog/ai/evaluations/_scorer_types.py +++ b/posthog/ai/evaluations/_scorer_types.py @@ -96,6 +96,7 @@ class Scorer(Generic[_ConfigT]): Read ``current_version_id`` once to pin the version for an experiment. Names are not unique; use ``id`` for subsequent management operations. Configuration dictionaries are local snapshots, not live server state. + ``tags`` contains normalized tag names and is independent of versions. """ id: str @@ -110,6 +111,7 @@ class Scorer(Generic[_ConfigT]): updated_at: datetime created_by: dict[str, Any] | None team: int + tags: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/posthog/ai/evaluations/_scorers.py b/posthog/ai/evaluations/_scorers.py index 2c5dc3d10..1d9e15483 100644 --- a/posthog/ai/evaluations/_scorers.py +++ b/posthog/ai/evaluations/_scorers.py @@ -5,7 +5,7 @@ import copy import json from datetime import datetime -from typing import Any, Callable, Literal, TypeVar, cast, overload +from typing import Any, Callable, Literal, Sequence, TypeVar, cast, overload from urllib.parse import parse_qs, urlsplit from uuid import UUID @@ -22,6 +22,7 @@ ScorerVersion, ScorerVersionPage, ) +from ._tags import _response_tags, _tag_names from ._transport import AsyncTransport, SyncTransport @@ -100,6 +101,7 @@ def _scorer(data: dict[str, Any]) -> Scorer[ScorerConfig]: updated_at=_timestamp(data["updated_at"]), created_by=None if created_by is None else _mapping(created_by), team=_integer(data["team"], minimum=1), + tags=_response_tags(data.get("tags", [])), ) @@ -176,6 +178,7 @@ def _create_body( kind: ScorerKind, config: ScorerConfig, description: str | None | _Omitted, + tags: Sequence[str] | _Omitted, ) -> bytes: data: dict[str, Any] = { "name": name, @@ -184,6 +187,8 @@ def _create_body( } if not isinstance(description, _Omitted): data["description"] = description + if not isinstance(tags, _Omitted): + data["tags"] = _tag_names(tags) return _encode(data) @@ -191,8 +196,9 @@ def _metadata( name: str | _Omitted, description: str | None | _Omitted, archived: bool | _Omitted = _OMITTED, + tags: Sequence[str] | _Omitted = _OMITTED, ) -> dict[str, Any]: - return { + data: dict[str, Any] = { key: value for key, value in { "name": name, @@ -201,6 +207,9 @@ def _metadata( }.items() if not isinstance(value, _Omitted) } + if not isinstance(tags, _Omitted): + data["tags"] = _tag_names(tags) + return data def _version_body( @@ -223,7 +232,11 @@ def _list_params( kind: ScorerKind | None, archived: bool | None, order_by: ScorerOrder | None, + tags: Sequence[str] | None, + tag_match: Literal["any", "all"], ) -> dict[str, str | int]: + if tag_match not in ("any", "all"): + raise ValueError("tag_match must be any or all.") params: dict[str, str | int] = { "limit": _integer(limit, minimum=1), "offset": _integer(offset), @@ -238,6 +251,11 @@ def _list_params( params["archived"] = "true" if archived else "false" if order_by is not None: params["order_by"] = order_by + if tags is not None: + params["tags"] = json.dumps(_tag_names(tags, limit=20), ensure_ascii=False) + params["tag_match"] = tag_match + elif tag_match != "any": + params["tag_match"] = tag_match return params @@ -273,6 +291,7 @@ def create( kind: Literal["boolean"], config: BooleanScorerConfig, description: str | None | _Omitted = _OMITTED, + tags: Sequence[str] | _Omitted = _OMITTED, ) -> Scorer[BooleanScorerConfig]: ... @overload @@ -283,6 +302,7 @@ def create( kind: Literal["numeric"], config: NumericScorerConfig, description: str | None | _Omitted = _OMITTED, + tags: Sequence[str] | _Omitted = _OMITTED, ) -> Scorer[NumericScorerConfig]: ... @overload @@ -293,6 +313,7 @@ def create( kind: Literal["categorical"], config: CategoricalScorerConfig, description: str | None | _Omitted = _OMITTED, + tags: Sequence[str] | _Omitted = _OMITTED, ) -> Scorer[CategoricalScorerConfig]: ... def create( @@ -302,14 +323,18 @@ def create( kind: ScorerKind, config: ScorerConfig, description: str | None | _Omitted = _OMITTED, + tags: Sequence[str] | _Omitted = _OMITTED, ) -> Scorer[ScorerConfig]: - """Create an active scorer and its first immutable configuration version.""" + """Create a scorer and its first version, optionally assigning tags. + + Tag names are trimmed, lowercased, deduplicated, and sorted by the server. + """ return _parse( _scorer, self._transport.request( "POST", self._base, - body=_create_body(name, kind, config, description), + body=_create_body(name, kind, config, description, tags), retry_safe=False, ), ) @@ -327,18 +352,24 @@ def list( kind: ScorerKind | None = None, archived: bool | None = None, order_by: ScorerOrder | None = None, + tags: Sequence[str] | None = None, + tag_match: Literal["any", "all"] = "any", ) -> ScorerPage: """Fetch one page. Omit ``archived`` to list active scorers. Keep the same filters and pass the returned ``next_offset`` to continue. Search matches names and descriptions; scorer names are not unique. + ``tags`` accepts up to 20 names. ``tag_match`` matches any (default) or + all selected tags; an empty sequence disables tag filtering. """ return _parse( _scorer_page, self._transport.request( "GET", self._base, - params=_list_params(limit, offset, search, kind, archived, order_by), + params=_list_params( + limit, offset, search, kind, archived, order_by, tags, tag_match + ), ), ) @@ -349,17 +380,20 @@ def update( name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, archived: bool | _Omitted = _OMITTED, + tags: Sequence[str] | _Omitted = _OMITTED, ) -> Scorer[ScorerConfig]: """Update metadata. Use ``create_version`` to change configuration. Omitted fields are preserved. An explicit null description clears it. + ``tags`` replaces the current tags; pass [] to clear them. Tag changes + do not create a scorer version. """ return _parse( _scorer, self._transport.request( "PATCH", self._path(scorer_id), - body=_encode(_metadata(name, description, archived)), + body=_encode(_metadata(name, description, archived, tags)), retry_safe=False, ), ) @@ -435,6 +469,7 @@ async def create( kind: Literal["boolean"], config: BooleanScorerConfig, description: str | None | _Omitted = _OMITTED, + tags: Sequence[str] | _Omitted = _OMITTED, ) -> Scorer[BooleanScorerConfig]: ... @overload @@ -445,6 +480,7 @@ async def create( kind: Literal["numeric"], config: NumericScorerConfig, description: str | None | _Omitted = _OMITTED, + tags: Sequence[str] | _Omitted = _OMITTED, ) -> Scorer[NumericScorerConfig]: ... @overload @@ -455,6 +491,7 @@ async def create( kind: Literal["categorical"], config: CategoricalScorerConfig, description: str | None | _Omitted = _OMITTED, + tags: Sequence[str] | _Omitted = _OMITTED, ) -> Scorer[CategoricalScorerConfig]: ... async def create( @@ -464,14 +501,15 @@ async def create( kind: ScorerKind, config: ScorerConfig, description: str | None | _Omitted = _OMITTED, + tags: Sequence[str] | _Omitted = _OMITTED, ) -> Scorer[ScorerConfig]: - """Create an active scorer and its first version; never retry mutations.""" + """Create a scorer with optional tags. See Scorers.create.""" return _parse( _scorer, await self._transport.request( "POST", self._base, - body=_create_body(name, kind, config, description), + body=_create_body(name, kind, config, description, tags), retry_safe=False, ), ) @@ -491,14 +529,18 @@ async def list( kind: ScorerKind | None = None, archived: bool | None = None, order_by: ScorerOrder | None = None, + tags: Sequence[str] | None = None, + tag_match: Literal["any", "all"] = "any", ) -> ScorerPage: - """Fetch one page; preserve filters and use ``next_offset`` to continue.""" + """Fetch one page with optional tag filters. See Scorers.list.""" return _parse( _scorer_page, await self._transport.request( "GET", self._base, - params=_list_params(limit, offset, search, kind, archived, order_by), + params=_list_params( + limit, offset, search, kind, archived, order_by, tags, tag_match + ), ), ) @@ -509,14 +551,15 @@ async def update( name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, archived: bool | _Omitted = _OMITTED, + tags: Sequence[str] | _Omitted = _OMITTED, ) -> Scorer[ScorerConfig]: - """Update metadata; omission preserves values and null clears description.""" + """Update metadata, including replacement tags. See Scorers.update.""" return _parse( _scorer, await self._transport.request( "PATCH", self._path(scorer_id), - body=_encode(_metadata(name, description, archived)), + body=_encode(_metadata(name, description, archived, tags)), retry_safe=False, ), ) diff --git a/posthog/ai/evaluations/_tags.py b/posthog/ai/evaluations/_tags.py new file mode 100644 index 000000000..be1448f70 --- /dev/null +++ b/posthog/ai/evaluations/_tags.py @@ -0,0 +1,28 @@ +"""Shared validation for mutable scorer and experiment tags.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + + +def _tag_names(tags: Sequence[str], *, limit: int = 100) -> list[str]: + if isinstance(tags, (str, bytes)) or not isinstance(tags, Sequence): + raise ValueError("tags must be a sequence of strings.") + if len(tags) > limit: + raise ValueError(f"tags must contain at most {limit} names.") + names = list(tags) + for name in names: + if not isinstance(name, str) or not name.strip() or len(name.strip()) > 255: + raise ValueError( + "Each tag must be a nonblank string of at most 255 characters." + ) + # The server trims, lowercases, deduplicates, and sorts names. Preserve the + # caller's spelling on the wire and return the server's canonical response. + return names + + +def _response_tags(value: Any) -> tuple[str, ...]: + if not isinstance(value, list) or any(not isinstance(tag, str) for tag in value): + raise ValueError("Expected a list of tag names.") + return tuple(value) diff --git a/posthog/test/ai/test_evaluations.py b/posthog/test/ai/test_evaluations.py index f6f2f96f9..b2e76d0eb 100644 --- a/posthog/test/ai/test_evaluations.py +++ b/posthog/test/ai/test_evaluations.py @@ -114,6 +114,101 @@ def result(item=None, **kwargs): ) +def test_experiment_tags_are_separate_from_creation_and_lifecycle_receipts(client): + evaluations, transport = client + experiment = invoke(evaluations.create_experiment, name="Tagged run") + invoke(experiment.complete) + submission, receipt = experiment.submission, experiment.receipt + transport.request.reset_mock() + transport.request.side_effect = None + response = { + "id": experiment.id, + "status": "completed", + "tags": ["production", "quality"], + } + transport.request.return_value = response + tags = invoke(experiment.set_tags, [" Quality ", "production", "Quality"]) + assert tags == ("production", "quality") + transport.request.assert_called_once_with( + "PATCH", + BASE + experiment.id + "/", + body=b'{"tags":[" Quality ","production","Quality"]}', + retry_safe=False, + ) + response["tags"].append("later") + assert tags == ("production", "quality") + assert experiment.receipt is receipt + assert experiment.submission == submission + assert "tags" not in experiment.submission + + response["tags"] = [] + assert invoke(experiment.set_tags, ()) == () + assert json.loads(transport.request.call_args.kwargs["body"]) == {"tags": []} + assert invoke(experiment.get_tags) == () + transport.request.assert_called_with("GET", BASE + experiment.id + "/") + + +def test_resumed_experiment_can_fetch_tags(client): + evaluations, transport = client + experiment = evaluations.resume_experiment(EXPERIMENT_ID) + transport.request.assert_not_called() + transport.request.side_effect = None + transport.request.return_value = {"id": EXPERIMENT_ID, "tags": ["quality"]} + assert invoke(experiment.get_tags) == ("quality",) + assert experiment.receipt is None + assert experiment.submission is None + + +@pytest.mark.parametrize( + "tags", [None, "quality", {"quality"}, [1], [""], [" "], ["x" * 256], ["q"] * 101] +) +def test_invalid_experiment_tags_are_rejected_before_io(client, tags): + evaluations, transport = client + experiment = evaluations.resume_experiment(EXPERIMENT_ID) + with pytest.raises(ValueError): + invoke(experiment.set_tags, tags) + transport.request.assert_not_called() + + +@pytest.mark.parametrize("operation", ["get_tags", "set_tags"]) +@pytest.mark.parametrize( + "response", + [ + {"id": EXPERIMENT_ID}, + {"id": EXPERIMENT_ID, "tags": None}, + {"id": EXPERIMENT_ID, "tags": "quality"}, + {"id": EXPERIMENT_ID, "tags": [1]}, + {"tags": []}, + {"id": VERSION_ID, "tags": []}, + ], +) +def test_malformed_experiment_tag_responses_are_not_acknowledged( + client, operation, response +): + evaluations, transport = client + experiment = evaluations.resume_experiment(EXPERIMENT_ID) + transport.request.side_effect = None + transport.request.return_value = response + with pytest.raises(EvaluationAPIError) as caught: + invoke( + getattr(experiment, operation), *([[]] if operation == "set_tags" else []) + ) + assert caught.value.code == "invalid_response" + assert caught.value.persistence == "unknown" + assert caught.value.response == response + + +def test_experiment_tag_permission_error_is_preserved(client): + evaluations, transport = client + experiment = evaluations.resume_experiment(EXPERIMENT_ID) + error = EvaluationAPIError(status=403, persistence="rejected") + transport.request.side_effect = error + with pytest.raises(EvaluationAPIError) as caught: + invoke(experiment.set_tags, ["quality"]) + assert caught.value is error + transport.request.assert_called_once() + + def test_create_upload_and_explicit_completion(client): evaluations, transport = client experiment = invoke( diff --git a/posthog/test/ai/test_evaluations_scorers.py b/posthog/test/ai/test_evaluations_scorers.py index 128834c0f..9fa40e918 100644 --- a/posthog/test/ai/test_evaluations_scorers.py +++ b/posthog/test/ai/test_evaluations_scorers.py @@ -174,6 +174,98 @@ def test_metadata_update_preserves_omission_null_and_false(scorers): assert json.loads(transport.request.call_args.kwargs["body"]) == {"name": "Renamed"} +@pytest.mark.parametrize("tags", [[" Quality ", "production", "Quality"], (), []]) +@pytest.mark.parametrize("operation", ["create", "update"]) +def test_tags_are_mutable_metadata_with_normalized_response(scorers, tags, operation): + namespace, transport = scorers + normalized = sorted({tag.strip().lower() for tag in tags}) + transport.request.return_value = scorer_response(tags=normalized) + if operation == "create": + scorer = invoke( + namespace.create, name="Quality", kind="numeric", config={}, tags=tags + ) + else: + scorer = invoke(namespace.update, SCORER_ID, tags=tags) + assert json.loads(transport.request.call_args.kwargs["body"]) == { + "tags": list(tags) + } + assert json.loads(transport.request.call_args.kwargs["body"])["tags"] == list(tags) + assert transport.request.call_args.kwargs["retry_safe"] is False + assert scorer.tags == tuple(normalized) + assert scorer.current_version_id == VERSION_ID + normalized.append("later") + assert "later" not in scorer.tags + transport.request.assert_called_once() + + +def test_scorer_tags_are_optional_for_older_servers(scorers): + namespace, _ = scorers + assert invoke(namespace.get, SCORER_ID).tags == () + + +@pytest.mark.parametrize("tags", [None, "quality", [None], [1], {}]) +def test_invalid_scorer_response_tags_raise(scorers, tags): + namespace, transport = scorers + transport.request.return_value = scorer_response(tags=tags) + with pytest.raises(EvaluationAPIError) as caught: + invoke(namespace.get, SCORER_ID) + assert caught.value.code == "invalid_response" + assert caught.value.persistence == "unknown" + + +@pytest.mark.parametrize("tag_match", ["any", "all"]) +@pytest.mark.parametrize("tags", [[], ("quality", 'a,b"c', "品質")]) +def test_scorer_list_tag_filters_use_json_and_preserve_other_filters( + scorers, tags, tag_match +): + namespace, transport = scorers + transport.request.return_value = { + "count": 1, + "next": None, + "previous": None, + "results": [scorer_response(tags=["quality"])], + } + page = invoke( + namespace.list, tags=tags, tag_match=tag_match, offset=3, search="score" + ) + params = transport.request.call_args.kwargs["params"] + assert json.loads(params["tags"]) == list(tags) + assert params["tag_match"] == tag_match + assert params["offset"] == 3 + assert params["search"] == "score" + assert page.results[0].tags == ("quality",) + + +@pytest.mark.parametrize("operation", ["create", "update", "list"]) +@pytest.mark.parametrize( + "tags", ["quality", {"quality": True}, [1], [None], [" "], ["x" * 256]] +) +def test_invalid_tag_names_fail_before_scorer_request(scorers, tags, operation): + namespace, transport = scorers + with pytest.raises(ValueError): + if operation == "create": + invoke(namespace.create, name="Score", kind="numeric", config={}, tags=tags) + elif operation == "update": + invoke(namespace.update, SCORER_ID, tags=tags) + else: + invoke(namespace.list, tags=tags) + transport.request.assert_not_called() + + +def test_tag_count_limits_and_null_writes_are_rejected(scorers): + namespace, transport = scorers + for tags in (None, ["quality"] * 101): + with pytest.raises(ValueError): + invoke(namespace.create, name="Score", kind="numeric", config={}, tags=tags) + with pytest.raises(ValueError): + invoke(namespace.update, SCORER_ID, tags=tags) + with pytest.raises(ValueError): + invoke(namespace.list, tags=["quality"] * 21) + with pytest.raises(ValueError): + invoke(namespace.list, tag_match="invalid") + transport.request.assert_not_called() + + def test_version_bump_preserves_configuration_and_concurrency_guard(scorers): namespace, transport = scorers config = NumericScorerConfig(min=0, max=1) diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index 29a1c477e..5a093b650 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -486,6 +486,7 @@ attribute posthog.ai.evaluations.Scorer.description: str attribute posthog.ai.evaluations.Scorer.id: str attribute posthog.ai.evaluations.Scorer.kind: ScorerKind attribute posthog.ai.evaluations.Scorer.name: str +attribute posthog.ai.evaluations.Scorer.tags: tuple[str, ...] = () attribute posthog.ai.evaluations.Scorer.team: int attribute posthog.ai.evaluations.Scorer.updated_at: datetime attribute posthog.ai.evaluations.ScorerConfig = BooleanScorerConfig | NumericScorerConfig | CategoricalScorerConfig @@ -1148,7 +1149,7 @@ class posthog.ai.evaluations.NumericScorerConfig class posthog.ai.evaluations.OfflineEvaluations(*, project_id: int, secret_key: str, host: str | None = None, timeout: float = 15, max_retries: int = 3) class posthog.ai.evaluations.ResultOptions class posthog.ai.evaluations.ResultReceipt(id: str, created: bool, accepted_at: datetime, item_id: str, scorer_version_id: str) -class posthog.ai.evaluations.Scorer(id: str, name: str, description: str, kind: ScorerKind, archived: bool, current_version: int, current_version_id: str, config: _ConfigT, created_at: datetime, updated_at: datetime, created_by: dict[str, Any] | None, team: int) +class posthog.ai.evaluations.Scorer(id: str, name: str, description: str, kind: ScorerKind, archived: bool, current_version: int, current_version_id: str, config: _ConfigT, created_at: datetime, updated_at: datetime, created_by: dict[str, Any] | None, team: int, tags: tuple[str, ...] = ()) class posthog.ai.evaluations.ScorerPage(count: int, results: tuple[Scorer[ScorerConfig], ...], next: str | None, previous: str | None, next_offset: int | None) class posthog.ai.evaluations.ScorerVersion(id: str, definition_id: str, version: int, kind: ScorerKind, config: _ConfigT, created_at: datetime, created_by: dict[str, Any] | None) class posthog.ai.evaluations.ScorerVersionPage(count: int, results: tuple[ScorerVersion[ScorerConfig], ...], next_cursor: str | None) @@ -1502,36 +1503,40 @@ method posthog.ai.claude_agent_sdk.client.PostHogClaudeSDKClient.set_permission_ method posthog.ai.claude_agent_sdk.processor.PostHogClaudeAgentProcessor.query(*, prompt: Any, options: Optional[ClaudeAgentOptions] = None, transport: Any = None, posthog_distinct_id: Optional[Union[str, Callable[[ResultMessage], Optional[str]]]] = None, posthog_trace_id: Optional[str] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_privacy_mode: Optional[bool] = None, posthog_groups: Optional[Dict[str, Any]] = None) method posthog.ai.evaluations.AsyncExperiment.complete() method posthog.ai.evaluations.AsyncExperiment.fail() +method posthog.ai.evaluations.AsyncExperiment.get_tags() +method posthog.ai.evaluations.AsyncExperiment.set_tags(tags: Sequence[str]) method posthog.ai.evaluations.AsyncExperiment.upload_result(*, item: EvaluationItem | str | UUID, scorer_version_id: str | UUID, **options: Unpack[ResultOptions]) method posthog.ai.evaluations.AsyncExperiment.upload_results(results: Sequence[EvaluationResult]) method posthog.ai.evaluations.AsyncOfflineEvaluations.aclose() method posthog.ai.evaluations.AsyncOfflineEvaluations.create_experiment(*, name: str, **options: Unpack[ExperimentOptions]) method posthog.ai.evaluations.AsyncOfflineEvaluations.resume_experiment(id: str | UUID) -method posthog.ai.evaluations.AsyncScorers.create(*, name: str, kind: ScorerKind, config: ScorerConfig, description: str | None | _Omitted = _OMITTED) +method posthog.ai.evaluations.AsyncScorers.create(*, name: str, kind: ScorerKind, config: ScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED) method posthog.ai.evaluations.AsyncScorers.create_version(scorer_id: str | UUID, *, config: _ConfigT, base_version: int | None = None, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED) method posthog.ai.evaluations.AsyncScorers.get(scorer_id: str | UUID) method posthog.ai.evaluations.AsyncScorers.get_version(scorer_id: str | UUID, version_id: str | UUID) -method posthog.ai.evaluations.AsyncScorers.list(*, limit: int = 100, offset: int = 0, search: str | None = None, kind: ScorerKind | None = None, archived: bool | None = None, order_by: ScorerOrder | None = None) +method posthog.ai.evaluations.AsyncScorers.list(*, limit: int = 100, offset: int = 0, search: str | None = None, kind: ScorerKind | None = None, archived: bool | None = None, order_by: ScorerOrder | None = None, tags: Sequence[str] | None = None, tag_match: Literal['any', 'all'] = 'any') method posthog.ai.evaluations.AsyncScorers.list_versions(scorer_id: str | UUID, *, limit: int = 50, cursor: str | None = None) -method posthog.ai.evaluations.AsyncScorers.update(scorer_id: str | UUID, *, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, archived: bool | _Omitted = _OMITTED) +method posthog.ai.evaluations.AsyncScorers.update(scorer_id: str | UUID, *, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, archived: bool | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED) method posthog.ai.evaluations.EvaluationItem.from_dict(data: Mapping[str, Any]) method posthog.ai.evaluations.EvaluationItem.to_dict() method posthog.ai.evaluations.EvaluationResult.from_dict(data: Mapping[str, Any], *, item: EvaluationItem | None = None) method posthog.ai.evaluations.EvaluationResult.to_dict() method posthog.ai.evaluations.Experiment.complete() method posthog.ai.evaluations.Experiment.fail() +method posthog.ai.evaluations.Experiment.get_tags() +method posthog.ai.evaluations.Experiment.set_tags(tags: Sequence[str]) method posthog.ai.evaluations.Experiment.upload_result(*, item: EvaluationItem | str | UUID, scorer_version_id: str | UUID, **options: Unpack[ResultOptions]) method posthog.ai.evaluations.Experiment.upload_results(results: Sequence[EvaluationResult]) method posthog.ai.evaluations.OfflineEvaluations.close() method posthog.ai.evaluations.OfflineEvaluations.create_experiment(*, name: str, **options: Unpack[ExperimentOptions]) method posthog.ai.evaluations.OfflineEvaluations.resume_experiment(id: str | UUID) -method posthog.ai.evaluations.Scorers.create(*, name: str, kind: ScorerKind, config: ScorerConfig, description: str | None | _Omitted = _OMITTED) +method posthog.ai.evaluations.Scorers.create(*, name: str, kind: ScorerKind, config: ScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED) method posthog.ai.evaluations.Scorers.create_version(scorer_id: str | UUID, *, config: _ConfigT, base_version: int | None = None, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED) method posthog.ai.evaluations.Scorers.get(scorer_id: str | UUID) method posthog.ai.evaluations.Scorers.get_version(scorer_id: str | UUID, version_id: str | UUID) -method posthog.ai.evaluations.Scorers.list(*, limit: int = 100, offset: int = 0, search: str | None = None, kind: ScorerKind | None = None, archived: bool | None = None, order_by: ScorerOrder | None = None) +method posthog.ai.evaluations.Scorers.list(*, limit: int = 100, offset: int = 0, search: str | None = None, kind: ScorerKind | None = None, archived: bool | None = None, order_by: ScorerOrder | None = None, tags: Sequence[str] | None = None, tag_match: Literal['any', 'all'] = 'any') method posthog.ai.evaluations.Scorers.list_versions(scorer_id: str | UUID, *, limit: int = 50, cursor: str | None = None) -method posthog.ai.evaluations.Scorers.update(scorer_id: str | UUID, *, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, archived: bool | _Omitted = _OMITTED) +method posthog.ai.evaluations.Scorers.update(scorer_id: str | UUID, *, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, archived: bool | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED) method posthog.ai.gemini.gemini.Models.embed_content(model: str, contents, posthog_distinct_id: Optional[str] = None, posthog_trace_id: Optional[str] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_privacy_mode: Optional[bool] = None, posthog_groups: Optional[Dict[str, Any]] = None, **kwargs: Any) method posthog.ai.gemini.gemini.Models.generate_content(model: str, contents, posthog_distinct_id: Optional[str] = None, posthog_trace_id: Optional[str] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_privacy_mode: Optional[bool] = None, posthog_groups: Optional[Dict[str, Any]] = None, **kwargs: Any) method posthog.ai.gemini.gemini.Models.generate_content_stream(model: str, contents, posthog_distinct_id: Optional[str] = None, posthog_trace_id: Optional[str] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_privacy_mode: Optional[bool] = None, posthog_groups: Optional[Dict[str, Any]] = None, **kwargs: Any)