feat(llma): support scorer and experiment tags

.sampo/changesets/evaluation-tags.md

@@ -0,0 +1,5 @@
+---
+pypi/posthog: minor
+---
+
+Support tags in synchronous and asynchronous AI observability scorer creation, metadata updates, responses, and list filters. Add `get_tags()` and `set_tags()` to offline experiment handles for reading, replacing, and clearing experiment tags independently of uploads and completion.

examples/example-offline-evaluations/README.md

@@ -11,6 +11,8 @@ Offline evaluations must be enabled for your PostHog project. Use the project's
 | Create experiments, upload results, complete/fail | Personal API key or project secret key with `offline_evaluation_ingestion:write` |
 | Fetch scorers and versions | Personal API key with `llm_analytics:read` |
 | Create/update scorers and create versions | Personal API key with `llm_analytics:write` |
+| Read experiment tags | Personal API key with `evaluation:read` |
+| Replace experiment tags | Personal API key with `evaluation:write` |
 
 A personal key can cover both scorer management and uploads. Project secret keys can upload using a known scorer-version UUID without scorer read access. All operations use the explicit project/environment; scorer names are not unique.
 
@@ -147,6 +149,41 @@ version = evaluations.scorers.get_version(scorer.id, updated.current_version_id)
 
 Scorer creation, version creation, and metadata updates are never automatically retried. If a response is lost, inspect server state before repeating a mutation. Version creation is not made idempotent by `base_version`.
 
+## Tags
+
+Assign tags when creating a scorer, or replace them with a metadata update:
+
+```python
+scorer = evaluations.scorers.create(
+    name="Answer relevance",
+    kind="numeric",
+    config=NumericScorerConfig(min=0, max=1),
+    tags=["quality", "production"],
+)
+print(scorer.tags)  # ("production", "quality")
+scorer = evaluations.scorers.update(scorer.id, tags=["quality"])
+page = evaluations.scorers.list(tags=["quality", "production"], tag_match="all")
+evaluations.scorers.update(scorer.id, tags=[])  # Remove all tags.
+```
+
+Omitting `tags` preserves existing tags on updates. `tags=[]` clears them; `None` is not a valid replacement. Lists and tuples are accepted. The server trims and lowercases names, removes duplicates, and sorts them. Returned tags are immutable tuples. Writes accept up to 100 names, each nonblank and at most 255 characters after trimming.
+
+List filters accept up to 20 tags. `tag_match="any"` (the default) matches at least one selected tag; `"all"` requires every selected tag. An empty filter disables tag filtering. Preserve these filters when requesting subsequent pages.
+
+Experiment tags are separate from the creation/upload declaration. Set them on a newly created or resumed handle, including after completion:
+
+```python
+experiment = evaluations.resume_experiment(experiment_id)
+tags = experiment.set_tags(["quality", "baseline"])
+print(tags)  # ("baseline", "quality")
+print(experiment.get_tags())
+experiment.set_tags([])  # Remove all tags.
+```
+
+Reading and writing experiment tags require the personal-key scopes listed above; an upload-only project secret key cannot manage tags. Tag updates do not change the saved `submission`, lifecycle `receipt`, or scorer versions. They are not retried automatically: after a lost acknowledgment, read the current tags before deciding whether to repeat the replacement, since another caller may have edited them.
+
+The async client exposes the same API: await scorer methods and `experiment.get_tags()` / `experiment.set_tags(...)`.
+
 ## Receipts, errors, and recovery
 
 Upload receipts contain persisted item/result UUIDs, original acceptance times, and `created` flags. `created=False` acknowledges an exact duplicate. A bulk upload returns `chunks`; each chunk commits independently. A malformed or incomplete acknowledgment raises rather than reporting success.

posthog/ai/evaluations/_client.py

@@ -23,6 +23,7 @@
     _timestamp,
     _uuid,
 )
+from ._tags import _response_tags, _tag_names
 from ._transport import AsyncTransport, SyncTransport
 from ._types import (
     BulkUploadError,
@@ -38,6 +39,20 @@
 )
 
 
+def _experiment_tags(data: dict[str, Any], identity: str) -> tuple[str, ...]:
+    try:
+        if _uuid(data["id"]) != identity:
+            raise ValueError("Mismatched experiment ID.")
+        return _response_tags(data["tags"])
+    except (KeyError, TypeError, ValueError):
+        raise EvaluationAPIError(
+            code="invalid_response",
+            detail="The experiment API returned invalid tag metadata.",
+            response=data,
+            persistence="unknown",
+        ) from None
+
+
 def _experiment_body(name: str, options: ExperimentOptions) -> bytes:
     allowed = set(ExperimentOptions.__annotations__)
     if set(options) - allowed:
@@ -179,6 +194,26 @@ def submission(self) -> dict[str, Any] | None:
         """Copy of original creation fields; unavailable on a resumed handle."""
         return None if self._submission is None else json.loads(self._submission)
 
+    def get_tags(self) -> tuple[str, ...]:
+        """Fetch current tags using a personal key with ``evaluation:read``."""
+        with self._lock:
+            data = self._transport.request("GET", self._path)
+            return _experiment_tags(data, self.id)
+
+    def set_tags(self, tags: Sequence[str]) -> tuple[str, ...]:
+        """Replace tags and return the server's normalized names; [] clears them.
+
+        Requires a personal key with ``evaluation:write``. Tags can change after
+        completion and are separate from the immutable creation ``submission``.
+        Updates are not retried automatically: inspect tags after a lost response.
+        """
+        body = _encode({"tags": _tag_names(tags)})
+        with self._lock:
+            data = self._transport.request(
+                "PATCH", self._path, body=body, retry_safe=False
+            )
+            return _experiment_tags(data, self.id)
+
     def upload_result(
         self,
         *,
@@ -289,6 +324,21 @@ def submission(self) -> dict[str, Any] | None:
         """Copy of original creation fields; unavailable on a resumed handle."""
         return None if self._submission is None else json.loads(self._submission)
 
+    async def get_tags(self) -> tuple[str, ...]:
+        """Fetch current tags. See Experiment.get_tags for key requirements."""
+        async with self._lock:
+            data = await self._transport.request("GET", self._path)
+            return _experiment_tags(data, self.id)
+
+    async def set_tags(self, tags: Sequence[str]) -> tuple[str, ...]:
+        """Replace tags. See Experiment.set_tags for permissions and recovery."""
+        body = _encode({"tags": _tag_names(tags)})
+        async with self._lock:
+            data = await self._transport.request(
+                "PATCH", self._path, body=body, retry_safe=False
+            )
+            return _experiment_tags(data, self.id)
+
     async def upload_result(
         self,
         *,

posthog/ai/evaluations/_scorer_types.py

@@ -96,6 +96,7 @@ class Scorer(Generic[_ConfigT]):
     Read ``current_version_id`` once to pin the version for an experiment.
     Names are not unique; use ``id`` for subsequent management operations.
     Configuration dictionaries are local snapshots, not live server state.
+    ``tags`` contains normalized tag names and is independent of versions.
     """
 
     id: str
@@ -110,6 +111,7 @@ class Scorer(Generic[_ConfigT]):
     updated_at: datetime
     created_by: dict[str, Any] | None
     team: int
+    tags: tuple[str, ...] = ()
 
 
 @dataclass(frozen=True)