From 6d28952337293e0b41893594b76de34117972a80 Mon Sep 17 00:00:00 2001 From: Bradford Levy <5950006+bradfordlynch@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:23:19 -0500 Subject: [PATCH] Accept object content in disclosure items; add option-implied statistics helpers From 2026Q3 an event's bundle may carry a third item, `option-implied-stats`, whose content is an object of three option-market statistics. `DisclosureItem.content` was typed `list[str] | str`, so that one item failed validation of the whole bundle, the facts and the preview with it. Anyone who copied the model would have lost every event that carries the item. `content` is now `Any`. The model is documented as deliberately lenient so that new kinds never break parsing, and a narrow annotation on the one field whose JSON type follows `kind` contradicted that. `examples.option_stats` adds a lookup by id and kind, a reader that returns a value only when its status is "ok", and a frame. It documents what the three numbers mean and when they are missing. There is no notebook yet, and the bundled 2026Q3 sample predates the item. --- README.md | 24 +++- notebooks/01_historical_archive.ipynb | 2 +- src/examples/option_stats.py | 159 ++++++++++++++++++++++++++ src/examples/schemas.py | 22 +++- tests/test_option_stats.py | 142 +++++++++++++++++++++++ 5 files changed, 338 insertions(+), 11 deletions(-) create mode 100644 src/examples/option_stats.py create mode 100644 tests/test_option_stats.py diff --git a/README.md b/README.md index c3425c8..f1a1632 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,10 @@ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) Notebook examples for exploring the [Explaining Markets](https://explainingmarkets.ai/) -competition API and its historical data archive — including the two disclosure items -each earnings event carries, the ten call facts and (from 2026Q3) the earnings -preview. Notebook-first, with the repeated +competition API and its historical data archive — including the disclosure items +an earnings event can carry: the ten call facts, and from 2026Q3 the earnings +preview and three option-implied statistics. +Notebook-first, with the repeated API / parsing / plotting boilerplate factored into a small `examples` package so the notebooks stay short and readable. @@ -22,6 +23,21 @@ notebooks stay short and readable. | [`notebooks/02_earnings_call_facts.ipynb`](notebooks/02_earnings_call_facts.ipynb) | Where each event's ten-fact `disclosure` comes from — the exact prompt, model config, and parsing behind it, and how extraction failures surface | | [`notebooks/03_earnings_previews.ipynb`](notebooks/03_earnings_previews.ipynb) | The **earnings preview** — the result of running [Anthropic's Earnings Preview Skill](https://github.com/anthropics/financial-services/blob/main/plugins/agent-plugins/earnings-reviewer/skills/earnings-preview/SKILL.md) prior to each earnings announcement. Starting Q4 2026, events carry this as a second disclosure item. The notebook covers how it is built, what it looks like, and how it pairs with the facts | +### The option-implied statistics + +From 2026Q3, an event may also carry an `option-implied-stats` item (`kind="stats"`), for stocks with liquid listed options: fewer than half of events. Its `content` is an **object**, unlike the facts (a list of strings) and the preview (one string): + +```python +from examples.option_stats import option_implied_stats_from_disclosure, statistic_value + +stats = option_implied_stats_from_disclosure(record) # dict, or None when absent +move = statistic_value(stats, "implied_absolute_earnings_move") # float, or None unless status is "ok" +``` + +`examples.option_stats` documents what the three numbers mean and when they are missing. There is no notebook for it yet, and the bundled 2026Q3 sample predates the item, so run against the live archive to see it. + +If you parse bundles with your own typed model, make sure an item's `content` accepts any JSON type. A model that allows only strings and lists of strings rejects the whole bundle, facts included, the first time it meets this item. `examples.schemas.DisclosureItem` had exactly that annotation until this release. + All four run top-to-bottom. With an API key they hit the live API; **without one they fall back to a small bundled sample** (`data/sample/`), so you can run them — and CI can execute them — with no credentials. @@ -63,7 +79,7 @@ events_frame(events).head() ```text notebooks/ the numbered, narrative examples -src/examples/ helper package: config · client · schemas · frames · plotting · archive · scoring · summary · preview +src/examples/ helper package: config · client · schemas · frames · plotting · archive · scoring · summary · preview · option_stats scripts/ small runnable scripts (e.g. headless archive download) tests/ offline unit tests (mocked HTTP + bundled sample data) data/ diff --git a/notebooks/01_historical_archive.ipynb b/notebooks/01_historical_archive.ipynb index 42ac9ca..6434f98 100644 --- a/notebooks/01_historical_archive.ipynb +++ b/notebooks/01_historical_archive.ipynb @@ -143,7 +143,7 @@ "\n", "Pull a single row and read its focal assets and disclosure. The bundled sample inlines disclosures under a `disclosure` field; the live archive carries additional fields \u2014 inspect `df.columns` to see what a given dataset provides.\n", "\n", - "`disclosure.items[]` can hold **two** items: the ten earnings-call facts (`kind=\"facts\"`, the subject of notebook 02) and, for recent events, the earnings preview (`id=\"earnings-preview\"`, `kind=\"text\"`, section 5). `facts_from_disclosure` and `earnings_preview_from_disclosure` look each one up and return `[]` / `None` when it is absent \u2014 never assume a position in the list." + "`disclosure.items[]` can hold up to **three** items: the ten earnings-call facts (`kind=\"facts\"`, the subject of notebook 02); for recent events, the earnings preview (`id=\"earnings-preview\"`, `kind=\"text\"`, section 5); and, for live events from October 2026, the option-implied statistics (`id=\"option-implied-stats\"`, `kind=\"stats\"`), whose `content` is an object and not text. `facts_from_disclosure`, `earnings_preview_from_disclosure` and `option_implied_stats_from_disclosure` look each one up and return `[]` / `None` when it is absent \u2014 never assume a position in the list, or that every item's content is a string." ] }, { diff --git a/src/examples/option_stats.py b/src/examples/option_stats.py new file mode 100644 index 0000000..3c9d9bb --- /dev/null +++ b/src/examples/option_stats.py @@ -0,0 +1,159 @@ +"""The option-implied statistics: what the options market expected of a release. + +From 2026Q3 onward, an ``EARNINGS_RELEASE`` event's ``disclosure.items[]`` may +carry a third item: ``id="option-implied-stats"`` (``kind="stats"``, +``source="option_market"``, ``media_type="application/json"``). Unlike the facts (a +list of strings) and the preview (one markdown string), its ``content`` is an OBJECT:: + + { + "as_of": "2026-10-05T19:36:12Z", + "methodology": "v1", + "implied_earnings_volatility": {"value": 0.0847, "status": "ok"}, + "implied_absolute_earnings_move": {"value": 0.0676, "status": "ok"}, + "skew_25_delta": {"value": null, "status": "illiquid_wings"} + } + +Every statistic is a ``{value, status}`` pair. ``value`` is a decimal (``0.0847`` is +8.47%) and is ``None`` unless ``status`` is ``"ok"``. Treat any status you do not +recognise as unavailable. + +What the three numbers are: + +``implied_earnings_volatility`` + The risk-neutral standard deviation of the stock's jump on the release. Option + prices before a release embed two things, ordinary volatility and the jump, and + the jump weighs more on a short-dated option than a long-dated one. Comparing + at-the-money implied volatility across the first two expirations after the + release separates them (the term-structure estimator of Dubinsky, Johannes, + Kaeck and Seeger, *Review of Financial Studies*, 2019). + +``implied_absolute_earnings_move`` + The risk-neutral expected absolute return from that jump: ``4 * Phi(J / 2) - 2`` + for volatility ``J``, which is about ``0.8 * J``. It is a magnitude. It says + nothing about direction, and being risk-neutral it is not a forecast of the + realised move. + +``skew_25_delta`` + The 25-delta call implied volatility minus the 25-delta put implied volatility, + on the nearest expiration after the release, in volatility points as a decimal. + Negative means downside puts are richer than upside calls. It is an asymmetry in + option prices, not a predicted return or a probability. + +Timing: ``as_of`` is when the market was observed, and it is always strictly before +the event's ``knowledge_cutoff``. ``methodology`` changes whenever the calculation +does. + +Coverage: the item is absent when a stock has no listed options or they trade too +thinly to measure, which is most smaller companies. Fewer than half of 2026Q3 events +carry it. The skew can be unavailable on its own when only the out-of-the-money +options are thin. Events before 2026Q3 never carry the item. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +import pandas as pd + +OPTION_STATS_ID = "option-implied-stats" +OPTION_STATS_SOURCE = "option_market" +STATS_KIND = "stats" +OK = "ok" + +STATISTICS = ( + "implied_earnings_volatility", + "implied_absolute_earnings_move", + "skew_25_delta", +) + +_COLUMNS = [ + "event_datetime", + "knowledge_cutoff", + "quarter", + "tickers", + "event_id", + "has_option_stats", + "as_of", + "methodology", + *STATISTICS, + "skew_status", +] + + +def option_implied_stats_from_disclosure(record: dict) -> dict | None: + """Pull the option-implied statistics item's content out of an event or archive record. + + Returns the item's ``content`` object as-is, or ``None`` when the record carries no + such item. ``record`` may be a raw archive line, an event payload, or a + ``load_archive`` row turned back into a dict (where a missing ``disclosure`` is + ``NaN``, hence the type checks). + """ + disclosure = record.get("disclosure") + items = disclosure.get("items") if isinstance(disclosure, dict) else None + for item in items if isinstance(items, list) else []: + if ( + isinstance(item, dict) + and item.get("id") == OPTION_STATS_ID + and item.get("kind") == STATS_KIND + ): + content = item.get("content") + return content if isinstance(content, dict) and content else None + return None + + +def statistic_value(stats: dict | None, name: str) -> float | None: + """One statistic's value, or ``None`` unless its status is ``"ok"``. + + Reads defensively: a missing block, an unknown status, or a non-numeric value all + come back as ``None`` rather than raising. + """ + block = (stats or {}).get(name) + if not isinstance(block, dict) or block.get("status") != OK: + return None + value = block.get("value") + if isinstance(value, bool) or not isinstance(value, int | float): + return None + return float(value) + + +def statistic_status(stats: dict | None, name: str) -> str | None: + block = (stats or {}).get(name) + status = block.get("status") if isinstance(block, dict) else None + return status if isinstance(status, str) else None + + +def option_stats_frame(records: Iterable[dict]) -> pd.DataFrame: + """One row per record with the three statistics as plain columns. + + Unavailable statistics are ``NaN``. ``skew_status`` says why the skew is missing + when the volatility is present. + """ + rows = [] + for record in records: + stats = option_implied_stats_from_disclosure(record) + focal = record.get("focal_assets") + rows.append( + { + "event_datetime": record.get("event_datetime"), + "knowledge_cutoff": record.get("knowledge_cutoff"), + "quarter": record.get("quarter"), + "tickers": ", ".join( + a["identifier_value"] for a in (focal if isinstance(focal, list) else []) + ), + "event_id": record.get("event_id"), + "has_option_stats": stats is not None, + "as_of": (stats or {}).get("as_of"), + "methodology": (stats or {}).get("methodology"), + **{name: statistic_value(stats, name) for name in STATISTICS}, + "skew_status": statistic_status(stats, "skew_25_delta"), + } + ) + df = pd.DataFrame(rows, columns=_COLUMNS) + for name in STATISTICS: + # None -> NaN, and a float column even when every value is missing. + df[name] = pd.to_numeric(df[name], errors="coerce").astype("float64") + if not df.empty: + for column in ("event_datetime", "knowledge_cutoff", "as_of"): + df[column] = pd.to_datetime(df[column], utc=True, format="ISO8601") + df = df.sort_values("event_datetime").reset_index(drop=True) + return df diff --git a/src/examples/schemas.py b/src/examples/schemas.py index 95c8bab..6d2fa74 100644 --- a/src/examples/schemas.py +++ b/src/examples/schemas.py @@ -14,6 +14,8 @@ from __future__ import annotations +from typing import Any + from pydantic import BaseModel, ConfigDict @@ -51,11 +53,19 @@ class DisclosureItem(_Model): ``id="earnings-preview"`` (``kind="text"``, ``source="claude_code_web_research"``, ``media_type="text/markdown"``), carries an agent-written pre-release research note as ONE markdown string; it may be absent, and :mod:`examples.preview` - documents where it comes from. ``content``'s JSON type follows - ``kind`` (``facts`` → list of strings, ``text`` → string). All of these are - open string sets, like ``event_type`` — new kinds and sources can appear - without breaking parsing, so match on them rather than assuming a single - item or a position. + documents where it comes from. A third item, ``id="option-implied-stats"`` + (``kind="stats"``, ``source="option_market"``, ``media_type="application/json"``), + carries three option-market statistics as an OBJECT; it may be absent too, and + :mod:`examples.option_stats` documents it. ``content``'s JSON type follows + ``kind`` (``facts`` → list of strings, ``text`` → string, ``stats`` → object). + All of these are open string sets, like ``event_type`` — new kinds and sources + can appear without breaking parsing, so match on them rather than assuming a + single item or a position. + + ``content`` is typed ``Any`` for the same reason: a new ``kind`` can bring a + JSON type this model has never seen, and a narrower annotation would reject + the WHOLE bundle over one item nobody asked for. (It was ``list[str] | str`` + until the ``stats`` kind arrived and did exactly that.) Larger or binary artifacts are referenced rather than inlined, via ``media_type`` / ``url`` / ``bytes`` / ``sha256``. @@ -64,7 +74,7 @@ class DisclosureItem(_Model): id: str | None = None kind: str source: str | None = None - content: list[str] | str | None = None + content: Any = None media_type: str | None = None url: str | None = None bytes: int | None = None diff --git a/tests/test_option_stats.py b/tests/test_option_stats.py new file mode 100644 index 0000000..7495e6e --- /dev/null +++ b/tests/test_option_stats.py @@ -0,0 +1,142 @@ +"""The option-implied-stats disclosure item and its helpers.""" + +from __future__ import annotations + +import math + +import pytest + +from examples.option_stats import ( + OPTION_STATS_ID, + OPTION_STATS_SOURCE, + option_implied_stats_from_disclosure, + option_stats_frame, + statistic_status, + statistic_value, +) +from examples.preview import earnings_preview_from_disclosure +from examples.schemas import Disclosure +from examples.summary import facts_from_disclosure +from tests.test_preview import FACTS_ITEM, PREVIEW_ITEM + +STATS = { + "as_of": "2026-10-05T19:36:12Z", + "methodology": "v1", + "implied_earnings_volatility": {"value": 0.0847, "status": "ok"}, + "implied_absolute_earnings_move": {"value": 0.0676, "status": "ok"}, + "skew_25_delta": {"value": None, "status": "illiquid_wings"}, +} +STATS_ITEM = { + "id": OPTION_STATS_ID, + "kind": "stats", + "source": OPTION_STATS_SOURCE, + "media_type": "application/json", + "content": STATS, +} + + +def record(*items, **fields) -> dict: + return {"disclosure": {"schema_version": "1.0", "items": list(items)}, **fields} + + +def test_disclosure_schema_accepts_an_object_content_item() -> None: + """The regression this module exists for: ``content`` used to be typed + ``list[str] | str``, so one ``stats`` item failed validation of the whole + bundle, facts and preview included.""" + parsed = Disclosure.model_validate( + {"schema_version": "1.0", "items": [FACTS_ITEM, PREVIEW_ITEM, STATS_ITEM]} + ) + assert [i.id for i in parsed.items] == [ + "earnings-call-facts", + "earnings-preview", + OPTION_STATS_ID, + ] + assert parsed.items[0].content == ["f"] + assert parsed.items[1].content == PREVIEW_ITEM["content"] + assert parsed.items[2].content == STATS and parsed.items[2].kind == "stats" + + +@pytest.mark.parametrize( + "content", [{"nested": {"deep": [1, 2]}}, 7, 1.5, True, [1, {"a": 2}], None] +) +def test_disclosure_schema_never_rejects_a_bundle_over_an_unknown_content_type(content) -> None: + parsed = Disclosure.model_validate( + {"items": [FACTS_ITEM, {"id": "future", "kind": "something-new", "content": content}]} + ) + assert parsed.items[0].content == ["f"] and parsed.items[1].content == content + + +def test_stats_are_read_by_id_and_kind_wherever_they_sit() -> None: + assert ( + option_implied_stats_from_disclosure(record(FACTS_ITEM, PREVIEW_ITEM, STATS_ITEM)) == STATS + ) + assert option_implied_stats_from_disclosure(record(STATS_ITEM, FACTS_ITEM)) == STATS + # The other helpers are untouched by the new item. + full = record(FACTS_ITEM, PREVIEW_ITEM, STATS_ITEM) + assert facts_from_disclosure(full) == ["f"] + assert earnings_preview_from_disclosure(full) == PREVIEW_ITEM["content"] + + +def test_stats_are_none_when_absent_or_unusable() -> None: + assert option_implied_stats_from_disclosure(record(FACTS_ITEM, PREVIEW_ITEM)) is None + assert option_implied_stats_from_disclosure({}) is None + assert option_implied_stats_from_disclosure({"disclosure": float("nan")}) is None + assert option_implied_stats_from_disclosure(record({**STATS_ITEM, "content": {}})) is None + assert option_implied_stats_from_disclosure(record({**STATS_ITEM, "content": "8.2%"})) is None + assert option_implied_stats_from_disclosure(record({**STATS_ITEM, "kind": "text"})) is None + + +def test_a_value_counts_only_when_its_status_is_ok() -> None: + assert statistic_value(STATS, "implied_earnings_volatility") == 0.0847 + assert statistic_value(STATS, "skew_25_delta") is None + assert statistic_status(STATS, "skew_25_delta") == "illiquid_wings" + # A status this code has never heard of is unavailable, not an error. + assert statistic_value({"x": {"value": 0.3, "status": "provisional"}}, "x") is None + # Defensive against a malformed block. + for block in ( + {"value": "0.08", "status": "ok"}, + {"value": True, "status": "ok"}, + {"status": "ok"}, + "ok", + None, + ): + assert statistic_value({"x": block}, "x") is None + assert statistic_value(None, "x") is None and statistic_status(None, "x") is None + + +def test_the_move_is_reproducible_from_the_volatility() -> None: + j = statistic_value(STATS, "implied_earnings_volatility") + phi = 0.5 * (1 + math.erf((j / 2) / math.sqrt(2))) + assert statistic_value(STATS, "implied_absolute_earnings_move") == pytest.approx( + 4 * phi - 2, abs=1e-4 + ) + + +def test_option_stats_frame() -> None: + focal = [{"identifier_type": "TICKER", "identifier_value": "NKE"}] + frame = option_stats_frame( + [ + record( + FACTS_ITEM, + event_id="ea_B", + event_datetime="2026-10-06T20:05:00+00:00", + focal_assets=focal, + ), + record( + FACTS_ITEM, + STATS_ITEM, + event_id="ea_A", + event_datetime="2026-10-05T20:05:00+00:00", + knowledge_cutoff="2026-10-05T20:00:00+00:00", + focal_assets=focal, + ), + ] + ) + assert list(frame["event_id"]) == ["ea_A", "ea_B"] # sorted by event time + assert list(frame["has_option_stats"]) == [True, False] + with_stats, without = frame.iloc[0], frame.iloc[1] + assert with_stats["implied_earnings_volatility"] == 0.0847 + assert math.isnan(with_stats["skew_25_delta"]) and with_stats["skew_status"] == "illiquid_wings" + assert with_stats["as_of"] < with_stats["knowledge_cutoff"] + assert math.isnan(without["implied_earnings_volatility"]) and without["tickers"] == "NKE" + assert option_stats_frame([]).empty