From 0de46ede9d06e76fc5d2bdd2891a08e37dd7ec8f Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 00:24:29 -0400 Subject: [PATCH 01/15] feat(calendar): model NYSE early closes by rule Co-Authored-By: Claude Opus 5.5 --- .../infrastructure/market_data/sessions.py | 33 +++++++- .../market_data/us_market_calendar.py | 39 ++++++++- .../market_data/test_us_market_calendar.py | 84 +++++++++++++++++++ 3 files changed, 152 insertions(+), 4 deletions(-) create mode 100644 dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py diff --git a/dashboard/backend/infrastructure/market_data/sessions.py b/dashboard/backend/infrastructure/market_data/sessions.py index 41d0a8fee..4b0ea6f0c 100644 --- a/dashboard/backend/infrastructure/market_data/sessions.py +++ b/dashboard/backend/infrastructure/market_data/sessions.py @@ -15,7 +15,7 @@ from __future__ import annotations -from datetime import datetime, time, timedelta +from datetime import date, datetime, time, timedelta from typing import Any, Mapping import pytz @@ -23,6 +23,9 @@ from dashboard.backend.infrastructure.market_data.profiles import ( registered_market_timezones, ) +from dashboard.backend.infrastructure.market_data.us_market_calendar import ( + is_early_close, +) DEFAULT_MARKET = "US" @@ -44,6 +47,34 @@ def session_windows(market: object) -> tuple[tuple[time, time], ...]: return ((time(9, 30), time(16, 0)),) +#: The US cash close on a full session. Read off ``session_windows`` rather +#: than restated, so the bound keeps one owner. +REGULAR_CLOSE_TIME: time = session_windows(DEFAULT_MARKET)[-1][1] +#: The US cash close on an NYSE early-close day +#: (``us_market_calendar.nyse_early_closes``). +EARLY_CLOSE_TIME: time = time(13, 0) + + +def session_close_time(day: date, market: object = DEFAULT_MARKET) -> time: + """When ``market``'s last session ends on ``day``, in its own local time. + + A market on US sessions closes at :data:`EARLY_CLOSE_TIME` on the NYSE + early-close days the calendar lists and at :data:`REGULAR_CLOSE_TIME` + otherwise; any other market closes at its last window's end. Whether + ``day`` trades at all is the calendar's question + (``us_market_calendar.is_trading_day``), not this one's. + + The close-stamped helpers (:func:`time_in_session`, :func:`session_windows`, + :func:`is_session_close`) do not consult it: bar aggregation and the + engine still end every US day at 16:00. That gap is D19 in the US + conformance design, and the engine rewrite owns it. + """ + windows = session_windows(market) + if windows == session_windows(DEFAULT_MARKET) and is_early_close(day): + return EARLY_CLOSE_TIME + return windows[-1][1] + + def timezone_for_market(market: object) -> str: """The timezone a registered profile pairs with ``market``. Unknown -> US. diff --git a/dashboard/backend/infrastructure/market_data/us_market_calendar.py b/dashboard/backend/infrastructure/market_data/us_market_calendar.py index 476ecaf5b..c107c9b11 100644 --- a/dashboard/backend/infrastructure/market_data/us_market_calendar.py +++ b/dashboard/backend/infrastructure/market_data/us_market_calendar.py @@ -1,4 +1,5 @@ -"""NYSE full-day holidays, computed by rule (no data file, no dependency). +"""NYSE full-day holidays and early closes, computed by rule (no data file, +no dependency). Shared US market-data infrastructure: the Live Trading Leaderboard walks it forward day by day, and the Run Backtest modal's default period @@ -11,8 +12,10 @@ freeze day whose one-day increment had no bars and failed every model, and it padded "Day N of M" and the chart axis with a session that never trades. -Covers the ten NYSE holidays in force since 2022 (Juneteenth added that year). -Early closes (13:00 ET) are not modelled; they are still trading days. +Covers the ten NYSE holidays in force since 2022 (Juneteenth added that year) +and the recurring 13:00 ET early closes (``nyse_early_closes``). An early close +is still a trading day. The hour it ends at is a session bound, so it lives +with the other bounds in ``sessions.py`` (``session_close_time``). """ from __future__ import annotations @@ -84,6 +87,36 @@ def is_trading_day(day: date) -> bool: return day.weekday() < 5 and day not in nyse_holidays(day.year) +@lru_cache(maxsize=16) +def nyse_early_closes(year: int) -> FrozenSet[date]: + """NYSE's 13:00 ET early closes in ``year``, by rule. + + - the day after Thanksgiving; + - December 24 when it falls Monday to Thursday (a Friday one is the + observed Christmas holiday, a weekend one is no session); + - July 3 when Independence Day falls Tuesday to Friday (a Saturday July 4 + makes July 3 the observed holiday; a Sunday or Monday one puts July 3 on + a weekend). + + A candidate that is not a trading day is dropped, so this set can never + contradict ``nyse_holidays``. The rules reproduce NYSE's published 2025-2028 + calendars; an unscheduled closure is not predictable by rule, and the Live + board's session-gap check stays the backstop for one. + """ + candidates = {_nth_weekday(year, 11, 3, 4) + timedelta(days=1)} + christmas_eve = date(year, 12, 24) + if christmas_eve.weekday() < 4: + candidates.add(christmas_eve) + if 1 <= date(year, 7, 4).weekday() <= 4: + candidates.add(date(year, 7, 3)) + return frozenset(day for day in candidates if is_trading_day(day)) + + +def is_early_close(day: date) -> bool: + """Whether ``day`` is an NYSE session that ends at 13:00 ET.""" + return day in nyse_early_closes(day.year) + + def trading_weeks_back(day: date) -> Iterator[Tuple[date, date]]: """First and last session of the Mon-Fri week holding ``day``, then of each earlier week, newest first. Unbounded: the caller stops it. diff --git a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py new file mode 100644 index 000000000..3fb7a190c --- /dev/null +++ b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py @@ -0,0 +1,84 @@ +"""NYSE early closes, by rule, pinned to the exchange's published calendars. + +The Live board's 2026-11-27 session is why this exists: the calendar knew the +holidays but not the 13:00 early closes, so that day would have had seven +chart nodes and three model decisions on late-session bars. Sources: the NYSE +Group releases "2025, 2026 and 2027 Holiday and Early Closings Calendar" +(2024-11-08) and "2026, 2027 and 2028 ..." (2025-12-23), and nyse.com +"Holidays & Trading Hours". +""" + +from datetime import date, time + +import pytest + +from dashboard.backend.infrastructure.market_data import sessions +from dashboard.backend.infrastructure.market_data.us_market_calendar import ( + is_early_close, + is_trading_day, + nyse_early_closes, + nyse_holidays, +) + +PUBLISHED_EARLY_CLOSES = { + 2025: ["2025-07-03", "2025-11-28", "2025-12-24"], + 2026: ["2026-11-27", "2026-12-24"], + 2027: ["2027-11-26"], + 2028: ["2028-07-03", "2028-11-24"], +} + + +@pytest.mark.parametrize("year", sorted(PUBLISHED_EARLY_CLOSES)) +def test_early_closes_match_the_nyse_published_calendar(year): + assert sorted(d.isoformat() for d in nyse_early_closes(year)) == ( + PUBLISHED_EARLY_CLOSES[year] + ) + + +def test_a_holiday_or_weekend_is_never_an_early_close(): + # Independence Day 2026 is observed on Friday July 3; Christmas 2027 on + # Friday December 24; December 24, 2028 is a Sunday. + assert date(2026, 7, 3) in nyse_holidays(2026) + assert date(2026, 7, 3) not in nyse_early_closes(2026) + assert date(2027, 12, 24) in nyse_holidays(2027) + assert date(2027, 12, 24) not in nyse_early_closes(2027) + assert date(2028, 12, 24) not in nyse_early_closes(2028) + + +@pytest.mark.parametrize("year", range(2022, 2031)) +def test_an_early_close_is_always_a_trading_day(year): + for day in nyse_early_closes(year): + assert is_trading_day(day), day + + +def test_is_early_close(): + assert is_early_close(date(2026, 11, 27)) + assert is_early_close(date(2026, 12, 24)) + assert not is_early_close(date(2026, 11, 25)) + # Thanksgiving itself is a holiday, not an early close. + assert not is_early_close(date(2026, 11, 26)) + + +def test_the_close_times_have_one_owner(): + assert sessions.EARLY_CLOSE_TIME == time(13, 0) + assert sessions.REGULAR_CLOSE_TIME == time(16, 0) + assert sessions.REGULAR_CLOSE_TIME == sessions.session_windows("US")[-1][1] + + +@pytest.mark.parametrize( + ("day", "market", "close"), + [ + (date(2026, 11, 27), "US", time(13, 0)), + (date(2026, 12, 24), "us", time(13, 0)), + (date(2026, 11, 25), "US", time(16, 0)), + (date(2026, 11, 30), None, time(16, 0)), + # A-shares keep their own close; the NYSE calendar is not theirs. + (date(2026, 11, 27), "CN", time(15, 0)), + ], +) +def test_session_close_time(day, market, close): + assert sessions.session_close_time(day, market) == close + + +def test_session_close_time_defaults_to_the_us_market(): + assert sessions.session_close_time(date(2026, 11, 27)) == time(13, 0) From a2613c3f40b3d33a19a79f12fe9a9d5912695772 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 00:28:14 -0400 Subject: [PATCH 02/15] fix(sessions): end the open-stamped US session at 13:00 on NYSE early closes Co-Authored-By: Claude Opus 5.5 --- .../infrastructure/market_data/sessions.py | 12 +- .../backend/tests/test_market_sessions.py | 132 ++++++++++++++++++ 2 files changed, 143 insertions(+), 1 deletion(-) diff --git a/dashboard/backend/infrastructure/market_data/sessions.py b/dashboard/backend/infrastructure/market_data/sessions.py index 4b0ea6f0c..33edea3a4 100644 --- a/dashboard/backend/infrastructure/market_data/sessions.py +++ b/dashboard/backend/infrastructure/market_data/sessions.py @@ -170,6 +170,12 @@ def is_in_session( open, and that 09:00 bar is out too: its open, high, low and volume include half an hour of pre-market trading, so admitting it would price the day's first fill off a pre-market print. + + The day's last session ends at :func:`session_close_time`: 13:00 ET on an + NYSE early close, where Alpaca still serves the late session as bars + stamped 13:00, 14:00 and 15:00. Only this open-stamped rule reads the + day's close; the close-stamped rule (:func:`time_in_session`) still ends + every US day at 16:00 (D19, owned by the engine rewrite). """ local = market_local(timestamp, timezone) if open_stamped_minutes is None: @@ -179,9 +185,13 @@ def is_in_session( return False open_minute = local.time().replace(second=0, microsecond=0) close_minute = close.time().replace(second=0, microsecond=0) + windows = session_windows(market) + windows = windows[:-1] + ( + (windows[-1][0], session_close_time(local.date(), market)), + ) return any( start <= open_minute and close_minute <= end - for start, end in session_windows(market) + for start, end in windows ) diff --git a/dashboard/backend/tests/test_market_sessions.py b/dashboard/backend/tests/test_market_sessions.py index 09c12e3ba..220006ac3 100644 --- a/dashboard/backend/tests/test_market_sessions.py +++ b/dashboard/backend/tests/test_market_sessions.py @@ -177,6 +177,138 @@ def test_open_stamped_hourly_day_is_six_bars(): assert kept == ["10:00", "11:00", "12:00", "13:00", "14:00", "15:00"] +def _open_stamped_kept(stamps, minutes=60): + return [ + ts.strftime("%H:%M") for ts in stamps + if sessions.is_in_session( + ts, market="US", timezone="US/Eastern", open_stamped_minutes=minutes + ) + ] + + +def test_open_stamped_hourly_early_close_is_three_bars(): + """2026-11-27 closes at 13:00 ET. Alpaca serves the late session as bars + stamped 13:00, 14:00 and 15:00, and a fixed 16:00 close kept them: three + decisions priced off thin after-hours prints.""" + day = pd.date_range("2026-11-27 04:00", "2026-11-27 19:00", freq="h", + tz="US/Eastern") + assert _open_stamped_kept(day) == ["10:00", "11:00", "12:00"] + + +def test_the_day_before_thanksgiving_is_a_full_session(): + day = pd.date_range("2026-11-25 04:00", "2026-11-25 19:00", freq="h", + tz="US/Eastern") + assert _open_stamped_kept(day) == [ + "10:00", "11:00", "12:00", "13:00", "14:00", "15:00", + ] + + +def test_open_stamped_five_minute_bars_stop_at_the_early_close(): + bars = pd.date_range("2026-12-24 12:45", "2026-12-24 13:15", freq="5min", + tz="US/Eastern") + assert _open_stamped_kept(bars, minutes=5) == ["12:45", "12:50", "12:55"] + + +def test_the_early_close_is_read_on_the_market_local_date(): + """18:00Z on 2026-11-27 is 13:00 ET, after that day's close; a naive stamp + is market-local, as everywhere else in this module.""" + def kept(ts): + return sessions.is_in_session( + ts, market="US", timezone="US/Eastern", open_stamped_minutes=60 + ) + + assert kept(pd.Timestamp("2026-11-27 17:00", tz="UTC")) + assert not kept(pd.Timestamp("2026-11-27 18:00", tz="UTC")) + assert not kept(pd.Timestamp("2026-11-27 13:00")) + + +def test_an_early_close_leaves_the_close_stamped_path_alone(): + """Bar aggregation and the engine's close-stamped filter still end every US + day at 16:00. That is D19, measured by conformance T05 and owned by the + engine rewrite; only the open-stamped rule reads the day's close.""" + assert sessions.session_windows("US") == ((clock_time(9, 30), clock_time(16, 0)),) + assert sessions.is_in_session( + pd.Timestamp("2026-11-27 15:00", tz="US/Eastern"), + market="US", + timezone="US/Eastern", + ) + assert sessions.is_session_close( + pd.Timestamp("2026-11-27 16:00", tz="US/Eastern"), + market="US", + timezone="US/Eastern", + ) + + +def test_cn_open_stamped_sessions_ignore_the_nyse_calendar(): + day = pd.date_range("2026-11-27 09:00", "2026-11-27 15:00", freq="h", + tz="Asia/Shanghai") + kept = [ + ts.strftime("%H:%M") for ts in day + if sessions.is_in_session( + ts, market="CN", timezone="Asia/Shanghai", open_stamped_minutes=60 + ) + ] + assert kept == ["10:00", "13:00", "14:00"] + + +def test_every_us_filter_drops_the_late_session_on_an_early_close(): + """The board's model, its Alpaca baselines, a My Agents backtest and a + protocol run all filter through ``is_in_session``. One rule for all of them + is what keeps an agent and its baseline on the same bars on a half day.""" + from dashboard.backend import baseline_generator + from dashboard.backend.domain.backtesting import market_data_store as mds + from dashboard.backend.domain.backtesting.engine import HourlyBacktester + from dashboard.backend.domain.leaderboard.strategies import _common + + stamps = list( + pd.date_range("2026-11-27 04:00", "2026-11-27 19:00", freq="h", + tz="US/Eastern").tz_convert("UTC") + ) + frame = pd.DataFrame({"close": 1.0}, index=pd.DatetimeIndex(stamps)) + frame.attrs[sessions.FRAME_ATTR_OPEN_STAMPED_MINUTES] = 60 + expected = [ + ts for ts in stamps if ts.tz_convert("US/Eastern").hour in (10, 11, 12) + ] + engine = HourlyBacktester.__new__(HourlyBacktester) + engine.profile = profiles.get_market_profile(profiles.ALPACA) + + assert baseline_generator._market_hours_only( + stamps, "US/Eastern", {"X": frame} + ) == expected + assert mds._build_trading_timestamps( + {"X": frame}, market="US", timezone="US/Eastern" + ) == expected + assert _common.market_timestamps({"X": frame}) == expected + assert engine._market_hours_only(stamps, {"X": frame}) == expected + + +def test_minute_mode_stops_filling_at_an_early_close(): + """My Agents' minute mode values and fills on open-stamped 5m source bars, + which now stop at 12:55 on 2026-11-27. The close-stamped aggregation still + forms 13:30-16:00 buckets from late-session prints (D19), but none of them + fills, so none is a step: nothing trades off a post-close print.""" + from dashboard.backend.domain.backtesting.engine import HourlyBacktester + + source = pd.date_range("2026-11-27 09:30", "2026-11-27 15:55", freq="5min", + tz="US/Eastern").tz_convert("UTC") + frame = pd.DataFrame({"close": 1.0}, index=source) + frame.attrs[sessions.FRAME_ATTR_OPEN_STAMPED_MINUTES] = 5 + engine = HourlyBacktester.__new__(HourlyBacktester) + engine.profile = profiles.get_market_profile(profiles.ALPACA) + engine.intraday_mode = True + engine.source_timeframe = "5m" + engine.source_data = {"X": frame} + decisions = [ + pd.Timestamp(f"2026-11-27 {clock}", tz="US/Eastern").tz_convert("UTC") + for clock in ("10:30", "11:30", "12:30", "13:30", "14:30", "15:30", "16:00") + ] + steps, valuation, _fills = engine._plan_executions(decisions) + assert [ts.tz_convert("US/Eastern").strftime("%H:%M") for ts in steps] == [ + "10:30", "11:30", "12:30", + ] + assert max(valuation).tz_convert("US/Eastern").strftime("%H:%M") == "12:55" + + def test_session_close_is_each_window_end(): assert sessions.is_session_close(_et("16:00"), market="US", timezone="US/Eastern") assert not sessions.is_session_close( From aa016faa6c02bc6d60d70ebb9ce10b809ec8791b Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 00:31:13 -0400 Subject: [PATCH 03/15] fix(live): per-day cash close on the live axis, clock and node mapping Co-Authored-By: Claude Opus 5.5 --- dashboard/backend/domain/leaderboard/live.py | 69 ++++-- .../tests/test_frontend_live_leaderboard.py | 7 + .../backend/tests/test_live_leaderboard.py | 234 ++++++++++++++++++ dashboard/frontend/app.html | 2 +- 4 files changed, 286 insertions(+), 26 deletions(-) diff --git a/dashboard/backend/domain/leaderboard/live.py b/dashboard/backend/domain/leaderboard/live.py index fd22f774d..214e33cbd 100644 --- a/dashboard/backend/domain/leaderboard/live.py +++ b/dashboard/backend/domain/leaderboard/live.py @@ -3,8 +3,9 @@ This board is not Daily stretched to 30 days and not the fixed contest window. One continuous paper-trading month per entry: the chart axis is the current America/New_York calendar month, hourly nodes are the closes of the US cash -session's hourly bars (10:00–16:00 ET, NYSE trading days), and points after -the freeze close stay empty so the line cannot interpolate into the future. +session's hourly bars (10:00–16:00 ET on NYSE trading days, 10:00–13:00 on +NYSE early closes), and points after the freeze close stay empty so the line +cannot interpolate into the future. Every curve is written by ``refresh_live_leaderboard`` (the cron hook or the refresh script), never on a public GET: GET only reads the latest stored @@ -39,21 +40,26 @@ allow_recent_sip, sip_delay_minutes, ) -from dashboard.backend.infrastructure.market_data.sessions import session_windows +from dashboard.backend.infrastructure.market_data.sessions import ( + session_close_time, + session_windows, +) from dashboard.backend.infrastructure.market_data.us_market_calendar import is_trading_day from dashboard.backend.paths import DATA_DIR import dashboard.backend.domain.leaderboard.service as lb_service _US_EASTERN = ZoneInfo("America/New_York") # The US cash session has one owner (market_data/sessions.py); restating it here -# is how the board's copies drifted apart before #529. -_US_CASH_OPEN, _US_CASH_CLOSE = session_windows("US")[0] +# is how the board's copies drifted apart before #529. The open is fixed; the +# close is per day (``_cash_close``), because an NYSE early close ends at 13:00. +_US_CASH_OPEN = session_windows("US")[0][0] # Axis nodes are hourly bar *closes*. Since #529 the board's Alpaca bars are # open-stamped (09:00 … 15:00, each closing an hour later) and Yahoo's index # bars are open-stamped on the half hour (09:30 … 15:30), so a stored point is -# placed at the hour its bar closed in (see ``_axis_node_key``): seven nodes a -# day, the last one holding the 16:00 close. -_RTH_HOURS = (10, 11, 12, 13, 14, 15, 16) +# placed at the hour its bar closed in (see ``_axis_node_key``). The first node +# is the 09:00 bar's 10:00 close and the last is the day's cash close: seven +# nodes on a full session, four (10:00 … 13:00) on an early close. +_FIRST_NODE_HOUR = _US_CASH_OPEN.hour + 1 _BAR_MINUTES = 60 # A session joins the freeze only once its closing bar is final on the tape the # board is priced off. A Basic plan clamps a SIP request to now − delay, so a @@ -86,18 +92,28 @@ _live_refresh_running = False +def _cash_close(day: date) -> datetime: + """``day``'s US cash close in New York: 16:00, or 13:00 on an NYSE early close.""" + return datetime.combine(day, session_close_time(day, "US"), tzinfo=_US_EASTERN) + + +def _axis_hours(day: date) -> Tuple[int, ...]: + """Hours of ``day``'s axis nodes: 10 … 16, or 10 … 13 on an early close. + + NYSE closes on the hour, so the last node is the close itself. + """ + return tuple(range(_FIRST_NODE_HOUR, _cash_close(day).hour + 1)) + + def _coerce_as_of_eastern(as_of: Optional[Union[date, datetime]] = None) -> datetime: + """``as_of`` in New York; a bare date means that day's cash close.""" if as_of is None: return datetime.now(_US_EASTERN) if isinstance(as_of, datetime): if as_of.tzinfo is None: return as_of.replace(tzinfo=_US_EASTERN) return as_of.astimezone(_US_EASTERN) - return datetime( - as_of.year, as_of.month, as_of.day, - _US_CASH_CLOSE.hour, _US_CASH_CLOSE.minute, - tzinfo=_US_EASTERN, - ) + return _cash_close(as_of) def live_month_dates(as_of: Optional[Union[date, datetime]] = None) -> Tuple[str, str]: @@ -121,12 +137,15 @@ def _trading_days_inclusive(start: date, end: date) -> List[date]: def live_month_hourly_axis(start_date: str, end_date: str) -> List[str]: - """Hourly bar-close ISO timestamps covering the month's NYSE trading days.""" + """Hourly bar-close ISO timestamps covering the month's NYSE trading days. + + Seven per full session (10:00 … 16:00), four per early close (10:00 … 13:00). + """ start = date.fromisoformat(start_date) end = date.fromisoformat(end_date) out: List[str] = [] for day in _trading_days_inclusive(start, end): - for hour in _RTH_HOURS: + for hour in _axis_hours(day): ts = datetime(day.year, day.month, day.day, hour, 0, tzinfo=_US_EASTERN) out.append(ts.isoformat()) return out @@ -178,7 +197,7 @@ def live_increment_bounds( def freeze_settle_time(day: date) -> datetime: """When ``day``'s closing bar is final on the board's tape.""" delay = 0 if allow_recent_sip() else sip_delay_minutes() - close = datetime.combine(day, _US_CASH_CLOSE, tzinfo=_US_EASTERN) + close = _cash_close(day) return close + timedelta(minutes=delay + _FREEZE_SETTLE_MARGIN_MINUTES) @@ -204,7 +223,7 @@ def live_clock(as_of: Optional[Union[date, datetime]] = None) -> Dict[str, Any]: session_state = "preopen" frozen_through = _previous_trading_day(today) live_day = None - elif now.time() < _US_CASH_CLOSE: + elif now.time() < session_close_time(today, "US"): session_state = "rth" frozen_through = _previous_trading_day(today) live_day = today @@ -1215,13 +1234,14 @@ def _axis_node_key(bar_open: datetime) -> Optional[str]: Alpaca's 09:00 bar (09:30 open → 10:00) lands on 10:00 and its 15:00 bar on the 16:00 close; Yahoo's 09:30 bar closes 10:30 and lands on 10:00, and its - 15:30 half-bar is capped at the 16:00 close. A bar opening at or after the - close is after-hours and has no node. + 15:30 half-bar is capped at the 16:00 close. On an NYSE early close the cap + is 13:00, where Alpaca's 12:00 bar and Yahoo's 12:30 half-bar both land. A + bar opening at or after the day's close is after-hours and has no node. """ - if bar_open.time() >= _US_CASH_CLOSE: + session_close = _cash_close(bar_open.date()) + if bar_open >= session_close: return None close = bar_open + timedelta(minutes=_BAR_MINUTES) - session_close = datetime.combine(close.date(), _US_CASH_CLOSE, tzinfo=_US_EASTERN) if close > session_close: close = session_close return f"{close.date().isoformat()}T{close.hour:02d}:00" @@ -1251,9 +1271,7 @@ def reindex_frozen_curve( if key is not None: by_hour[key] = float(equity) * scale - freeze_dt = datetime.combine( - date.fromisoformat(freeze_end), _US_CASH_CLOSE, tzinfo=_US_EASTERN - ) + freeze_dt = _cash_close(date.fromisoformat(freeze_end)) last: Optional[float] = None out: List[Dict[str, Any]] = [] for ts in axis: @@ -1523,7 +1541,8 @@ def get_live_leaderboard( "description": ( f"Live month {month_label}. Axis is the calendar month's NYSE " "trading days; each node is the close of an hourly US cash-session " - "bar (10:00–16:00 America/New_York). " + "bar (10:00–16:00 America/New_York; 10:00–13:00 on NYSE early-close " + "days). " f"Frozen history is the hourly backtest through {clock['frozen_through']}; " "a session is appended once, after it closes and settles. " "Each freeze is stored as a monthly snapshot in agent_runs " diff --git a/dashboard/backend/tests/test_frontend_live_leaderboard.py b/dashboard/backend/tests/test_frontend_live_leaderboard.py index 8dc381ed9..b4f4a6c6f 100644 --- a/dashboard/backend/tests/test_frontend_live_leaderboard.py +++ b/dashboard/backend/tests/test_frontend_live_leaderboard.py @@ -68,6 +68,13 @@ def test_live_chart_does_not_span_empty_future_with_fake_values(): assert "2026-04-15" not in source +def test_live_chart_hint_names_the_early_close(): + """The axis ends at 13:00 on an NYSE early close; the caption under the + chart must not promise a 16:00 node that day.""" + view = _APP_HTML.split('id="liveLeaderboardView"')[1].split('id="leaderboardView"')[0] + assert "10:00–13:00 on early closes" in view + + def _y_bounds_source() -> str: start = _LIVE_JS.index("const LIVE_Y_PAD_RATIO") end = _LIVE_JS.index("function renderLiveChart") diff --git a/dashboard/backend/tests/test_live_leaderboard.py b/dashboard/backend/tests/test_live_leaderboard.py index 56f5a0c7f..efe7f22c1 100644 --- a/dashboard/backend/tests/test_live_leaderboard.py +++ b/dashboard/backend/tests/test_live_leaderboard.py @@ -246,6 +246,240 @@ def test_nyse_holiday_rules(): assert date(2022, 1, 1) not in nyse_holidays(2022) +# ── NYSE early closes (13:00 ET) ──────────────────────────────────────────── + + +@pytest.fixture +def sip_delay_15(monkeypatch): + monkeypatch.delenv("ALPACA_ALLOW_RECENT_SIP", raising=False) + monkeypatch.setenv("ALPACA_SIP_DELAY_MINUTES", "15") + + +def _node_clocks(axis, day): + return [ts[11:16] for ts in axis if ts.startswith(day)] + + +def test_hourly_axis_ends_at_13_00_on_an_early_close(): + """2026-11-27 closes at 13:00: four nodes (10:00 carries the prior close, + 11:00-13:00 hold the 10:00-12:00 bars), not seven with three phantoms.""" + from dashboard.backend.infrastructure.market_data.us_market_calendar import ( + is_early_close, + ) + + axis = live.live_month_hourly_axis("2026-11-01", "2026-11-30") + days = live._trading_days_inclusive(date(2026, 11, 1), date(2026, 11, 30)) + assert len(days) == 20 + assert [day for day in days if is_early_close(day)] == [date(2026, 11, 27)] + assert len(axis) == 19 * 7 + 4 + assert _node_clocks(axis, "2026-11-27") == ["10:00", "11:00", "12:00", "13:00"] + assert _node_clocks(axis, "2026-11-25") == [ + "10:00", "11:00", "12:00", "13:00", "14:00", "15:00", "16:00", + ] + assert _node_clocks(axis, "2026-11-26") == [] + + +def test_december_axis_has_one_early_close(): + axis = live.live_month_hourly_axis("2026-12-01", "2026-12-31") + days = live._trading_days_inclusive(date(2026, 12, 1), date(2026, 12, 31)) + assert len(days) == 22 + assert len(axis) == 21 * 7 + 4 + assert _node_clocks(axis, "2026-12-24") == ["10:00", "11:00", "12:00", "13:00"] + assert axis[-1].startswith("2026-12-31T16:00") + + +def test_an_early_close_settles_30_minutes_after_13_00(sip_delay_15): + assert live.freeze_settle_time(date(2026, 11, 27)) == datetime( + 2026, 11, 27, 13, 30, tzinfo=_ET + ) + assert live.freeze_settle_time(date(2026, 11, 25)) == datetime( + 2026, 11, 25, 16, 30, tzinfo=_ET + ) + + +@pytest.mark.parametrize( + ("clock", "state", "frozen_through", "live_day"), + [ + ((12, 59), "rth", "2026-11-25", "2026-11-27"), + ((13, 5), "settling", "2026-11-25", "2026-11-27"), + ((13, 30), "closed", "2026-11-27", None), + ((15, 0), "closed", "2026-11-27", None), + ], +) +def test_the_clock_closes_at_13_00_on_an_early_close( + sip_delay_15, clock, state, frozen_through, live_day +): + """Thanksgiving (11-26) is skipped: before the half day settles, the + freeze is still Wednesday's.""" + status = live.live_clock(datetime(2026, 11, 27, *clock, tzinfo=_ET)) + assert ( + status["session_state"], status["frozen_through"], status["live_day"] + ) == (state, frozen_through, live_day) + + +def test_a_bare_date_means_that_days_cash_close(): + assert live._coerce_as_of_eastern(date(2026, 11, 27)) == datetime( + 2026, 11, 27, 13, 0, tzinfo=_ET + ) + assert live._coerce_as_of_eastern(date(2026, 11, 25)) == datetime( + 2026, 11, 25, 16, 0, tzinfo=_ET + ) + + +def test_axis_node_key_caps_at_the_early_close(): + def key(hour, minute=0, day=27): + return live._axis_node_key(datetime(2026, 11, day, hour, minute, tzinfo=_ET)) + + assert key(12) == "2026-11-27T13:00" # Alpaca's last cash-session bar + assert key(12, 30) == "2026-11-27T13:00" # Yahoo's 12:30-13:00 half bar + assert key(13) is None # the late session has no node + assert key(15) is None + assert key(15, day=25) == "2026-11-25T16:00" + + +def test_frozen_curve_through_an_early_close_stops_at_13_00(): + """Open-stamped bars 10:00-15:00 ET (15:00Z-20:00Z in EST): the three + late-session prints must not paint 14:00-16:00 nodes.""" + axis = live.live_month_hourly_axis("2026-11-01", "2026-11-30") + points = [ + _pt(f"2026-11-27T{15 + i}:00:00+00:00", SEED + 100 * i) for i in range(6) + ] + curve = live.reindex_frozen_curve(points, axis, "2026-11-27", SEED) + assert len(curve) == 18 * 7 + 4 + assert [ + (p["timestamp"][11:16], p["equity"]) + for p in curve if p["timestamp"].startswith("2026-11-27") + ] == [ + ("10:00", SEED), ("11:00", SEED), ("12:00", SEED + 100), ("13:00", SEED + 200), + ] + + +def test_the_window_description_names_the_early_close(no_alpaca): + payload = live.get_live_leaderboard(as_of=datetime(2026, 11, 27, 17, 39, tzinfo=_ET)) + assert "10:00–13:00 on NYSE early-close days" in payload["window"]["description"] + + +def test_the_live_board_keeps_no_fixed_session_close(): + """A module-level 16:00 close and a fixed seven-hour axis are how the early + close slipped through: every reader asks for the day's close instead.""" + assert not hasattr(live, "_US_CASH_CLOSE") + assert not hasattr(live, "_RTH_HOURS") + + +def _alpaca_hourly_bars(*days): + """AAPL bars shaped like Alpaca's: open-stamped hourly, 04:00-19:00 ET, so + the late session after an early close is in the response.""" + import pandas as pd + + from dashboard.backend.infrastructure.market_data.sessions import ( + FRAME_ATTR_OPEN_STAMPED_MINUTES, + ) + + index = pd.DatetimeIndex([ + ts + for day in days + for ts in pd.date_range(f"{day} 04:00", f"{day} 19:00", freq="h", + tz="US/Eastern") + ]).tz_convert("UTC") + frame = pd.DataFrame( + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0, "volume": 1000}, + index=index, + ) + frame.attrs[FRAME_ATTR_OPEN_STAMPED_MINUTES] = 60 + return {"AAPL": frame} + + +def test_a_replay_across_thanksgiving_and_the_early_close_is_not_a_gap(monkeypatch): + """The real decision loop over 11-25 (full), 11-26 (holiday) and 11-27 + (13:00 close): six decisions then three, a checkpoint per session, no + LiveSessionGapError, and four nodes on the half day.""" + from dashboard.backend.domain.leaderboard.strategies import llm_agent + + decided = [] + + class Manager: + def __init__(self, **kwargs): + self.equity_history = [] + self.trades = [] + self.llm_calls = self.llm_decisions = 0 + self.input_tokens = self.output_tokens = 0 + + def restore_state(self, snapshot): + pass + + def snapshot_state(self): + return {"cash": SEED, "positions": {}} + + def get_portfolio_state(self, *args): + return {} + + def make_trading_decision_with_llm(self, state, client, **kwargs): + decided.append(state["timestamp"]) + self.llm_calls += 1 + self.llm_decisions += 1 + return {"actions": []} + + def execute_actions(self, *args): + pass + + def update_equity(self, market_data, price_cache, ts): + equity = SEED + 1000 * (len(self.equity_history) + 1) + self.equity_history.append( + {"timestamp": ts, "equity": equity, "cash": equity, "positions_value": 0} + ) + + def get_equity_curve(self): + return self.equity_history + + monkeypatch.setattr(llm_agent, "PortfolioManager", Manager) + monkeypatch.setattr(llm_agent.LLMAgentStrategy, "_make_client", lambda self: object()) + monkeypatch.setattr( + lb_service, + "fetch_hourly_bars", + lambda symbols, start, end: _alpaca_hourly_bars("2026-11-25", "2026-11-27"), + ) + _insert_live_run( + "lb_deepseek_v4_pro_20261101_20261124", "deepseek_v4_pro", "2026-11-24", + [_pt("2026-11-24T20:00:00+00:00", SEED)], + start_date="2026-11-01", + metadata={live.LIVE_SNAPSHOT_KEY: {"cash": SEED, "positions": {}}}, + ) + as_of = datetime(2026, 11, 27, 17, 39, tzinfo=_ET) + freeze_cfg = live.live_freeze_config(as_of) + assert freeze_cfg["end_date"] == "2026-11-27" + + row = live.deploy_live_model_increment(_deepseek(freeze_cfg), freeze_cfg) + + assert row["segment"] == {"start_date": "2026-11-25", "end_date": "2026-11-27"} + by_day = {} + for ts in decided: + et = ts.tz_convert("US/Eastern") + by_day.setdefault(et.date().isoformat(), []).append(et.strftime("%H:%M")) + assert by_day == { + "2026-11-25": ["10:00", "11:00", "12:00", "13:00", "14:00", "15:00"], + "2026-11-27": ["10:00", "11:00", "12:00"], + } + stored = db.get_run(row["run_id"]) + assert stored["end_date"] == "2026-11-27" + assert stored["llm_calls"] == 9 + assert [r["end_date"] for r in _month_rows("deepseek_v4_pro")] == ["2026-11-27"] + + payload = live.get_live_leaderboard(as_of=as_of) + assert _node_clocks(payload["chart_axis"], "2026-11-27") == [ + "10:00", "11:00", "12:00", "13:00", + ] + deepseek = next(e for e in payload["entries"] if e["entry_id"] == "deepseek_v4_pro") + assert [ + (p["timestamp"][11:16], p["equity"]) + for p in deepseek["equity_curve"] if p["timestamp"].startswith("2026-11-27") + ] == [ + ("10:00", SEED + 6000), + ("11:00", SEED + 7000), + ("12:00", SEED + 8000), + ("13:00", SEED + 9000), + ] + assert deepseek["portfolio_value"] == SEED + 9000 + + def test_payload_without_runs_keeps_empty_curves(no_alpaca): payload = live.get_live_leaderboard( as_of=datetime(2026, 8, 27, 14, 0, tzinfo=_ET) diff --git a/dashboard/frontend/app.html b/dashboard/frontend/app.html index b51b62d89..5b6a085a3 100644 --- a/dashboard/frontend/app.html +++ b/dashboard/frontend/app.html @@ -1705,7 +1705,7 @@

Live Trading Leaderboard

-
Hourly bar closes · US cash session 10:00–16:00 ET, NYSE trading days · future days stay empty
+
Hourly bar closes · US cash session 10:00–16:00 ET (10:00–13:00 on early closes), NYSE trading days · future days stay empty
From 3801e0b6fac314cb5997dd122a4888e78a2d966a Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 00:32:07 -0400 Subject: [PATCH 04/15] test(live): pin six decisions per full session and three per early close Co-Authored-By: Claude Opus 5.5 --- .../leaderboard/test_llm_agent_cadence.py | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 dashboard/backend/tests/domain/leaderboard/test_llm_agent_cadence.py diff --git a/dashboard/backend/tests/domain/leaderboard/test_llm_agent_cadence.py b/dashboard/backend/tests/domain/leaderboard/test_llm_agent_cadence.py new file mode 100644 index 000000000..ea4ba3ddb --- /dev/null +++ b/dashboard/backend/tests/domain/leaderboard/test_llm_agent_cadence.py @@ -0,0 +1,149 @@ +"""The Live board's decision cadence, pinned (atl-direction spec §6, §7). + +Six model decisions per full session, on the open-stamped hourly bars 10:00 to +15:00 ET, and three on an NYSE early close (10:00 to 12:00). The budget in spec +§7 (126 calls for a 21-session month) is this number times the calendar, so a +filter change that adds or drops one bar a day moves the bill by a sixth. +Nothing here is new behaviour: these stop the cadence drifting. +""" + +from collections import Counter +from datetime import date + +import pandas as pd +import pytest + +from dashboard.backend.domain.leaderboard import live +from dashboard.backend.domain.leaderboard.strategies import _common, llm_agent +from dashboard.backend.infrastructure.market_data.sessions import ( + FRAME_ATTR_OPEN_STAMPED_MINUTES, +) +from dashboard.backend.infrastructure.market_data.us_market_calendar import ( + is_early_close, +) + +FULL_SESSION = ["10:00", "11:00", "12:00", "13:00", "14:00", "15:00"] +EARLY_CLOSE = ["10:00", "11:00", "12:00"] + + +def _alpaca_hourly(days): + """Open-stamped hourly bars, 04:00-19:00 ET on each day: Alpaca serves + extended hours, so the late session after an early close is present.""" + index = pd.DatetimeIndex([ + ts + for day in days + for ts in pd.date_range(f"{day} 04:00", f"{day} 19:00", freq="h", + tz="US/Eastern") + ]).tz_convert("UTC") + frame = pd.DataFrame({"close": 100.0}, index=index) + frame.attrs[FRAME_ATTR_OPEN_STAMPED_MINUTES] = 60 + return {"AAPL": frame} + + +def _clocks_by_day(stamps): + days = {} + for ts in stamps: + et = ts.tz_convert("US/Eastern") + days.setdefault(et.date(), []).append(et.strftime("%H:%M")) + return days + + +def test_six_decision_bars_on_a_full_session_and_three_on_an_early_close(): + stamps = _common.market_timestamps(_alpaca_hourly(["2026-11-25", "2026-11-27"])) + assert _clocks_by_day(stamps) == { + date(2026, 11, 25): FULL_SESSION, + date(2026, 11, 27): EARLY_CLOSE, + } + + +class _CountingManager: + """Stands in for PortfolioManager: records every model call.""" + + calls = [] + + def __init__(self, **kwargs): + self.equity_history = [] + self.trades = [] + self.llm_calls = self.llm_decisions = 0 + self.input_tokens = self.output_tokens = 0 + + def restore_state(self, snapshot): + pass + + def snapshot_state(self): + return {"cash": 1.0, "positions": {}} + + def get_portfolio_state(self, *args): + return {} + + def make_trading_decision_with_llm(self, state, client, **kwargs): + _CountingManager.calls.append(state["timestamp"]) + self.llm_calls += 1 + self.llm_decisions += 1 + return {"actions": []} + + def execute_actions(self, *args): + pass + + def update_equity(self, market_data, price_cache, ts): + self.equity_history.append({"timestamp": ts, "equity": 1.0}) + + def get_equity_curve(self): + return self.equity_history + + +def test_the_decision_loop_asks_the_model_once_per_kept_bar(monkeypatch): + _CountingManager.calls = [] + monkeypatch.setattr(llm_agent, "PortfolioManager", _CountingManager) + stamps = _common.market_timestamps(_alpaca_hourly(["2026-11-25", "2026-11-27"])) + strategy = llm_agent.LLMAgentStrategy({"strategy": "llm_agent", "model_id": "m"}) + closes = [] + strategy._run_decision_loop( + client=object(), + timestamps=stamps, + symbols=["AAPL"], + data={}, + price_cache={}, + initial_capital=1.0, + model_id="m", + on_session_close=lambda day, curve: closes.append((day, strategy.llm_calls)), + ) + assert Counter( + ts.tz_convert("US/Eastern").date() for ts in _CountingManager.calls + ) == {date(2026, 11, 25): 6, date(2026, 11, 27): 3} + # One checkpoint per session, the counters cumulative when it fires. + assert closes == [(date(2026, 11, 25), 6), (date(2026, 11, 27), 9)] + + +@pytest.mark.parametrize( + ("month_start", "month_end", "sessions", "early_closes", "decisions"), + [ + ("2026-08-01", "2026-08-31", 21, 0, 126), + ("2026-11-01", "2026-11-30", 20, 1, 117), + ("2026-12-01", "2026-12-31", 22, 1, 129), + ], +) +def test_a_months_decisions_follow_the_calendar( + month_start, month_end, sessions, early_closes, decisions +): + """Spec §7's 126 calls is 6 x 21 sessions. An early close takes three off; + retries are billed on top of this floor.""" + days = live._trading_days_inclusive( + date.fromisoformat(month_start), date.fromisoformat(month_end) + ) + assert len(days) == sessions + assert sum(is_early_close(day) for day in days) == early_closes + stamps = _common.market_timestamps(_alpaca_hourly([d.isoformat() for d in days])) + assert len(stamps) == decisions == 6 * sessions - 3 * early_closes + + +def test_every_axis_node_after_a_days_first_is_where_a_decision_bar_closes(): + """The 10:00 node carries the prior close (the 09:00 bar straddles the open + and is never a decision); every later node is one decision bar's close. + Pins the chart axis and the decision filter to one calendar.""" + days = live._trading_days_inclusive(date(2026, 11, 1), date(2026, 11, 30)) + axis = live.live_month_hourly_axis("2026-11-01", "2026-11-30") + stamps = _common.market_timestamps(_alpaca_hourly([d.isoformat() for d in days])) + decision_nodes = [live._axis_node_key(live._parse_to_et(ts.isoformat())) for ts in stamps] + first_nodes = {f"{day.isoformat()}T10:00" for day in days} + assert decision_nodes == [ts[:16] for ts in axis if ts[:16] not in first_nodes] From d52232b809b651c1b61fd3279a9ef099666a9a3d Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 00:34:43 -0400 Subject: [PATCH 05/15] fix(baselines): put Yahoo's early-close index level on the 13:00 node On an NYSE early close Yahoo returns the 12:30-13:00 half bar as null and sends the closing level as a point stamped 13:00, which an open-stamped reader drops as after-hours. Restamp it into the half bar so the Live board's ^DJI/^GSPC baselines end the day on the official close. Pinned by two recorded ^DJI responses (2025-11-28 early close, 2025-11-26 full). Co-Authored-By: Claude Opus 5.5 --- .../domain/leaderboard/strategies/_yahoo.py | 46 +++++++++- .../yahoo-chart-dji-1h-2025-11-26.json | 41 +++++++++ .../yahoo-chart-dji-1h-2025-11-28.json | 37 +++++++++ dashboard/backend/tests/test_yahoo_chart.py | 83 +++++++++++++++++++ 4 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-11-26.json create mode 100644 dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-11-28.json diff --git a/dashboard/backend/domain/leaderboard/strategies/_yahoo.py b/dashboard/backend/domain/leaderboard/strategies/_yahoo.py index d823c1b93..6946e77c7 100644 --- a/dashboard/backend/domain/leaderboard/strategies/_yahoo.py +++ b/dashboard/backend/domain/leaderboard/strategies/_yahoo.py @@ -8,12 +8,19 @@ from __future__ import annotations import datetime as dt -from typing import List, Tuple +from typing import Dict, List, Optional, Tuple +from zoneinfo import ZoneInfo import requests +from dashboard.backend.infrastructure.market_data.sessions import ( + session_close_time, + session_windows, +) + _CHART_URL = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}" _HEADERS = {"User-Agent": "Mozilla/5.0"} +_US_EXCHANGE_ZONE = "America/New_York" class YahooChartError(requests.RequestException): @@ -121,4 +128,39 @@ def fetch_index_hourly( if close is None or not (start_e <= ts < end_e): continue points.append((dt.datetime.fromtimestamp(ts, dt.timezone.utc), float(close))) - return points + return _restamp_close_levels(points, (res.get("meta") or {}).get("exchangeTimezoneName")) + + +def _restamp_close_levels( + points: List[Tuple[dt.datetime, float]], exchange_zone: Optional[str] +) -> List[Tuple[dt.datetime, float]]: + """Move a US index level stamped *at* the session close into the half bar it closes. + + Every other point here is open-stamped on Yahoo's hourly grid, which starts + at the 09:30 open, so a session's last slot is a half bar (15:30-16:00). On + an NYSE early close Yahoo returns that half bar (12:30-13:00) with a null + close and sends the closing level as a separate point stamped 13:00. That + was recorded for ^DJI and ^GSPC on every early close from 2024-11-29 to + 2025-12-24; full sessions carry no such point. Any reader that treats the + series as open-stamped (the Live board's ``_axis_node_key``) drops a 13:00 + stamp as after-hours, so the day would end on its 12:30 level. Restamped to + the half bar's open, the closing level lands where that half bar's close + would. If Yahoo also filled the half bar, the closing level replaces it. + + Only for a US exchange: the close comes from the NYSE calendar + (``sessions.session_close_time``). + """ + if exchange_zone != _US_EXCHANGE_ZONE: + return points + zone = ZoneInfo(_US_EXCHANGE_ZONE) + session_open = session_windows("US")[0][0] + hour = dt.timedelta(hours=1) + by_stamp: Dict[dt.datetime, float] = {} + for stamp, level in points: + local = stamp.astimezone(zone) + if local.time() == session_close_time(local.date(), "US"): + opened = dt.datetime.combine(local.date(), session_open, tzinfo=zone) + last_slot = (local - opened) % hour or hour + stamp = (local - last_slot).astimezone(dt.timezone.utc) + by_stamp[stamp] = level + return sorted(by_stamp.items()) diff --git a/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-11-26.json b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-11-26.json new file mode 100644 index 000000000..e347797d2 --- /dev/null +++ b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-11-26.json @@ -0,0 +1,41 @@ +{ + "chart": { + "result": [ + { + "meta": { + "symbol": "^DJI", + "exchangeTimezoneName": "America/New_York", + "instrumentType": "INDEX", + "dataGranularity": "1h" + }, + "timestamp": [ + 1764167400, + 1764171000, + 1764174600, + 1764178200, + 1764181800, + 1764185400, + 1764189000, + 1792008000 + ], + "indicators": { + "quote": [ + { + "close": [ + 47325.21875, + 47466.30078125, + 47511.30078125, + 47526.578125, + 47530.98046875, + 47506.828125, + 47419.671875, + 51621.48046875 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-11-28.json b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-11-28.json new file mode 100644 index 000000000..ea3e851a4 --- /dev/null +++ b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-11-28.json @@ -0,0 +1,37 @@ +{ + "chart": { + "result": [ + { + "meta": { + "symbol": "^DJI", + "exchangeTimezoneName": "America/New_York", + "instrumentType": "INDEX", + "dataGranularity": "1h" + }, + "timestamp": [ + 1764340200, + 1764343800, + 1764347400, + 1764351000, + 1764352800, + 1792008000 + ], + "indicators": { + "quote": [ + { + "close": [ + 47677.671875, + 47699.78125, + 47660.55859375, + null, + 47716.421875, + 51621.48046875 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/dashboard/backend/tests/test_yahoo_chart.py b/dashboard/backend/tests/test_yahoo_chart.py index 254333f7d..8f904be19 100644 --- a/dashboard/backend/tests/test_yahoo_chart.py +++ b/dashboard/backend/tests/test_yahoo_chart.py @@ -12,7 +12,10 @@ """ import datetime as dt +import json +from pathlib import Path from types import SimpleNamespace +from zoneinfo import ZoneInfo import pytest import requests @@ -156,3 +159,83 @@ def test_yahoo_chart_error_is_a_request_exception(): # Load-bearing: equity_plot catches requests.RequestException, so this is # what routes a 200-delivered failure into the degrade-and-flag path. assert issubclass(YahooChartError, requests.RequestException) + + +# -------------------------------------------------------------------------- +# NYSE early closes: Yahoo stamps the closing level at the close itself +# -------------------------------------------------------------------------- +# +# Both fixtures are real ^DJI responses to the request fetch_index_hourly +# sends (period1 = the day's 00:00Z, period2 = +1 day, interval=1h), recorded +# 2026-10-15 with only ``meta`` trimmed to the fields that matter here. The +# trailing 2026-10-14T20:00Z point is Yahoo appending the latest close to every +# hourly response, which the window filter drops. On 2025-11-28 (13:00 close) +# the 12:30 half bar comes back null and the closing level, 47,716.42, arrives +# stamped 13:00 (18:00Z); the full session of 2025-11-26 has no such point. The +# same shape was recorded for ^DJI and ^GSPC on every NYSE early close from +# 2024-11-29 to 2025-12-24, and Yahoo's own ``tradingPeriods`` gives +# 2025-11-28 a 14:30Z-18:00Z regular session. + +_FIXTURES = Path(__file__).parent / "fixtures" +_NEW_YORK = ZoneInfo("America/New_York") + + +def _recorded(name): + return json.loads((_FIXTURES / name).read_text()) + + +def _clock(points): + return [(stamp.astimezone(_NEW_YORK).strftime("%H:%M"), value) for stamp, value in points] + + +def test_an_early_close_level_lands_in_the_last_half_bar(monkeypatch): + _stub_response(monkeypatch, _recorded("yahoo-chart-dji-1h-2025-11-28.json")) + points = fetch_index_hourly("^DJI", "2025-11-28", "2025-11-28") + assert _clock(points) == [ + ("09:30", 47677.671875), + ("10:30", 47699.78125), + ("11:30", 47660.55859375), + ("12:30", 47716.421875), + ] + + +def test_the_early_close_level_is_the_live_boards_13_00_node(monkeypatch): + """Read as an open-stamped bar, a point stamped 13:00 opens after the close + and has no node, so the index curve's day would end on its 12:30 level.""" + from dashboard.backend.domain.leaderboard import live + + _stub_response(monkeypatch, _recorded("yahoo-chart-dji-1h-2025-11-28.json")) + points = fetch_index_hourly("^DJI", "2025-11-28", "2025-11-28") + nodes = {live._axis_node_key(stamp.astimezone(_NEW_YORK)): value for stamp, value in points} + assert nodes == { + "2025-11-28T10:00": 47677.671875, + "2025-11-28T11:00": 47699.78125, + "2025-11-28T12:00": 47660.55859375, + "2025-11-28T13:00": 47716.421875, + } + + +def test_a_full_session_is_left_as_yahoo_sent_it(monkeypatch): + _stub_response(monkeypatch, _recorded("yahoo-chart-dji-1h-2025-11-26.json")) + points = fetch_index_hourly("^DJI", "2025-11-26", "2025-11-26") + assert [clock for clock, _value in _clock(points)] == [ + "09:30", "10:30", "11:30", "12:30", "13:30", "14:30", "15:30", + ] + assert points[-1][1] == 47419.671875 + + +def test_the_close_stamped_level_wins_over_a_half_bar_yahoo_also_filled(monkeypatch): + payload = _recorded("yahoo-chart-dji-1h-2025-11-28.json") + payload["chart"]["result"][0]["indicators"]["quote"][0]["close"][3] = 47700.0 + _stub_response(monkeypatch, payload) + points = fetch_index_hourly("^DJI", "2025-11-28", "2025-11-28") + assert _clock(points)[-1] == ("12:30", 47716.421875) + assert len(points) == 4 + + +def test_a_non_us_index_is_not_restamped_on_an_nyse_early_close(monkeypatch): + payload = _recorded("yahoo-chart-dji-1h-2025-11-28.json") + payload["chart"]["result"][0]["meta"]["exchangeTimezoneName"] = "Asia/Shanghai" + _stub_response(monkeypatch, payload) + points = fetch_index_hourly("^DJI", "2025-11-28", "2025-11-28") + assert _clock(points)[-1] == ("13:00", 47716.421875) From d3bbf68bbd94d76bf0ea3c773e3b390ef66517c9 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 00:36:40 -0400 Subject: [PATCH 06/15] test(calendar): pin NYSE early closes to exchange_calendars XNYS, 2004-2035 The rule was pinned to four published years. A list generated once from exchange_calendars 4.13.2 (no new dependency) widens that to 32: 22 years of sessions that happened and 10 years of an independent implementation. States where the rule stops holding (pre-2004 policy) and corrects the post-close bar wording: on SIP the 13:00 bar opens with the closing auction, it is not just thin after-hours prints. Co-Authored-By: Claude Opus 5.5 --- .../market_data/us_market_calendar.py | 8 ++- .../market_data/test_us_market_calendar.py | 53 ++++++++++++++++++- .../backend/tests/test_market_sessions.py | 5 +- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/dashboard/backend/infrastructure/market_data/us_market_calendar.py b/dashboard/backend/infrastructure/market_data/us_market_calendar.py index c107c9b11..3b062549e 100644 --- a/dashboard/backend/infrastructure/market_data/us_market_calendar.py +++ b/dashboard/backend/infrastructure/market_data/us_market_calendar.py @@ -100,8 +100,12 @@ def nyse_early_closes(year: int) -> FrozenSet[date]: A candidate that is not a trading day is dropped, so this set can never contradict ``nyse_holidays``. The rules reproduce NYSE's published 2025-2028 - calendars; an unscheduled closure is not predictable by rule, and the Live - board's session-gap check stays the backstop for one. + calendars and every year 2004-2035 of exchange_calendars' XNYS calendar + (the test module pins both). Earlier years followed other rules: a Thursday + July 4 put the half day on Friday July 5 (1996, 2002), and 1997-12-26, + 1999-12-31 and 2003-12-26 were one-off half days. An unscheduled closure is + not predictable by rule, and the Live board's session-gap check stays the + backstop for one. """ candidates = {_nth_weekday(year, 11, 3, 4) + timedelta(days=1)} christmas_eve = date(year, 12, 24) diff --git a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py index 3fb7a190c..82f2f670a 100644 --- a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py +++ b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py @@ -5,7 +5,15 @@ chart nodes and three model decisions on late-session bars. Sources: the NYSE Group releases "2025, 2026 and 2027 Holiday and Early Closings Calendar" (2024-11-08) and "2026, 2027 and 2028 ..." (2025-12-23), and nyse.com -"Holidays & Trading Hours". +"Holidays & Trading Hours" (https://www.nyse.com/trade/hours-calendars). + +``XNYS_EARLY_CLOSES`` widens the pin to 2004-2035. It was generated once, +offline, from exchange_calendars 4.13.2 (``get_calendar("XNYS").closes``: every +session closing before 16:00, all of them at 13:00), the maintained calendar +library zipline's descendants use; it is not a dependency. For 2004-2025 it is +sessions that already happened; for 2026-2035 it is a second, independent +implementation of the same rules, so it catches a slip in ours rather than a +change in NYSE's policy. That only NYSE's next release can show. """ from datetime import date, time @@ -35,6 +43,49 @@ def test_early_closes_match_the_nyse_published_calendar(year): ) +XNYS_EARLY_CLOSES = { + 2004: ["11-26"], + 2005: ["11-25"], + 2006: ["07-03", "11-24"], + 2007: ["07-03", "11-23", "12-24"], + 2008: ["07-03", "11-28", "12-24"], + 2009: ["11-27", "12-24"], + 2010: ["11-26"], + 2011: ["11-25"], + 2012: ["07-03", "11-23", "12-24"], + 2013: ["07-03", "11-29", "12-24"], + 2014: ["07-03", "11-28", "12-24"], + 2015: ["11-27", "12-24"], + 2016: ["11-25"], + 2017: ["07-03", "11-24"], + 2018: ["07-03", "11-23", "12-24"], + 2019: ["07-03", "11-29", "12-24"], + 2020: ["11-27", "12-24"], + 2021: ["11-26"], + 2022: ["11-25"], + 2023: ["07-03", "11-24"], + 2024: ["07-03", "11-29", "12-24"], + 2025: ["07-03", "11-28", "12-24"], + 2026: ["11-27", "12-24"], + 2027: ["11-26"], + 2028: ["07-03", "11-24"], + 2029: ["07-03", "11-23", "12-24"], + 2030: ["07-03", "11-29", "12-24"], + 2031: ["07-03", "11-28", "12-24"], + 2032: ["11-26"], + 2033: ["11-25"], + 2034: ["07-03", "11-24"], + 2035: ["07-03", "11-23", "12-24"], +} + + +@pytest.mark.parametrize("year", sorted(XNYS_EARLY_CLOSES)) +def test_early_closes_match_exchange_calendars_xnys(year): + assert sorted(d.strftime("%m-%d") for d in nyse_early_closes(year)) == ( + XNYS_EARLY_CLOSES[year] + ) + + def test_a_holiday_or_weekend_is_never_an_early_close(): # Independence Day 2026 is observed on Friday July 3; Christmas 2027 on # Friday December 24; December 24, 2028 is a Sunday. diff --git a/dashboard/backend/tests/test_market_sessions.py b/dashboard/backend/tests/test_market_sessions.py index 220006ac3..d7e409632 100644 --- a/dashboard/backend/tests/test_market_sessions.py +++ b/dashboard/backend/tests/test_market_sessions.py @@ -189,7 +189,10 @@ def _open_stamped_kept(stamps, minutes=60): def test_open_stamped_hourly_early_close_is_three_bars(): """2026-11-27 closes at 13:00 ET. Alpaca serves the late session as bars stamped 13:00, 14:00 and 15:00, and a fixed 16:00 close kept them: three - decisions priced off thin after-hours prints.""" + decisions taken after the session had ended. On SIP the 13:00 bar opens + with the closing auction, as the 16:00 bar does on a full day; the 14:00 + and 15:00 bars are late-session prints (NYSE Arca and others trade to + 17:00 on an early close).""" day = pd.date_range("2026-11-27 04:00", "2026-11-27 19:00", freq="h", tz="US/Eastern") assert _open_stamped_kept(day) == ["10:00", "11:00", "12:00"] From a09bdfb6ab99b3af5eb8db5a410bb1e7a65fbe64 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 00:46:24 -0400 Subject: [PATCH 07/15] test(sessions): check the open-stamped rule against a naive oracle, 2025-2028 Every trading day, each span the Alpaca loader stamps (1, 5, 60 min): a bar is in session iff it opens at or after 09:30 and closes by that day's close. The pattern is exchange_calendars' TestTradingIndex; the hourly-shape pin is the pandas_market_calendars #138 failure mode (a short session losing bars). Co-Authored-By: Claude Opus 5.5 --- .../backend/tests/test_market_sessions.py | 67 ++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/dashboard/backend/tests/test_market_sessions.py b/dashboard/backend/tests/test_market_sessions.py index d7e409632..568753ab3 100644 --- a/dashboard/backend/tests/test_market_sessions.py +++ b/dashboard/backend/tests/test_market_sessions.py @@ -8,13 +8,19 @@ import ast import functools +from datetime import date, datetime, timedelta from datetime import time as clock_time from pathlib import Path import pandas as pd import pytest +import pytz -from dashboard.backend.infrastructure.market_data import profiles, sessions +from dashboard.backend.infrastructure.market_data import ( + profiles, + sessions, + us_market_calendar, +) BACKEND = Path(__file__).resolve().parents[1] @@ -212,6 +218,65 @@ def test_open_stamped_five_minute_bars_stop_at_the_early_close(): assert _open_stamped_kept(bars, minutes=5) == ["12:45", "12:50", "12:55"] +def _trading_days(first_year, last_year): + day = date(first_year, 1, 1) + while day.year <= last_year: + if us_market_calendar.is_trading_day(day): + yield day + day += timedelta(days=1) + + +@pytest.mark.parametrize( + ("minutes", "sweeps"), + [ + (60, [(clock_time(4, 0), clock_time(20, 0))]), + (5, [(clock_time(9, 0), clock_time(10, 0)), + (clock_time(12, 30), clock_time(16, 30))]), + (1, [(clock_time(9, 25), clock_time(9, 35)), + (clock_time(12, 50), clock_time(13, 10)), + (clock_time(15, 50), clock_time(16, 10))]), + ], +) +def test_the_open_stamped_rule_matches_a_naive_oracle_every_day(minutes, sweeps): + """exchange_calendars tests ``trading_index`` against a deliberately naive + loop (``TestTradingIndex``); this does the same for the open-stamped rule, + on every trading day 2025-2028 and every span the Alpaca loader stamps. + + A bar is in session iff it opens at or after 09:30 and closes at or before + that day's close: a left-closed, whole-bar index that never emits a + partial last bar (``closed="left"``, ``force_close=False`` there). The + hourly shapes are pinned too, because pandas_market_calendars #138 was a + short session silently losing its bars, not a long one keeping extra.""" + zone = pytz.timezone("US/Eastern") + span = timedelta(minutes=minutes) + early_closes = 0 + for day in _trading_days(2025, 2028): + close = sessions.session_close_time(day, "US") + early_closes += close == sessions.EARLY_CLOSE_TIME + kept = [] + for start, end in sweeps: + stamp = datetime.combine(day, start) + while stamp.time() < end: + bar_close = stamp + span + expected = ( + stamp.time() >= clock_time(9, 30) + and bar_close.date() == day + and bar_close.time() <= close + ) + actual = sessions.is_in_session( + zone.localize(stamp), market="US", timezone="US/Eastern", + open_stamped_minutes=minutes, + ) + assert actual == expected, (stamp, minutes) + if actual: + kept.append(stamp.hour) + stamp = bar_close + if minutes == 60: + last = 12 if close == sessions.EARLY_CLOSE_TIME else 15 + assert kept == list(range(10, last + 1)), day + assert early_closes == 8 + + def test_the_early_close_is_read_on_the_market_local_date(): """18:00Z on 2026-11-27 is 13:00 ET, after that day's close; a naive stamp is market-local, as everywhere else in this module.""" From c403732fd2aa8f842348db52f9877b73229bee16 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 00:50:45 -0400 Subject: [PATCH 08/15] fix(calendar): close NYSE's unscheduled closure days, 2004 onward The rules missed the days NYSE closed at short notice, so is_trading_day called 2025-01-09 (President Carter's day of mourning) a session, and so did everything that walks the calendar: backtest window sizing, the default window, the Live board's expected sessions. Dated data beside the rules, as exchange_calendars keeps its XNYS adhoc_holidays: Reagan, Ford, Sandy (2), G. H. W. Bush, Carter. Pinned by sessions per year against exchange_calendars 4.13.2 XNYS, 2004-2035 (generated once offline; not a dependency). Co-Authored-By: Claude Opus 5.5 --- .../market_data/us_market_calendar.py | 34 ++++++++++++++--- .../market_data/test_us_market_calendar.py | 37 ++++++++++++++++++- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/dashboard/backend/infrastructure/market_data/us_market_calendar.py b/dashboard/backend/infrastructure/market_data/us_market_calendar.py index 3b062549e..0f8c04743 100644 --- a/dashboard/backend/infrastructure/market_data/us_market_calendar.py +++ b/dashboard/backend/infrastructure/market_data/us_market_calendar.py @@ -12,9 +12,10 @@ freeze day whose one-day increment had no bars and failed every model, and it padded "Day N of M" and the chart axis with a session that never trades. -Covers the ten NYSE holidays in force since 2022 (Juneteenth added that year) -and the recurring 13:00 ET early closes (``nyse_early_closes``). An early close -is still a trading day. The hour it ends at is a session bound, so it lives +Covers the ten NYSE holidays in force since 2022 (Juneteenth added that year), +the dated unscheduled closures since 2004 (``UNSCHEDULED_CLOSURES``) and the +recurring 13:00 ET early closes (``nyse_early_closes``). An early close is +still a trading day. The hour it ends at is a session bound, so it lives with the other bounds in ``sessions.py`` (``session_close_time``). """ @@ -61,8 +62,27 @@ def _observed(day: date) -> date: return day +#: Full-day closures NYSE called at short notice, which no rule reproduces: +#: national days of mourning and Hurricane Sandy. Dated data beside the rules, +#: the split exchange_calendars makes (``adhoc_holidays`` in its XNYS +#: calendar, where these come from). The next one is a one-line addition; until +#: it lands, the Live board's session-gap check (``live._session_gap``) fails +#: that night's refresh loudly rather than trading a closed day. +UNSCHEDULED_CLOSURES: FrozenSet[date] = frozenset({ + date(2004, 6, 11), # President Reagan + date(2007, 1, 2), # President Ford + date(2012, 10, 29), # Hurricane Sandy + date(2012, 10, 30), # Hurricane Sandy + date(2018, 12, 5), # President G. H. W. Bush + date(2025, 1, 9), # President Carter +}) + + @lru_cache(maxsize=16) def nyse_holidays(year: int) -> FrozenSet[date]: + """NYSE's full-day closures in ``year``: the rule-based holidays plus any + :data:`UNSCHEDULED_CLOSURES`. Matches every year 2004-2035 of + exchange_calendars' XNYS sessions (the test module pins the count).""" days = { _nth_weekday(year, 1, 0, 3), # Martin Luther King Jr. Day _nth_weekday(year, 2, 0, 3), # Washington's Birthday @@ -80,6 +100,7 @@ def nyse_holidays(year: int) -> FrozenSet[date]: days.add(_observed(new_year)) if year >= 2022: days.add(_observed(date(year, 6, 19))) # Juneteenth + days.update(day for day in UNSCHEDULED_CLOSURES if day.year == year) return frozenset(days) @@ -103,9 +124,10 @@ def nyse_early_closes(year: int) -> FrozenSet[date]: calendars and every year 2004-2035 of exchange_calendars' XNYS calendar (the test module pins both). Earlier years followed other rules: a Thursday July 4 put the half day on Friday July 5 (1996, 2002), and 1997-12-26, - 1999-12-31 and 2003-12-26 were one-off half days. An unscheduled closure is - not predictable by rule, and the Live board's session-gap check stays the - backstop for one. + 1999-12-31 and 2003-12-26 were one-off half days. NYSE has called no + unscheduled early close since 2004; if it does, the date needs adding by + hand, and nothing catches its absence: Alpaca still serves bars after an + early close, so the session-gap check sees a normal day. """ candidates = {_nth_weekday(year, 11, 3, 4) + timedelta(days=1)} christmas_eve = date(year, 12, 24) diff --git a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py index 82f2f670a..9d676bd3f 100644 --- a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py +++ b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py @@ -16,12 +16,13 @@ change in NYSE's policy. That only NYSE's next release can show. """ -from datetime import date, time +from datetime import date, time, timedelta import pytest from dashboard.backend.infrastructure.market_data import sessions from dashboard.backend.infrastructure.market_data.us_market_calendar import ( + UNSCHEDULED_CLOSURES, is_early_close, is_trading_day, nyse_early_closes, @@ -86,6 +87,40 @@ def test_early_closes_match_exchange_calendars_xnys(year): ) +#: Sessions per year in exchange_calendars 4.13.2's XNYS calendar, generated +#: the same way as ``XNYS_EARLY_CLOSES``. A count catches a missing or extra +#: closure anywhere in the year without pinning ~8,000 dates. +XNYS_SESSIONS_PER_YEAR = { + 2004: 252, 2005: 252, 2006: 251, 2007: 251, 2008: 253, 2009: 252, + 2010: 252, 2011: 252, 2012: 250, 2013: 252, 2014: 252, 2015: 252, + 2016: 252, 2017: 251, 2018: 251, 2019: 252, 2020: 253, 2021: 252, + 2022: 251, 2023: 250, 2024: 252, 2025: 250, 2026: 251, 2027: 251, + 2028: 251, 2029: 251, 2030: 251, 2031: 251, 2032: 252, 2033: 251, + 2034: 250, 2035: 251, +} + + +@pytest.mark.parametrize("year", sorted(XNYS_SESSIONS_PER_YEAR)) +def test_sessions_per_year_match_exchange_calendars_xnys(year): + day, sessions_in_year = date(year, 1, 1), 0 + while day.year == year: + sessions_in_year += is_trading_day(day) + day += timedelta(days=1) + assert sessions_in_year == XNYS_SESSIONS_PER_YEAR[year] + + +def test_unscheduled_closures_are_not_trading_days(): + """No rule reproduces these. 2025-01-09 (President Carter's national day of + mourning) is the one a current backtest window can reach.""" + assert date(2025, 1, 9) in UNSCHEDULED_CLOSURES + for day in UNSCHEDULED_CLOSURES: + assert day.weekday() < 5, day + assert day in nyse_holidays(day.year) + assert not is_trading_day(day) + assert is_trading_day(date(2025, 1, 8)) + assert is_trading_day(date(2025, 1, 10)) + + def test_a_holiday_or_weekend_is_never_an_early_close(): # Independence Day 2026 is observed on Friday July 3; Christmas 2027 on # Friday December 24; December 24, 2028 is a Sunday. From d0f695cc5a042db83f7a312b13c96fde86f03470 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 00:52:54 -0400 Subject: [PATCH 09/15] docs(sessions): say what the early-close bars hold and what the close mark is Research corrections, no behaviour change: - the bars after a 13:00 bell carry real prints (late sessions and Alpaca's half-day extended hours run to 17:00); the filter drops them by stamp; - every US day is marked pre-auction: the auction print opens the dropped bar. Yahoo's index level is the exception on a half day (official close, ~0.02% off a full day's basis), the lesser gap against a stale 11:30 level; - freeze_settle_time settles against the SIP delay clamp; Alpaca can still revise bars after it; - the dense 04:00-19:00 test grids are a superset of Alpaca's output, not its shape; the node-key test names Yahoo's restamped closing level. Co-Authored-By: Claude Opus 5.5 --- dashboard/backend/domain/leaderboard/live.py | 15 ++++++++++++--- .../domain/leaderboard/strategies/_yahoo.py | 9 +++++++++ .../infrastructure/market_data/sessions.py | 14 ++++++++++++-- .../domain/leaderboard/test_llm_agent_cadence.py | 6 ++++-- dashboard/backend/tests/test_live_leaderboard.py | 10 +++++++--- 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/dashboard/backend/domain/leaderboard/live.py b/dashboard/backend/domain/leaderboard/live.py index 214e33cbd..31d668535 100644 --- a/dashboard/backend/domain/leaderboard/live.py +++ b/dashboard/backend/domain/leaderboard/live.py @@ -195,7 +195,15 @@ def live_increment_bounds( def freeze_settle_time(day: date) -> datetime: - """When ``day``'s closing bar is final on the board's tape.""" + """When ``day`` can be frozen: its cash close (13:00 on an NYSE early + close) plus the SIP delay and a margin, so the delay clamp no longer + truncates the last kept bar and Alpaca's late-trade update has landed. + + Settled is not immutable. Alpaca applies SIP corrections after the 20:00 + extended session and rebuilds recent bars for a few days afterwards, so a + later refetch (a config-change replay of the month) can differ slightly + from the frozen checkpoint. + """ delay = 0 if allow_recent_sip() else sip_delay_minutes() close = _cash_close(day) return close + timedelta(minutes=delay + _FREEZE_SETTLE_MARGIN_MINUTES) @@ -1235,8 +1243,9 @@ def _axis_node_key(bar_open: datetime) -> Optional[str]: Alpaca's 09:00 bar (09:30 open → 10:00) lands on 10:00 and its 15:00 bar on the 16:00 close; Yahoo's 09:30 bar closes 10:30 and lands on 10:00, and its 15:30 half-bar is capped at the 16:00 close. On an NYSE early close the cap - is 13:00, where Alpaca's 12:00 bar and Yahoo's 12:30 half-bar both land. A - bar opening at or after the day's close is after-hours and has no node. + is 13:00. Alpaca's 12:00 bar lands there, and so does Yahoo's closing level, + which ``_yahoo`` restamps from its 13:00 stub to the 12:30 half-bar. A bar + opening at or after the day's close is after-hours and has no node. """ session_close = _cash_close(bar_open.date()) if bar_open >= session_close: diff --git a/dashboard/backend/domain/leaderboard/strategies/_yahoo.py b/dashboard/backend/domain/leaderboard/strategies/_yahoo.py index 6946e77c7..e098273a9 100644 --- a/dashboard/backend/domain/leaderboard/strategies/_yahoo.py +++ b/dashboard/backend/domain/leaderboard/strategies/_yahoo.py @@ -147,6 +147,15 @@ def _restamp_close_levels( the half bar's open, the closing level lands where that half bar's close would. If Yahoo also filled the half bar, the closing level replaces it. + That level is the official close, auction included: ^DJI 47,716.42 on + 2025-11-28, Yahoo's daily close to the cent. On a full session the 15:30 + half bar's close is pre-auction instead (47,419.67 against an official + 47,427.12 on 2025-11-26), as every agent and Alpaca-baseline mark is on + both kinds of day (``sessions.is_in_session``). The index curve therefore + ends a half day about 0.02% off the basis it ends a full day on. That is + the lesser gap: the only pre-auction level on offer is the 11:30 bar's, + 30 minutes stale. + Only for a US exchange: the close comes from the NYSE calendar (``sessions.session_close_time``). """ diff --git a/dashboard/backend/infrastructure/market_data/sessions.py b/dashboard/backend/infrastructure/market_data/sessions.py index 33edea3a4..4ed82ce7c 100644 --- a/dashboard/backend/infrastructure/market_data/sessions.py +++ b/dashboard/backend/infrastructure/market_data/sessions.py @@ -172,10 +172,20 @@ def is_in_session( first fill off a pre-market print. The day's last session ends at :func:`session_close_time`: 13:00 ET on an - NYSE early close, where Alpaca still serves the late session as bars - stamped 13:00, 14:00 and 15:00. Only this open-stamped rule reads the + NYSE early close. Trading goes on after that bell (the NYSE Arca, + American, National and Texas late sessions, FINRA TRF post-market + reporting and Alpaca's half-day extended session all run to 17:00), so + bars stamped 13:00 and later can carry real prints: the rule drops them by + their stamp, never by their absence. Only this open-stamped rule reads the day's close; the close-stamped rule (:func:`time_in_session`) still ends every US day at 16:00 (D19, owned by the engine rewrite). + + The close mark this leaves is pre-auction on every US day. The last kept + bar (the 12:00 bar on an early close, 15:00 otherwise) closes on the last + last-sale-eligible trade before the bell, not the official close: the + closing-auction print, stamped at or just after 13:00:00 (16:00:00), opens + the next bar, which is dropped. Marking on that bar instead was rejected, + because post-close prints in the same hour can set its close. """ local = market_local(timestamp, timezone) if open_stamped_minutes is None: diff --git a/dashboard/backend/tests/domain/leaderboard/test_llm_agent_cadence.py b/dashboard/backend/tests/domain/leaderboard/test_llm_agent_cadence.py index ea4ba3ddb..ba529ae73 100644 --- a/dashboard/backend/tests/domain/leaderboard/test_llm_agent_cadence.py +++ b/dashboard/backend/tests/domain/leaderboard/test_llm_agent_cadence.py @@ -27,8 +27,10 @@ def _alpaca_hourly(days): - """Open-stamped hourly bars, 04:00-19:00 ET on each day: Alpaca serves - extended hours, so the late session after an early close is present.""" + """Open-stamped hourly bars, every hour 04:00-19:00 ET on each day: a + deliberate superset of what Alpaca sends (no bar for an hour without a + valid trade; the half-day extended session ends at 17:00), so the late + session after an early close is certainly present.""" index = pd.DatetimeIndex([ ts for day in days diff --git a/dashboard/backend/tests/test_live_leaderboard.py b/dashboard/backend/tests/test_live_leaderboard.py index efe7f22c1..9fc3312bb 100644 --- a/dashboard/backend/tests/test_live_leaderboard.py +++ b/dashboard/backend/tests/test_live_leaderboard.py @@ -330,7 +330,9 @@ def key(hour, minute=0, day=27): return live._axis_node_key(datetime(2026, 11, day, hour, minute, tzinfo=_ET)) assert key(12) == "2026-11-27T13:00" # Alpaca's last cash-session bar - assert key(12, 30) == "2026-11-27T13:00" # Yahoo's 12:30-13:00 half bar + # Yahoo's closing level, restamped to 12:30 by _yahoo from its 13:00 stub + # (test_yahoo_chart pins that against recorded responses). + assert key(12, 30) == "2026-11-27T13:00" assert key(13) is None # the late session has no node assert key(15) is None assert key(15, day=25) == "2026-11-25T16:00" @@ -366,8 +368,10 @@ def test_the_live_board_keeps_no_fixed_session_close(): def _alpaca_hourly_bars(*days): - """AAPL bars shaped like Alpaca's: open-stamped hourly, 04:00-19:00 ET, so - the late session after an early close is in the response.""" + """Open-stamped hourly AAPL bars, every hour 04:00-19:00 ET: a deliberate + superset of what Alpaca sends. It emits no bar for an hour without a valid + trade, and its half-day extended session ends at 17:00. Dense, so the late + session after an early close is certainly in the response.""" import pandas as pd from dashboard.backend.infrastructure.market_data.sessions import ( From 2306d7b51214a8ac1bf77ebc199b8a2ae8ac3c75 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 00:56:26 -0400 Subject: [PATCH 10/15] fix(baselines): restamp Yahoo's close level on NYSE early closes only Yahoo appends the latest close to every hourly response, stamped at the close itself, so on a full session a fetch that reaches today held a point at 16:00 ET and the restamp moved it onto the 15:30 half bar: today marked on the official close, the same day replayed later marked pre-auction. The restamp now needs is_early_close as well as the close time. Pinned with the recorded response run through a window that keeps the appended point, and with three more recorded early closes (^GSPC 2025-11-28, ^DJI and ^GSPC 2025-12-24) whose last level is the published close. Co-Authored-By: Claude Opus 5.5 --- .../domain/leaderboard/strategies/_yahoo.py | 13 ++++-- .../yahoo-chart-dji-1h-2025-12-24.json | 37 +++++++++++++++ .../yahoo-chart-gspc-1h-2025-11-28.json | 37 +++++++++++++++ .../yahoo-chart-gspc-1h-2025-12-24.json | 37 +++++++++++++++ dashboard/backend/tests/test_yahoo_chart.py | 46 +++++++++++++++++-- 5 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-12-24.json create mode 100644 dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-11-28.json create mode 100644 dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-12-24.json diff --git a/dashboard/backend/domain/leaderboard/strategies/_yahoo.py b/dashboard/backend/domain/leaderboard/strategies/_yahoo.py index e098273a9..3f578ff01 100644 --- a/dashboard/backend/domain/leaderboard/strategies/_yahoo.py +++ b/dashboard/backend/domain/leaderboard/strategies/_yahoo.py @@ -17,6 +17,9 @@ session_close_time, session_windows, ) +from dashboard.backend.infrastructure.market_data.us_market_calendar import ( + is_early_close, +) _CHART_URL = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}" _HEADERS = {"User-Agent": "Mozilla/5.0"} @@ -156,8 +159,10 @@ def _restamp_close_levels( the lesser gap: the only pre-auction level on offer is the 11:30 bar's, 30 minutes stale. - Only for a US exchange: the close comes from the NYSE calendar - (``sessions.session_close_time``). + Only on an NYSE early close, for a US exchange. On a full session the one + point stamped at the close is the latest close Yahoo appends to every + response, inside the window whenever a fetch reaches today; moving it would + mark today on the official close and a replay of the same day pre-auction. """ if exchange_zone != _US_EXCHANGE_ZONE: return points @@ -167,7 +172,9 @@ def _restamp_close_levels( by_stamp: Dict[dt.datetime, float] = {} for stamp, level in points: local = stamp.astimezone(zone) - if local.time() == session_close_time(local.date(), "US"): + if is_early_close(local.date()) and local.time() == session_close_time( + local.date(), "US" + ): opened = dt.datetime.combine(local.date(), session_open, tzinfo=zone) last_slot = (local - opened) % hour or hour stamp = (local - last_slot).astimezone(dt.timezone.utc) diff --git a/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-12-24.json b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-12-24.json new file mode 100644 index 000000000..8068c541b --- /dev/null +++ b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-12-24.json @@ -0,0 +1,37 @@ +{ + "chart": { + "result": [ + { + "meta": { + "symbol": "^DJI", + "exchangeTimezoneName": "America/New_York", + "instrumentType": "INDEX", + "dataGranularity": "1h" + }, + "timestamp": [ + 1766586600, + 1766590200, + 1766593800, + 1766597400, + 1766599200, + 1792008000 + ], + "indicators": { + "quote": [ + { + "close": [ + 48617.87890625, + 48686.30859375, + 48738.96875, + null, + 48731.16015625, + 51621.48046875 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-11-28.json b/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-11-28.json new file mode 100644 index 000000000..f89281d92 --- /dev/null +++ b/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-11-28.json @@ -0,0 +1,37 @@ +{ + "chart": { + "result": [ + { + "meta": { + "symbol": "^GSPC", + "exchangeTimezoneName": "America/New_York", + "instrumentType": "INDEX", + "dataGranularity": "1h" + }, + "timestamp": [ + 1764340200, + 1764343800, + 1764347400, + 1764351000, + 1764352800, + 1792008000 + ], + "indicators": { + "quote": [ + { + "close": [ + 6831.4501953125, + 6836.14013671875, + 6837.9501953125, + null, + 6849.08984375, + 7718.93994140625 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-12-24.json b/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-12-24.json new file mode 100644 index 000000000..89d136b96 --- /dev/null +++ b/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-12-24.json @@ -0,0 +1,37 @@ +{ + "chart": { + "result": [ + { + "meta": { + "symbol": "^GSPC", + "exchangeTimezoneName": "America/New_York", + "instrumentType": "INDEX", + "dataGranularity": "1h" + }, + "timestamp": [ + 1766586600, + 1766590200, + 1766593800, + 1766597400, + 1766599200, + 1792008000 + ], + "indicators": { + "quote": [ + { + "close": [ + 6920.2099609375, + 6928.93994140625, + 6934.8798828125, + null, + 6932.0498046875, + 7718.93994140625 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/dashboard/backend/tests/test_yahoo_chart.py b/dashboard/backend/tests/test_yahoo_chart.py index 8f904be19..32853c8e4 100644 --- a/dashboard/backend/tests/test_yahoo_chart.py +++ b/dashboard/backend/tests/test_yahoo_chart.py @@ -165,9 +165,10 @@ def test_yahoo_chart_error_is_a_request_exception(): # NYSE early closes: Yahoo stamps the closing level at the close itself # -------------------------------------------------------------------------- # -# Both fixtures are real ^DJI responses to the request fetch_index_hourly -# sends (period1 = the day's 00:00Z, period2 = +1 day, interval=1h), recorded -# 2026-10-15 with only ``meta`` trimmed to the fields that matter here. The +# The fixtures are real ^DJI and ^GSPC responses to the request +# fetch_index_hourly sends (period1 = the day's 00:00Z, period2 = +1 day, +# interval=1h), recorded 2026-10-15 and trimmed to ``meta``'s relevant fields +# and the closes (Yahoo serves 1h data for the last 730 days only). The # trailing 2026-10-14T20:00Z point is Yahoo appending the latest close to every # hourly response, which the window filter drops. On 2025-11-28 (13:00 close) # the 12:30 half bar comes back null and the closing level, 47,716.42, arrives @@ -199,6 +200,29 @@ def test_an_early_close_level_lands_in_the_last_half_bar(monkeypatch): ] +@pytest.mark.parametrize( + ("fixture", "symbol", "day", "official_close"), + [ + ("yahoo-chart-dji-1h-2025-11-28.json", "^DJI", "2025-11-28", 47716.421875), + ("yahoo-chart-gspc-1h-2025-11-28.json", "^GSPC", "2025-11-28", 6849.08984375), + ("yahoo-chart-dji-1h-2025-12-24.json", "^DJI", "2025-12-24", 48731.16015625), + ("yahoo-chart-gspc-1h-2025-12-24.json", "^GSPC", "2025-12-24", 6932.0498046875), + ], +) +def test_each_recorded_early_close_ends_on_the_official_close( + monkeypatch, fixture, symbol, day, official_close +): + """Each level is that day's published close, equal to Yahoo's own daily + bar: 47,716.42 and 6,849.09 on 2025-11-28, 48,731.16 and 6,932.05 on + 2025-12-24.""" + _stub_response(monkeypatch, _recorded(fixture)) + points = fetch_index_hourly(symbol, day, day) + assert [clock for clock, _value in _clock(points)] == [ + "09:30", "10:30", "11:30", "12:30", + ] + assert points[-1][1] == official_close + + def test_the_early_close_level_is_the_live_boards_13_00_node(monkeypatch): """Read as an open-stamped bar, a point stamped 13:00 opens after the close and has no node, so the index curve's day would end on its 12:30 level.""" @@ -224,6 +248,22 @@ def test_a_full_session_is_left_as_yahoo_sent_it(monkeypatch): assert points[-1][1] == 47419.671875 +def test_yahoos_appended_close_is_not_restamped_on_a_full_session(monkeypatch): + """Yahoo appends the latest close to every response, stamped at the close + itself: 16:00 ET on a full session. The nightly refresh fetches through + today, so the point is inside its window. Restamped, it would overwrite + today's 15:30 half bar with the official close, and a later replay of the + same day (the point gone by then) would mark it pre-auction again. Only an + NYSE early close restamps; here the point stays at 16:00, where the board's + node mapping drops it.""" + _stub_response(monkeypatch, _recorded("yahoo-chart-dji-1h-2025-11-26.json")) + points = fetch_index_hourly("^DJI", "2025-11-26", "2026-10-14") + assert points[-2][1] == 47419.671875 # 2025-11-26's 15:30 half bar + assert points[-1] == ( + dt.datetime(2026, 10, 14, 20, tzinfo=dt.timezone.utc), 51621.48046875, + ) + + def test_the_close_stamped_level_wins_over_a_half_bar_yahoo_also_filled(monkeypatch): payload = _recorded("yahoo-chart-dji-1h-2025-11-28.json") payload["chart"]["result"][0]["indicators"]["quote"][0]["close"][3] = 47700.0 From 0a2c1c9045e8cbc6c022806dfdb83686678df606 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 00:58:24 -0400 Subject: [PATCH 11/15] fix(calendar): dated unscheduled early closes; pin NYSE's 2022-2024 release - UNSCHEDULED_EARLY_CLOSES (empty: none since 2004) joins the rule's candidates ahead of the trading-day filter, so the next short-notice half day is a one-line addition that can shorten a session but never open a closed one. The docstring now says what nothing catches: a missing one, since bars still arrive after the bell. - PUBLISHED_EARLY_CLOSES gains 2022-2024 from NYSE's 2021-12-27 release (checked against the release text), so the hand-copied pin covers 2022-2028, with sources, retrieval date and a re-pin note. - The rules are described as what they are: NYSE Rule 7.2 lists holidays only, early closes are announced year by year, later years extrapolate. Co-Authored-By: Claude Opus 5.5 --- dashboard/backend/domain/leaderboard/live.py | 5 +-- .../market_data/us_market_calendar.py | 27 +++++++++----- .../market_data/test_us_market_calendar.py | 36 ++++++++++++++++--- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/dashboard/backend/domain/leaderboard/live.py b/dashboard/backend/domain/leaderboard/live.py index 31d668535..569a83c66 100644 --- a/dashboard/backend/domain/leaderboard/live.py +++ b/dashboard/backend/domain/leaderboard/live.py @@ -61,8 +61,9 @@ # nodes on a full session, four (10:00 … 13:00) on an early close. _FIRST_NODE_HOUR = _US_CASH_OPEN.hour + 1 _BAR_MINUTES = 60 -# A session joins the freeze only once its closing bar is final on the tape the -# board is priced off. A Basic plan clamps a SIP request to now − delay, so a +# A session joins the freeze only once its closing bar is complete on the tape +# the board is priced off (complete, not immutable: see ``freeze_settle_time``). +# A Basic plan clamps a SIP request to now − delay, so a # refresh at 16:02 would fetch a truncated 15:00 bar and cache that curve as # the day's close. The margin covers late prints and the cron's own jitter. _FREEZE_SETTLE_MARGIN_MINUTES = 15 diff --git a/dashboard/backend/infrastructure/market_data/us_market_calendar.py b/dashboard/backend/infrastructure/market_data/us_market_calendar.py index 0f8c04743..6ec4b788b 100644 --- a/dashboard/backend/infrastructure/market_data/us_market_calendar.py +++ b/dashboard/backend/infrastructure/market_data/us_market_calendar.py @@ -77,6 +77,10 @@ def _observed(day: date) -> date: date(2025, 1, 9), # President Carter }) +#: 13:00 closes NYSE called at short notice. None since 2004; the set exists +#: so the next one is a one-line addition, not a change to the rules. +UNSCHEDULED_EARLY_CLOSES: FrozenSet[date] = frozenset() + @lru_cache(maxsize=16) def nyse_holidays(year: int) -> FrozenSet[date]: @@ -110,7 +114,8 @@ def is_trading_day(day: date) -> bool: @lru_cache(maxsize=16) def nyse_early_closes(year: int) -> FrozenSet[date]: - """NYSE's 13:00 ET early closes in ``year``, by rule. + """NYSE's 13:00 ET early closes in ``year``: by rule, plus any + :data:`UNSCHEDULED_EARLY_CLOSES`. - the day after Thanksgiving; - December 24 when it falls Monday to Thursday (a Friday one is the @@ -119,15 +124,18 @@ def nyse_early_closes(year: int) -> FrozenSet[date]: makes July 3 the observed holiday; a Sunday or Monday one puts July 3 on a weekend). + NYSE Rule 7.2 lists holidays only; early closes are announced year by + year, so these rules restate the announced pattern. They reproduce NYSE's + published 2022-2028 calendars and every year 2004-2035 of + exchange_calendars' XNYS calendar (the test module pins both); a year past + NYSE's latest release is extrapolated. Earlier years followed other rules: + a Thursday July 4 put the half day on Friday July 5 (1996, 2002), and + 1997-12-26, 1999-12-31 and 2003-12-26 were one-off half days. + A candidate that is not a trading day is dropped, so this set can never - contradict ``nyse_holidays``. The rules reproduce NYSE's published 2025-2028 - calendars and every year 2004-2035 of exchange_calendars' XNYS calendar - (the test module pins both). Earlier years followed other rules: a Thursday - July 4 put the half day on Friday July 5 (1996, 2002), and 1997-12-26, - 1999-12-31 and 2003-12-26 were one-off half days. NYSE has called no - unscheduled early close since 2004; if it does, the date needs adding by - hand, and nothing catches its absence: Alpaca still serves bars after an - early close, so the session-gap check sees a normal day. + contradict ``nyse_holidays``. An unscheduled early close missing from + :data:`UNSCHEDULED_EARLY_CLOSES` goes unnoticed: bars still arrive after + the bell, so the Live board's session-gap check sees a normal day. """ candidates = {_nth_weekday(year, 11, 3, 4) + timedelta(days=1)} christmas_eve = date(year, 12, 24) @@ -135,6 +143,7 @@ def nyse_early_closes(year: int) -> FrozenSet[date]: candidates.add(christmas_eve) if 1 <= date(year, 7, 4).weekday() <= 4: candidates.add(date(year, 7, 3)) + candidates.update(day for day in UNSCHEDULED_EARLY_CLOSES if day.year == year) return frozenset(day for day in candidates if is_trading_day(day)) diff --git a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py index 9d676bd3f..65c3cdc3c 100644 --- a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py +++ b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py @@ -2,10 +2,13 @@ The Live board's 2026-11-27 session is why this exists: the calendar knew the holidays but not the 13:00 early closes, so that day would have had seven -chart nodes and three model decisions on late-session bars. Sources: the NYSE -Group releases "2025, 2026 and 2027 Holiday and Early Closings Calendar" -(2024-11-08) and "2026, 2027 and 2028 ..." (2025-12-23), and nyse.com -"Holidays & Trading Hours" (https://www.nyse.com/trade/hours-calendars). +chart nodes and three model decisions on late-session bars. +``PUBLISHED_EARLY_CLOSES`` is hand-copied from NYSE, never generated from the +code under test: the NYSE Group releases "2022, 2023 and 2024 Holiday and +Early Closings Calendar" (2021-12-27), "2025, 2026 and 2027 ..." (2024-11-08) +and "2026, 2027 and 2028 ..." (2025-12-23), and nyse.com "Holidays & Trading +Hours" (https://www.nyse.com/trade/hours-calendars), retrieved 2026-10-15. +Re-pin when NYSE publishes the next year, usually in December. ``XNYS_EARLY_CLOSES`` widens the pin to 2004-2035. It was generated once, offline, from exchange_calendars 4.13.2 (``get_calendar("XNYS").closes``: every @@ -30,6 +33,9 @@ ) PUBLISHED_EARLY_CLOSES = { + 2022: ["2022-11-25"], + 2023: ["2023-07-03", "2023-11-24"], + 2024: ["2024-07-03", "2024-11-29", "2024-12-24"], 2025: ["2025-07-03", "2025-11-28", "2025-12-24"], 2026: ["2026-11-27", "2026-12-24"], 2027: ["2027-11-26"], @@ -121,6 +127,28 @@ def test_unscheduled_closures_are_not_trading_days(): assert is_trading_day(date(2025, 1, 10)) +def test_an_unscheduled_early_close_never_opens_a_closed_day(monkeypatch): + """The dated set is empty today; adding to it must shorten a session, never + create one on a holiday or a weekend.""" + from dashboard.backend.infrastructure.market_data import us_market_calendar + + assert us_market_calendar.UNSCHEDULED_EARLY_CLOSES == frozenset() + monkeypatch.setattr(us_market_calendar, "UNSCHEDULED_EARLY_CLOSES", frozenset({ + date(2026, 3, 12), # a Thursday + date(2026, 4, 3), # Good Friday + date(2026, 3, 14), # a Saturday + })) + nyse_early_closes.cache_clear() + try: + assert nyse_early_closes(2026) == { + date(2026, 3, 12), date(2026, 11, 27), date(2026, 12, 24), + } + assert sessions.session_close_time(date(2026, 3, 12)) == time(13, 0) + assert not is_trading_day(date(2026, 4, 3)) + finally: + nyse_early_closes.cache_clear() + + def test_a_holiday_or_weekend_is_never_an_early_close(): # Independence Day 2026 is observed on Friday July 3; Christmas 2027 on # Friday December 24; December 24, 2028 is a Sunday. From f1119f24bf7f1bb5fa2abf60d6a93ae27593fd83 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 01:02:20 -0400 Subject: [PATCH 12/15] test(baselines): pin every recorded Yahoo early close; name LEAN as a third check - Six more recorded index responses (^DJI and ^GSPC on 2024-11-29, 2024-12-24 and 2025-07-03), so all five early closes from 2024-11-29 to 2025-12-24 are pinned on both indices, each ending on the published close. The 2024 ones leave Yahoo's 730-day hourly window in late 2026 and could not be recorded after that. - The fixture comment no longer says the window filter always drops Yahoo's appended close; it does unless the window reaches the latest session. - LEAN's market-hours database agrees with every early close and holiday 2004-2028 (diffed offline); it is not the pin because of its Sunday 2017-12-24 early close. Co-Authored-By: Claude Opus 5.5 --- .../yahoo-chart-dji-1h-2024-11-29.json | 37 +++++++++++++ .../yahoo-chart-dji-1h-2024-12-24.json | 37 +++++++++++++ .../yahoo-chart-dji-1h-2025-07-03.json | 37 +++++++++++++ .../yahoo-chart-gspc-1h-2024-11-29.json | 37 +++++++++++++ .../yahoo-chart-gspc-1h-2024-12-24.json | 37 +++++++++++++ .../yahoo-chart-gspc-1h-2025-07-03.json | 37 +++++++++++++ .../market_data/test_us_market_calendar.py | 5 ++ dashboard/backend/tests/test_yahoo_chart.py | 52 +++++++++++-------- 8 files changed, 256 insertions(+), 23 deletions(-) create mode 100644 dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2024-11-29.json create mode 100644 dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2024-12-24.json create mode 100644 dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-07-03.json create mode 100644 dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2024-11-29.json create mode 100644 dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2024-12-24.json create mode 100644 dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-07-03.json diff --git a/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2024-11-29.json b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2024-11-29.json new file mode 100644 index 000000000..e2bd2f1f8 --- /dev/null +++ b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2024-11-29.json @@ -0,0 +1,37 @@ +{ + "chart": { + "result": [ + { + "meta": { + "symbol": "^DJI", + "exchangeTimezoneName": "America/New_York", + "instrumentType": "INDEX", + "dataGranularity": "1h" + }, + "timestamp": [ + 1732890600, + 1732894200, + 1732897800, + 1732901400, + 1732903200, + 1792008000 + ], + "indicators": { + "quote": [ + { + "close": [ + 44881.328125, + 44972.83984375, + 45045.609375, + null, + 44910.6484375, + 51621.48046875 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2024-12-24.json b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2024-12-24.json new file mode 100644 index 000000000..f7556f74c --- /dev/null +++ b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2024-12-24.json @@ -0,0 +1,37 @@ +{ + "chart": { + "result": [ + { + "meta": { + "symbol": "^DJI", + "exchangeTimezoneName": "America/New_York", + "instrumentType": "INDEX", + "dataGranularity": "1h" + }, + "timestamp": [ + 1735050600, + 1735054200, + 1735057800, + 1735061400, + 1735063200, + 1792008000 + ], + "indicators": { + "quote": [ + { + "close": [ + 43035.98046875, + 43093.46875, + 43212.8203125, + null, + 43297.03125, + 51621.48046875 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-07-03.json b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-07-03.json new file mode 100644 index 000000000..b961c1d65 --- /dev/null +++ b/dashboard/backend/tests/fixtures/yahoo-chart-dji-1h-2025-07-03.json @@ -0,0 +1,37 @@ +{ + "chart": { + "result": [ + { + "meta": { + "symbol": "^DJI", + "exchangeTimezoneName": "America/New_York", + "instrumentType": "INDEX", + "dataGranularity": "1h" + }, + "timestamp": [ + 1751549400, + 1751553000, + 1751556600, + 1751560200, + 1751562000, + 1792008000 + ], + "indicators": { + "quote": [ + { + "close": [ + 44789.75, + 44807.26953125, + 44823.48828125, + null, + 44828.53125, + 51621.48046875 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2024-11-29.json b/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2024-11-29.json new file mode 100644 index 000000000..61eeb7028 --- /dev/null +++ b/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2024-11-29.json @@ -0,0 +1,37 @@ +{ + "chart": { + "result": [ + { + "meta": { + "symbol": "^GSPC", + "exchangeTimezoneName": "America/New_York", + "instrumentType": "INDEX", + "dataGranularity": "1h" + }, + "timestamp": [ + 1732890600, + 1732894200, + 1732897800, + 1732901400, + 1732903200, + 1792008000 + ], + "indicators": { + "quote": [ + { + "close": [ + 6025.3798828125, + 6034.2998046875, + 6039.18017578125, + null, + 6032.3798828125, + 7718.93994140625 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2024-12-24.json b/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2024-12-24.json new file mode 100644 index 000000000..2b98182be --- /dev/null +++ b/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2024-12-24.json @@ -0,0 +1,37 @@ +{ + "chart": { + "result": [ + { + "meta": { + "symbol": "^GSPC", + "exchangeTimezoneName": "America/New_York", + "instrumentType": "INDEX", + "dataGranularity": "1h" + }, + "timestamp": [ + 1735050600, + 1735054200, + 1735057800, + 1735061400, + 1735063200, + 1792008000 + ], + "indicators": { + "quote": [ + { + "close": [ + 6008.5400390625, + 6018.18994140625, + 6027.759765625, + null, + 6040.0400390625, + 7718.93994140625 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-07-03.json b/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-07-03.json new file mode 100644 index 000000000..d1090eeba --- /dev/null +++ b/dashboard/backend/tests/fixtures/yahoo-chart-gspc-1h-2025-07-03.json @@ -0,0 +1,37 @@ +{ + "chart": { + "result": [ + { + "meta": { + "symbol": "^GSPC", + "exchangeTimezoneName": "America/New_York", + "instrumentType": "INDEX", + "dataGranularity": "1h" + }, + "timestamp": [ + 1751549400, + 1751553000, + 1751556600, + 1751560200, + 1751562000, + 1792008000 + ], + "indicators": { + "quote": [ + { + "close": [ + 6271.8701171875, + 6276.58984375, + 6279.14990234375, + null, + 6279.35009765625, + 7718.93994140625 + ] + } + ] + } + } + ], + "error": null + } +} diff --git a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py index 65c3cdc3c..737c72576 100644 --- a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py +++ b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py @@ -17,6 +17,11 @@ sessions that already happened; for 2026-2035 it is a second, independent implementation of the same rules, so it catches a slip in ours rather than a change in NYSE's policy. That only NYSE's next release can show. + +A third calendar agrees too: QuantConnect LEAN's market-hours database +(``Equity-usa``) matches every early close and holiday here for 2004-2028 +(diffed 2026-10-15). It is not the pin because it carries a Sunday early +close, 2017-12-24. """ from datetime import date, time, timedelta diff --git a/dashboard/backend/tests/test_yahoo_chart.py b/dashboard/backend/tests/test_yahoo_chart.py index 32853c8e4..67bd9caf9 100644 --- a/dashboard/backend/tests/test_yahoo_chart.py +++ b/dashboard/backend/tests/test_yahoo_chart.py @@ -168,14 +168,16 @@ def test_yahoo_chart_error_is_a_request_exception(): # The fixtures are real ^DJI and ^GSPC responses to the request # fetch_index_hourly sends (period1 = the day's 00:00Z, period2 = +1 day, # interval=1h), recorded 2026-10-15 and trimmed to ``meta``'s relevant fields -# and the closes (Yahoo serves 1h data for the last 730 days only). The -# trailing 2026-10-14T20:00Z point is Yahoo appending the latest close to every -# hourly response, which the window filter drops. On 2025-11-28 (13:00 close) -# the 12:30 half bar comes back null and the closing level, 47,716.42, arrives -# stamped 13:00 (18:00Z); the full session of 2025-11-26 has no such point. The -# same shape was recorded for ^DJI and ^GSPC on every NYSE early close from -# 2024-11-29 to 2025-12-24, and Yahoo's own ``tradingPeriods`` gives -# 2025-11-28 a 14:30Z-18:00Z regular session. +# and the closes (Yahoo serves 1h data for the last 730 days only, so the +# 2024 ones could not be recorded after late 2026). The trailing +# 2026-10-14T20:00Z point is Yahoo appending the latest close to every hourly +# response; the window filter drops it unless the window reaches the latest +# session. On 2025-11-28 (13:00 close) the 12:30 half bar comes back null and +# the closing level, 47,716.42, arrives stamped 13:00 (18:00Z); the full +# session of 2025-11-26 has no such point. Every NYSE early close from +# 2024-11-29 to 2025-12-24 has that shape on both indices (all five are +# pinned below), and Yahoo's own ``tradingPeriods`` gives 2025-11-28 a +# 14:30Z-18:00Z regular session. _FIXTURES = Path(__file__).parent / "fixtures" _NEW_YORK = ZoneInfo("America/New_York") @@ -200,21 +202,25 @@ def test_an_early_close_level_lands_in_the_last_half_bar(monkeypatch): ] -@pytest.mark.parametrize( - ("fixture", "symbol", "day", "official_close"), - [ - ("yahoo-chart-dji-1h-2025-11-28.json", "^DJI", "2025-11-28", 47716.421875), - ("yahoo-chart-gspc-1h-2025-11-28.json", "^GSPC", "2025-11-28", 6849.08984375), - ("yahoo-chart-dji-1h-2025-12-24.json", "^DJI", "2025-12-24", 48731.16015625), - ("yahoo-chart-gspc-1h-2025-12-24.json", "^GSPC", "2025-12-24", 6932.0498046875), - ], -) -def test_each_recorded_early_close_ends_on_the_official_close( - monkeypatch, fixture, symbol, day, official_close -): - """Each level is that day's published close, equal to Yahoo's own daily - bar: 47,716.42 and 6,849.09 on 2025-11-28, 48,731.16 and 6,932.05 on - 2025-12-24.""" +#: Each early close's published close, equal to Yahoo's own daily bar. +_OFFICIAL_CLOSES = { + ("^DJI", "2024-11-29"): 44910.6484375, + ("^GSPC", "2024-11-29"): 6032.3798828125, + ("^DJI", "2024-12-24"): 43297.03125, + ("^GSPC", "2024-12-24"): 6040.0400390625, + ("^DJI", "2025-07-03"): 44828.53125, + ("^GSPC", "2025-07-03"): 6279.35009765625, + ("^DJI", "2025-11-28"): 47716.421875, + ("^GSPC", "2025-11-28"): 6849.08984375, + ("^DJI", "2025-12-24"): 48731.16015625, + ("^GSPC", "2025-12-24"): 6932.0498046875, +} + + +@pytest.mark.parametrize(("symbol", "day"), sorted(_OFFICIAL_CLOSES)) +def test_each_recorded_early_close_ends_on_the_official_close(monkeypatch, symbol, day): + fixture = f"yahoo-chart-{symbol[1:].lower()}-1h-{day}.json" + official_close = _OFFICIAL_CLOSES[(symbol, day)] _stub_response(monkeypatch, _recorded(fixture)) points = fetch_index_hourly(symbol, day, day) assert [clock for clock, _value in _clock(points)] == [ From 9919eecca8f096791e4312404857d1385b7146fe Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 01:21:06 -0400 Subject: [PATCH 13/15] fix(baselines): report a missing early-close index level; hedge unobserved claims Review follow-ups: - If Yahoo ever drops or shifts its 13:00 point, the restamp does nothing and the board's 13:00 node repeats the 11:30 level: plausible and wrong. One flushed ERROR (yahoo.early_close_level_missing) per settled early close with no level at its last half bar; a day within an hour of its close is skipped. Pinned with the recorded response minus that point, and the ten recorded early closes must stay silent. - Docstrings no longer state as observed what nobody has seen on an early close (Alpaca's late-session bars, the 13:00 auction print's bar); the 16:00 case is the measured one. - The market-local-time test now bites: a naive 12:00 stamp must be kept, which reading it as UTC would not. Co-Authored-By: Claude Opus 5.5 --- .../domain/leaderboard/strategies/_yahoo.py | 50 ++++++++++++++++--- .../infrastructure/market_data/sessions.py | 23 +++++---- .../backend/tests/test_market_sessions.py | 20 +++++--- dashboard/backend/tests/test_yahoo_chart.py | 50 ++++++++++++++++++- 4 files changed, 117 insertions(+), 26 deletions(-) diff --git a/dashboard/backend/domain/leaderboard/strategies/_yahoo.py b/dashboard/backend/domain/leaderboard/strategies/_yahoo.py index 3f578ff01..3d4264a49 100644 --- a/dashboard/backend/domain/leaderboard/strategies/_yahoo.py +++ b/dashboard/backend/domain/leaderboard/strategies/_yahoo.py @@ -131,7 +131,19 @@ def fetch_index_hourly( if close is None or not (start_e <= ts < end_e): continue points.append((dt.datetime.fromtimestamp(ts, dt.timezone.utc), float(close))) - return _restamp_close_levels(points, (res.get("meta") or {}).get("exchangeTimezoneName")) + exchange_zone = (res.get("meta") or {}).get("exchangeTimezoneName") + points = _restamp_close_levels(points, exchange_zone) + _report_missing_close_levels(symbol, points, exchange_zone) + return points + + +def _last_slot_open(local: dt.datetime) -> dt.datetime: + """The open of the last grid slot ending at or before ``local``, on Yahoo's + hourly grid from the 09:30 open (12:30 for a 13:00 close, 15:30 for 16:00).""" + session_open = session_windows("US")[0][0] + opened = dt.datetime.combine(local.date(), session_open, tzinfo=local.tzinfo) + hour = dt.timedelta(hours=1) + return local - ((local - opened) % hour or hour) def _restamp_close_levels( @@ -167,16 +179,42 @@ def _restamp_close_levels( if exchange_zone != _US_EXCHANGE_ZONE: return points zone = ZoneInfo(_US_EXCHANGE_ZONE) - session_open = session_windows("US")[0][0] - hour = dt.timedelta(hours=1) by_stamp: Dict[dt.datetime, float] = {} for stamp, level in points: local = stamp.astimezone(zone) if is_early_close(local.date()) and local.time() == session_close_time( local.date(), "US" ): - opened = dt.datetime.combine(local.date(), session_open, tzinfo=zone) - last_slot = (local - opened) % hour or hour - stamp = (local - last_slot).astimezone(dt.timezone.utc) + stamp = _last_slot_open(local).astimezone(dt.timezone.utc) by_stamp[stamp] = level return sorted(by_stamp.items()) + + +def _report_missing_close_levels( + symbol: str, + points: List[Tuple[dt.datetime, float]], + exchange_zone: Optional[str], + now: Optional[dt.datetime] = None, +) -> None: + """One flushed ERROR per settled NYSE early close that has no closing level. + + The restamp depends on Yahoo's half-day shape. If Yahoo stops sending the + 13:00 point, or stamps it a second off, the day ends on its 11:30 bar and + the board's 13:00 node repeats that level: a plausible curve, wrong, with + nothing else to notice it. A day still within an hour of its close is + skipped, since Yahoo may not have sent the level yet. + """ + if exchange_zone != _US_EXCHANGE_ZONE: + return + zone = ZoneInfo(_US_EXCHANGE_ZONE) + now = now or dt.datetime.now(dt.timezone.utc) + stamps = {stamp.astimezone(zone) for stamp, _level in points} + for day in sorted({stamp.date() for stamp in stamps if is_early_close(stamp.date())}): + close = dt.datetime.combine(day, session_close_time(day, "US"), tzinfo=zone) + if now < close + dt.timedelta(hours=1) or _last_slot_open(close) in stamps: + continue + print( + f"ERROR: yahoo.early_close_level_missing symbol={symbol} date={day} " + "(no level at the last half bar; the index curve ends on its 11:30 bar)", + flush=True, + ) diff --git a/dashboard/backend/infrastructure/market_data/sessions.py b/dashboard/backend/infrastructure/market_data/sessions.py index 4ed82ce7c..515b79e29 100644 --- a/dashboard/backend/infrastructure/market_data/sessions.py +++ b/dashboard/backend/infrastructure/market_data/sessions.py @@ -172,20 +172,21 @@ def is_in_session( first fill off a pre-market print. The day's last session ends at :func:`session_close_time`: 13:00 ET on an - NYSE early close. Trading goes on after that bell (the NYSE Arca, - American, National and Texas late sessions, FINRA TRF post-market - reporting and Alpaca's half-day extended session all run to 17:00), so - bars stamped 13:00 and later can carry real prints: the rule drops them by - their stamp, never by their absence. Only this open-stamped rule reads the - day's close; the close-stamped rule (:func:`time_in_session`) still ends - every US day at 16:00 (D19, owned by the engine rewrite). + NYSE early close. Trading goes on after that bell (NYSE's late sessions, + FINRA TRF post-market reporting and Alpaca's half-day extended session + all run to 17:00), so bars stamped 13:00 and later can carry real prints: + the rule drops them by their stamp, never by their absence. Only this + open-stamped rule reads the day's close; the close-stamped rule + (:func:`time_in_session`) still ends every US day at 16:00 (D19, owned by + the engine rewrite). The close mark this leaves is pre-auction on every US day. The last kept bar (the 12:00 bar on an early close, 15:00 otherwise) closes on the last - last-sale-eligible trade before the bell, not the official close: the - closing-auction print, stamped at or just after 13:00:00 (16:00:00), opens - the next bar, which is dropped. Marking on that bar instead was rejected, - because post-close prints in the same hour can set its close. + trade before the bell, not the official close: the closing-auction print, + stamped at or just after 16:00:00, opens the next bar, which is dropped + (measured on SIP), and the 13:00 auction is expected to do the same on an + early close. Marking on that bar instead was rejected, because post-close + prints in the same hour can set its close. """ local = market_local(timestamp, timezone) if open_stamped_minutes is None: diff --git a/dashboard/backend/tests/test_market_sessions.py b/dashboard/backend/tests/test_market_sessions.py index 568753ab3..d4b454e47 100644 --- a/dashboard/backend/tests/test_market_sessions.py +++ b/dashboard/backend/tests/test_market_sessions.py @@ -193,12 +193,13 @@ def _open_stamped_kept(stamps, minutes=60): def test_open_stamped_hourly_early_close_is_three_bars(): - """2026-11-27 closes at 13:00 ET. Alpaca serves the late session as bars + """2026-11-27 closes at 13:00 ET. Alpaca can serve the late session as bars stamped 13:00, 14:00 and 15:00, and a fixed 16:00 close kept them: three - decisions taken after the session had ended. On SIP the 13:00 bar opens - with the closing auction, as the 16:00 bar does on a full day; the 14:00 - and 15:00 bars are late-session prints (NYSE Arca and others trade to - 17:00 on an early close).""" + decisions taken after the session had ended. On SIP the 13:00 bar is + expected to open with the closing auction, as the 16:00 bar does on a full + day (measured there), and the 14:00 and 15:00 bars to hold late-session + prints (NYSE's late sessions run to 17:00 on an early close). Neither is + observed yet on an early close; the rule drops the bars either way.""" day = pd.date_range("2026-11-27 04:00", "2026-11-27 19:00", freq="h", tz="US/Eastern") assert _open_stamped_kept(day) == ["10:00", "11:00", "12:00"] @@ -277,9 +278,11 @@ def test_the_open_stamped_rule_matches_a_naive_oracle_every_day(minutes, sweeps) assert early_closes == 8 -def test_the_early_close_is_read_on_the_market_local_date(): - """18:00Z on 2026-11-27 is 13:00 ET, after that day's close; a naive stamp - is market-local, as everywhere else in this module.""" +def test_the_early_close_is_read_in_market_local_time(): + """17:00Z on 2026-11-27 is the 12:00 ET bar and 18:00Z the 13:00 one, after + that day's close. A naive stamp is market-local, as everywhere else in this + module: naive 12:00 is the last bar (read as UTC it would be 07:00 ET, before + the open), naive 13:00 is past the close.""" def kept(ts): return sessions.is_in_session( ts, market="US", timezone="US/Eastern", open_stamped_minutes=60 @@ -287,6 +290,7 @@ def kept(ts): assert kept(pd.Timestamp("2026-11-27 17:00", tz="UTC")) assert not kept(pd.Timestamp("2026-11-27 18:00", tz="UTC")) + assert kept(pd.Timestamp("2026-11-27 12:00")) assert not kept(pd.Timestamp("2026-11-27 13:00")) diff --git a/dashboard/backend/tests/test_yahoo_chart.py b/dashboard/backend/tests/test_yahoo_chart.py index 67bd9caf9..d16c979e3 100644 --- a/dashboard/backend/tests/test_yahoo_chart.py +++ b/dashboard/backend/tests/test_yahoo_chart.py @@ -218,7 +218,9 @@ def test_an_early_close_level_lands_in_the_last_half_bar(monkeypatch): @pytest.mark.parametrize(("symbol", "day"), sorted(_OFFICIAL_CLOSES)) -def test_each_recorded_early_close_ends_on_the_official_close(monkeypatch, symbol, day): +def test_each_recorded_early_close_ends_on_the_official_close( + monkeypatch, capsys, symbol, day +): fixture = f"yahoo-chart-{symbol[1:].lower()}-1h-{day}.json" official_close = _OFFICIAL_CLOSES[(symbol, day)] _stub_response(monkeypatch, _recorded(fixture)) @@ -227,6 +229,52 @@ def test_each_recorded_early_close_ends_on_the_official_close(monkeypatch, symbo "09:30", "10:30", "11:30", "12:30", ] assert points[-1][1] == official_close + assert "ERROR" not in capsys.readouterr().out + + +def _without_the_close_point(payload): + """The 2025-11-28 response as it would look if Yahoo dropped its 13:00 point.""" + result = payload["chart"]["result"][0] + at = result["timestamp"].index(1764352800) # 2025-11-28T18:00Z, 13:00 ET + del result["timestamp"][at] + del result["indicators"]["quote"][0]["close"][at] + return payload + + +def test_a_missing_early_close_level_is_reported(monkeypatch, capsys): + """Fail-visible: without the 13:00 point the day ends on its 11:30 bar and + the board's 13:00 node repeats it, a plausible and wrong curve.""" + _stub_response( + monkeypatch, _without_the_close_point(_recorded("yahoo-chart-dji-1h-2025-11-28.json")) + ) + points = fetch_index_hourly("^DJI", "2025-11-28", "2025-11-28") + assert [clock for clock, _value in _clock(points)] == ["09:30", "10:30", "11:30"] + assert ( + "ERROR: yahoo.early_close_level_missing symbol=^DJI date=2025-11-28" + in capsys.readouterr().out + ) + + +def test_an_early_close_within_its_settle_hour_is_not_reported(capsys): + from dashboard.backend.domain.leaderboard.strategies._yahoo import ( + _report_missing_close_levels, + ) + + morning = [ + (dt.datetime(2025, 11, 28, 14 + i, 30, tzinfo=dt.timezone.utc), 1.0) + for i in range(3) + ] + for hour, minute in ((13, 30), (13, 59)): + _report_missing_close_levels( + "^DJI", morning, "America/New_York", + now=dt.datetime(2025, 11, 28, hour, minute, tzinfo=_NEW_YORK), + ) + assert capsys.readouterr().out == "" + _report_missing_close_levels( + "^DJI", morning, "America/New_York", + now=dt.datetime(2025, 11, 28, 14, 0, tzinfo=_NEW_YORK), + ) + assert "date=2025-11-28" in capsys.readouterr().out def test_the_early_close_level_is_the_live_boards_13_00_node(monkeypatch): From 1acb75dc71c6ab3e1d425a4beb16ee6aef69fb68 Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 12:44:19 -0400 Subject: [PATCH 14/15] fix(sessions): bucket and fill an early close at 13:00, not only filter it Only the open-stamped filter knew the 13:00 close. Aggregation still built a 13:30 bucket from 12:30-12:59 and is_session_close still matched 16:00 only, so on an early close a minute-source run lost its closing decision. day_session_windows is now the per-day owner read by is_in_session, is_session_close and aggregate_bars. Drops REGULAR_CLOSE_TIME (unused; the CodeQL note) and decides US by market name, not by window shape. Conformance T05 now fails as T02 does (D2/D16) instead of losing the order (D19/D8). Co-Authored-By: Claude Opus 5.5 --- .../domain/backtesting/bar_aggregation.py | 12 ++- .../infrastructure/market_data/sessions.py | 95 +++++++++++-------- .../tests/backtesting/test_bar_aggregation.py | 72 ++++++++++++++ dashboard/backend/tests/conformance/cases.py | 5 +- .../tests/conformance/known_failures.json | 16 +--- .../test_conformance_current_engine.py | 9 +- .../market_data/test_us_market_calendar.py | 38 +++++++- .../backend/tests/test_market_sessions.py | 15 ++- .../2026-10-07-us-conformance-suite-design.md | 4 +- 9 files changed, 196 insertions(+), 70 deletions(-) diff --git a/dashboard/backend/domain/backtesting/bar_aggregation.py b/dashboard/backend/domain/backtesting/bar_aggregation.py index 50ca7e47b..c1a6d8959 100644 --- a/dashboard/backend/domain/backtesting/bar_aggregation.py +++ b/dashboard/backend/domain/backtesting/bar_aggregation.py @@ -27,6 +27,7 @@ # Re-exported: the session bounds have one owner, and it is not this module. from dashboard.backend.infrastructure.market_data.sessions import ( FRAME_ATTR_OPEN_STAMPED_MINUTES, + day_session_windows, is_session_close, market_local, session_windows, @@ -129,7 +130,10 @@ def aggregate_bars( ) local = _as_local_index(frame, timezone) - windows = session_windows(market) + # Each day's own sessions, so an NYSE early close buckets 12:30-13:00 as + # its final (half) bar, stamped 13:00, which ``plan_execution_fills`` then + # fills at the session close. Resolved once per date, not per source bar. + windows_by_day: dict[date, tuple[tuple[time, time], ...]] = {} # Walk the index, not the rows: iterrows builds a Series per source bar, # and the onboarding shape has ~16k of them (7 weekdays x 78 five-minute # bars x 30 symbols). That is the cheapest thing here to remove, not the @@ -153,6 +157,10 @@ def aggregate_bars( # `.iloc[0]` of each group and discarded the rest. ends_by_start: dict[pd.Timestamp, pd.Timestamp] = {} for timestamp in local.index: + day = timestamp.date() + windows = windows_by_day.get(day) + if windows is None: + windows = windows_by_day[day] = day_session_windows(day, market) session = _session_for_timestamp(timestamp, windows) if session is None: keep.append(False) @@ -373,7 +381,7 @@ def plan_execution_fills( kind :func:`aggregate_bars` accepts), sorted. A decision closes at its stamp, so it fills at the ``open`` of the source bar opening at that instant -- the first fill without look-ahead. A session's final bucket - (16:00 ET) has no such bar once after-hours bars are out of the source set, + (16:00 ET, 13:00 on an NYSE early close) has no such bar once after-hours bars are out of the source set, so it fills at the ``close`` of the source bar ending at that instant, priced at the last regular-hours trade and stamped at that bar's close rather than its open, so the trade never predates the decision. diff --git a/dashboard/backend/infrastructure/market_data/sessions.py b/dashboard/backend/infrastructure/market_data/sessions.py index 515b79e29..6e401a7da 100644 --- a/dashboard/backend/infrastructure/market_data/sessions.py +++ b/dashboard/backend/infrastructure/market_data/sessions.py @@ -40,39 +40,58 @@ def canonical_market(market: object) -> str: return str(market or DEFAULT_MARKET).strip().upper() or DEFAULT_MARKET -def session_windows(market: object) -> tuple[tuple[time, time], ...]: - """The market's trading sessions, in its own local time. Unknown -> US.""" - if canonical_market(market) == "CN": - return ((time(9, 30), time(11, 30)), (time(13, 0), time(15, 0))) - return ((time(9, 30), time(16, 0)),) - +_US_SESSIONS: tuple[tuple[time, time], ...] = ((time(9, 30), time(16, 0)),) +_CN_SESSIONS: tuple[tuple[time, time], ...] = ( + (time(9, 30), time(11, 30)), + (time(13, 0), time(15, 0)), +) -#: The US cash close on a full session. Read off ``session_windows`` rather -#: than restated, so the bound keeps one owner. -REGULAR_CLOSE_TIME: time = session_windows(DEFAULT_MARKET)[-1][1] #: The US cash close on an NYSE early-close day #: (``us_market_calendar.nyse_early_closes``). EARLY_CLOSE_TIME: time = time(13, 0) -def session_close_time(day: date, market: object = DEFAULT_MARKET) -> time: - """When ``market``'s last session ends on ``day``, in its own local time. +def _trades_us_sessions(market: object) -> bool: + """Whether ``market`` trades the US cash session. Unknown -> US.""" + return canonical_market(market) != "CN" + + +def session_windows(market: object) -> tuple[tuple[time, time], ...]: + """The market's trading sessions on a full day, in its own local time. + Unknown -> US. For a particular date, use :func:`day_session_windows`: + an NYSE early close ends the US session at 13:00.""" + return _US_SESSIONS if _trades_us_sessions(market) else _CN_SESSIONS - A market on US sessions closes at :data:`EARLY_CLOSE_TIME` on the NYSE - early-close days the calendar lists and at :data:`REGULAR_CLOSE_TIME` - otherwise; any other market closes at its last window's end. Whether - ``day`` trades at all is the calendar's question + +def day_session_windows(day: date, market: object) -> tuple[tuple[time, time], ...]: + """:func:`session_windows` for ``day``: a US day ends at + :data:`EARLY_CLOSE_TIME` on the NYSE early-close days the calendar lists. + Any other market's sessions do not vary by date here. Whether ``day`` + trades at all is the calendar's question (``us_market_calendar.is_trading_day``), not this one's. - The close-stamped helpers (:func:`time_in_session`, :func:`session_windows`, - :func:`is_session_close`) do not consult it: bar aggregation and the - engine still end every US day at 16:00. That gap is D19 in the US - conformance design, and the engine rewrite owns it. + Read by every rule that places a bar against the day's close: the + open-stamped filter (:func:`is_in_session`), the session-close fill + (:func:`is_session_close`) and the bucketing in + ``bar_aggregation.aggregate_bars``. They have to agree. When only the + filter knew about the 13:00 close, aggregation still built a 13:30 bucket + from the 12:30-12:59 minutes, nothing filled it, and an early-close day lost + its closing decision. The close-stamped rule (:func:`time_in_session`) does + not read it and still admits a bar stamped up to 16:00 (D19 in the US + conformance design, owned by the engine rewrite). No US source this repo + loads is close-stamped, and aggregation no longer produces a decision bar + after 13:00 on an early close, so nothing reaches that gap today. """ - windows = session_windows(market) - if windows == session_windows(DEFAULT_MARKET) and is_early_close(day): - return EARLY_CLOSE_TIME - return windows[-1][1] + if _trades_us_sessions(market) and is_early_close(day): + return ((_US_SESSIONS[0][0], EARLY_CLOSE_TIME),) + return session_windows(market) + + +def session_close_time(day: date, market: object = DEFAULT_MARKET) -> time: + """When ``market``'s last session ends on ``day``, in its own local time: + 16:00 ET, or :data:`EARLY_CLOSE_TIME` on an NYSE early close, for US + sessions; the last window's end for any other market.""" + return day_session_windows(day, market)[-1][1] def timezone_for_market(market: object) -> str: @@ -171,14 +190,13 @@ def is_in_session( half an hour of pre-market trading, so admitting it would price the day's first fill off a pre-market print. - The day's last session ends at :func:`session_close_time`: 13:00 ET on an - NYSE early close. Trading goes on after that bell (NYSE's late sessions, + The day's sessions come from :func:`day_session_windows`, so the last one + ends at 13:00 ET on an NYSE early close. Trading goes on after that bell (NYSE's late sessions, FINRA TRF post-market reporting and Alpaca's half-day extended session all run to 17:00), so bars stamped 13:00 and later can carry real prints: - the rule drops them by their stamp, never by their absence. Only this - open-stamped rule reads the day's close; the close-stamped rule - (:func:`time_in_session`) still ends every US day at 16:00 (D19, owned by - the engine rewrite). + the rule drops them by their stamp, never by their absence. The + close-stamped rule (:func:`time_in_session`) still ends every US day at + 16:00 (see :func:`day_session_windows`). The close mark this leaves is pre-auction on every US day. The last kept bar (the 12:00 bar on an early close, 15:00 otherwise) closes on the last @@ -196,21 +214,18 @@ def is_in_session( return False open_minute = local.time().replace(second=0, microsecond=0) close_minute = close.time().replace(second=0, microsecond=0) - windows = session_windows(market) - windows = windows[:-1] + ( - (windows[-1][0], session_close_time(local.date(), market)), - ) return any( start <= open_minute and close_minute <= end - for start, end in windows + for start, end in day_session_windows(local.date(), market) ) def is_session_close(timestamp: datetime, *, market: object, timezone: str) -> bool: - """Whether ``timestamp`` falls in the minute a session ends (16:00 ET, - 11:30 and 15:00 CST): the stamp of a final bucket, which no in-session - source bar opens at.""" - minute = market_local(timestamp, timezone).time().replace( - second=0, microsecond=0 + """Whether ``timestamp`` falls in the minute one of that day's sessions + ends (16:00 ET, 13:00 on an NYSE early close; 11:30 and 15:00 CST): the + stamp of a final bucket, which no in-session source bar opens at.""" + local = market_local(timestamp, timezone) + minute = local.time().replace(second=0, microsecond=0) + return any( + minute == end for _start, end in day_session_windows(local.date(), market) ) - return any(minute == end for _start, end in session_windows(market)) diff --git a/dashboard/backend/tests/backtesting/test_bar_aggregation.py b/dashboard/backend/tests/backtesting/test_bar_aggregation.py index 552570740..f99de1d06 100644 --- a/dashboard/backend/tests/backtesting/test_bar_aggregation.py +++ b/dashboard/backend/tests/backtesting/test_bar_aggregation.py @@ -251,3 +251,75 @@ def test_aggregated_bars_are_stamped_at_their_close(): assert FRAME_ATTR_OPEN_STAMPED_MINUTES not in result.attrs assert result.attrs["alpaca_feed"] == "sip" + + +def test_an_early_close_buckets_and_fills_on_its_13_00_close(): + """2026-11-27 closes at 13:00 ET. Alpaca serves prints after the bell (NYSE + late sessions and post-market reporting run to 17:00), so the source frame + reaches 16:55. The last decision bar is the 12:30-13:00 half bucket, filled + at the close of the 12:55 bar. Bucketing on the full-day session built a + 13:30 bucket from 12:30-12:59 instead, which no source bar could fill once + the open-stamped filter dropped everything from 13:00: the day lost its + closing decision.""" + from dashboard.backend.domain.backtesting.bar_aggregation import ( + plan_execution_fills, + ) + from dashboard.backend.infrastructure.market_data.sessions import is_in_session + + eastern = pytz.timezone("US/Eastern") + timestamps = pd.date_range( + eastern.localize(datetime(2026, 11, 27, 9, 30)), + eastern.localize(datetime(2026, 11, 27, 16, 55)), + freq="5min", + ) + result = aggregate_bars( + _bars(timestamps), + source_timeframe="5m", + decision_timeframe="60m", + market="US", + timezone="US/Eastern", + ) + local = [stamp.tz_convert(eastern).strftime("%H:%M") for stamp in result.index] + assert local == ["10:30", "11:30", "12:30", "13:00"] + assert result.iloc[-1]["expected_source_bars"] == 6 + assert bool(result.iloc[-1]["is_complete"]) is True + + source = [ + stamp + for stamp in timestamps + if is_in_session( + stamp, market="US", timezone="US/Eastern", open_stamped_minutes=5 + ) + ] + assert source[-1] == eastern.localize(datetime(2026, 11, 27, 12, 55)) + fills = plan_execution_fills( + result.index, + source, + source_minutes=5, + market="US", + timezone="US/Eastern", + ) + closing = fills[result.index[-1]] + assert closing.bar == source[-1] + assert closing.price_field == "close" + assert closing.filled_at == result.index[-1] + # The three hourly decisions fill at the open of the bar starting then. + assert all(fills[stamp].price_field == "open" for stamp in result.index[:-1]) + + +def test_a_full_session_still_ends_its_buckets_at_16_00(): + eastern = pytz.timezone("US/Eastern") + timestamps = pd.date_range( + eastern.localize(datetime(2026, 11, 25, 9, 30)), + eastern.localize(datetime(2026, 11, 25, 16, 55)), + freq="5min", + ) + result = aggregate_bars( + _bars(timestamps), + source_timeframe="5m", + decision_timeframe="60m", + market="US", + timezone="US/Eastern", + ) + assert result.index[-1].tz_convert(eastern).strftime("%H:%M") == "16:00" + assert len(result) == 7 diff --git a/dashboard/backend/tests/conformance/cases.py b/dashboard/backend/tests/conformance/cases.py index 3e109b55a..0d9f4b57e 100644 --- a/dashboard/backend/tests/conformance/cases.py +++ b/dashboard/backend/tests/conformance/cases.py @@ -1112,8 +1112,9 @@ def _p(verdict, fails=(), defects=(), note=""): "a decision with no fill bar is not a step, so o1 leaves no record (engine.py:1949)"), "T04": _p(FAIL, ("orders", "invariants"), ("D8",), "a fill with no source bar hits a bare continue, no order event (execution.py:526-527)"), - "T05": _p(FAIL, ("fills", "orders", "cash", "positions", "equity", "invariants"), ("D19", "D8"), - "the session is hard-coded to 16:00, so the 13:00 bucket never fills (sessions.py:40-44)"), + "T05": _p(FAIL, ("fills", "cash", "equity"), ("D2", "D16"), + "the 13:00 bucket fills at the 12:55 bar's close, as T02's 16:00 bucket does " + "(bar_aggregation.plan_execution_fills); D19's lost decision is gone since #638"), "C01": _p(FAIL, ("fills", "orders", "cash", "positions"), ("D5",), "actions run in listed order; the buy is rejected before the sell funds it (execution.py:442)"), "C02": _p(FAIL, ("fills", "orders", "cash", "positions"), ("D6", "D8"), diff --git a/dashboard/backend/tests/conformance/known_failures.json b/dashboard/backend/tests/conformance/known_failures.json index 1d1315268..fbd524916 100644 --- a/dashboard/backend/tests/conformance/known_failures.json +++ b/dashboard/backend/tests/conformance/known_failures.json @@ -329,22 +329,14 @@ }, "T05/D": { "cash": [ - "cash: expected 490, actual 1000" + "cash: expected 490, actual 500" ], "equity": [ - "equity[2026-11-30T10:30:00-05:00]: expected 1005, actual 1000" + "equity[2026-11-30T10:30:00-05:00]: expected 1005, actual 1015" ], "fills": [ - "fills[o1@2026-11-30T09:30:00-05:00]: expected price 102 qty 5, actual no fill on that bar" - ], - "invariants": [ - "invariant 7: o1 has 0 terminal OrderFinal(s), expected 1" - ], - "orders": [ - "orders[o1]: expected filled/- filled 5, actual no terminal OrderFinal" - ], - "positions": [ - "positions[AAPL]: expected 5, actual not held" + "fills[o1@2026-11-30T09:30:00-05:00]: expected price 102 qty 5, actual no fill on that bar", + "fills[o1@2026-11-27T12:30:00-05:00]: expected no fill, actual price 100 qty 5" ] } } diff --git a/dashboard/backend/tests/conformance/test_conformance_current_engine.py b/dashboard/backend/tests/conformance/test_conformance_current_engine.py index 3b9c222d0..1edf7929d 100644 --- a/dashboard/backend/tests/conformance/test_conformance_current_engine.py +++ b/dashboard/backend/tests/conformance/test_conformance_current_engine.py @@ -169,10 +169,9 @@ def test_synthesized_bars_aggregate_back_to_the_case(case): """§8.1, pinned: an adapter bug must never be scored as an engine defect. The engine's own aggregation of the synthesized 5m bars reproduces every - hourly case bar exactly, and spans every daily one with its O/H/L/C. The - one exception is T05's early-close bucket: the engine's session is fixed - at 16:00 (D19), so a 12:30-13:00 bucket is an incomplete 12:30-13:30 bar - there -- which is the defect T05 measures, not a synthesis error. + hourly case bar exactly, and spans every daily one with its O/H/L/C. That + includes T05's 12:30-13:00 half bucket: since #638 the aggregation ends an + NYSE early close at 13:00 (``sessions.day_session_windows``). """ hourly = aggregate_bars_by_symbol( synthesize_source_bars(case), source_timeframe="5m", decision_timeframe="60m" @@ -185,8 +184,6 @@ def test_synthesized_bars_aggregate_back_to_the_case(case): day = frame.loc[(frame.index > pd.Timestamp(bar.ts).tz_convert("UTC")) & (frame.index <= close)] assert len(day) == 7 and day["is_complete"].all(), bar got = [day["open"].iloc[0], day["high"].max(), day["low"].min(), day["close"].iloc[-1]] - elif bar.minutes == 30 and bar.close_ts.hour != 16: - continue else: row = frame.loc[close] assert bool(row["is_complete"]), bar diff --git a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py index 737c72576..d711baffb 100644 --- a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py +++ b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py @@ -180,8 +180,42 @@ def test_is_early_close(): def test_the_close_times_have_one_owner(): assert sessions.EARLY_CLOSE_TIME == time(13, 0) - assert sessions.REGULAR_CLOSE_TIME == time(16, 0) - assert sessions.REGULAR_CLOSE_TIME == sessions.session_windows("US")[-1][1] + # A full day's close is the session table's, not a second constant. + assert not hasattr(sessions, "REGULAR_CLOSE_TIME") + assert ( + sessions.session_close_time(date(2026, 11, 25), "US") + == sessions.session_windows("US")[-1][1] + == time(16, 0) + ) + + +@pytest.mark.parametrize( + ("day", "market", "windows"), + [ + (date(2026, 11, 27), "US", ((time(9, 30), time(13, 0)),)), + (date(2026, 11, 25), "US", ((time(9, 30), time(16, 0)),)), + # An unknown market trades US sessions, early closes included. + (date(2026, 11, 27), "XX", ((time(9, 30), time(13, 0)),)), + ( + date(2026, 11, 27), + "cn", + ((time(9, 30), time(11, 30)), (time(13, 0), time(15, 0))), + ), + ], +) +def test_day_session_windows(day, market, windows): + assert sessions.day_session_windows(day, market) == windows + + +def test_is_session_close_reads_the_days_close(): + from datetime import datetime + + def close(hour, day): + stamp = datetime(2026, 11, day, hour, 0) + return sessions.is_session_close(stamp, market="US", timezone="US/Eastern") + + assert close(13, 27) and not close(16, 27) + assert close(16, 25) and not close(13, 25) @pytest.mark.parametrize( diff --git a/dashboard/backend/tests/test_market_sessions.py b/dashboard/backend/tests/test_market_sessions.py index d4b454e47..5ed4ab2cb 100644 --- a/dashboard/backend/tests/test_market_sessions.py +++ b/dashboard/backend/tests/test_market_sessions.py @@ -294,10 +294,12 @@ def kept(ts): assert not kept(pd.Timestamp("2026-11-27 13:00")) -def test_an_early_close_leaves_the_close_stamped_path_alone(): - """Bar aggregation and the engine's close-stamped filter still end every US - day at 16:00. That is D19, measured by conformance T05 and owned by the - engine rewrite; only the open-stamped rule reads the day's close.""" +def test_an_early_close_leaves_only_the_close_stamped_rule_at_16_00(): + """The close-stamped filter still admits a bar stamped up to 16:00 on an + early close. That is D19, measured by conformance T05 and owned by the + engine rewrite. The rules that place a bar against the day's close (the + open-stamped filter, the session-close fill and the aggregation's buckets) + all read 13:00, so they agree on where the day's last decision is.""" assert sessions.session_windows("US") == ((clock_time(9, 30), clock_time(16, 0)),) assert sessions.is_in_session( pd.Timestamp("2026-11-27 15:00", tz="US/Eastern"), @@ -305,6 +307,11 @@ def test_an_early_close_leaves_the_close_stamped_path_alone(): timezone="US/Eastern", ) assert sessions.is_session_close( + pd.Timestamp("2026-11-27 13:00", tz="US/Eastern"), + market="US", + timezone="US/Eastern", + ) + assert not sessions.is_session_close( pd.Timestamp("2026-11-27 16:00", tz="US/Eastern"), market="US", timezone="US/Eastern", diff --git a/docs/superpowers/specs/2026-10-07-us-conformance-suite-design.md b/docs/superpowers/specs/2026-10-07-us-conformance-suite-design.md index dd4332084..06fc1352b 100644 --- a/docs/superpowers/specs/2026-10-07-us-conformance-suite-design.md +++ b/docs/superpowers/specs/2026-10-07-us-conformance-suite-design.md @@ -692,7 +692,7 @@ This ports `tests/scenarios/factory.py` and `tests/oracle/engine.py` (MIT), long | T02, T02d | FAIL | FAIL | D2, D16 | | T03 | FAIL (orders) | FAIL (orders) | D8 | | T04 | FAIL (orders) | FAIL (orders) | D8 | -| T05 | FAIL | FAIL | D19, D8 | +| T05 | FAIL | FAIL | D19, D8 (since #638: D2, D16) | | C01 | FAIL | FAIL | D5, D18 | | C02 | FAIL | FAIL | D6, D8 | | C03 | FAIL | FAIL | D6 | @@ -735,7 +735,7 @@ F01–F05 and CA01–CA03 count as FAIL rather than N/E under the §1 rule: cost | D16 | No daily decision timeframe | `profiles.py:198-218`; `external_run_service.py:297-299` | D, S | | D17 | S caps capital at ≤ 3,000 and order size at ≤ 10,000; v1 resolves weights at the decision close with floor division and applies a 25% cap | `constants.py:96-115`; `validator.py:69`; `domain/runs/protocol.py:112-114`; `domain/runs/service.py:916-985` | S | | D18 | The LLM translator's SELL always liquidates the full position, whatever its size; batches are truncated to one action per universe symbol | `portfolio_manager.py:802-812, 918-931` | DL | -| D19 | The US session is hard-coded to 09:30–16:00 with no early-close calendar, so a half day's final bucket is mis-stamped and its decision never fills | `infrastructure/market_data/sessions.py:40-44`; `domain/leaderboard/us_market_calendar.py:9` | D, S | +| D19 | The US session is hard-coded to 09:30–16:00 with no early-close calendar, so a half day's final bucket is mis-stamped and its decision never fills | `infrastructure/market_data/sessions.py:40-44`; `domain/leaderboard/us_market_calendar.py:9` | D, S. **Fixed by #638 (2026-10-15)**: `sessions.day_session_windows` ends an early close at 13:00 for aggregation, the open-stamped filter and the session-close fill, so the 13:00 bucket now fills at the 12:55 bar's close, which is D2's shape. Only the close-stamped `time_in_session` still admits stamps to 16:00. | --- From 87ba5117bf3d31c607762e2e5d323221e9aa294b Mon Sep 17 00:00:00 2001 From: FlyM1ss Date: Thu, 15 Oct 2026 12:59:06 -0400 Subject: [PATCH 15/15] fix(early-close): read the day's close in the board and Yahoo restamp - leaderboard/service.py: as-of and daily-window cutoffs use session_close_time(day, "US") instead of a fixed 16:00. - _yahoo.py: accept both New York zone spellings; a missing meta.exchangeTimezoneName logs one ERROR instead of silently skipping the restamp; ERRORs dedupe per (symbol, day) per process. - Live board copy names the 13:00 early close (leaderboard.js v=38). - Tests: early-close daily window, Yahoo zone/dedupe cases, copy guard; is_session_close asserts carry no side effects. Co-Authored-By: Claude Opus 5.5 --- .../backend/domain/leaderboard/service.py | 29 +++++------ .../domain/leaderboard/strategies/_yahoo.py | 51 +++++++++++++++---- .../infrastructure/market_data/sessions.py | 7 +-- .../market_data/test_us_market_calendar.py | 14 ++--- .../backend/tests/test_frontend_fast_boot.py | 2 +- .../tests/test_frontend_live_leaderboard.py | 10 ++++ .../backend/tests/test_leaderboard_api.py | 18 +++++++ dashboard/backend/tests/test_yahoo_chart.py | 46 ++++++++++++++++- dashboard/frontend/app.html | 4 +- dashboard/frontend/js/leaderboard.js | 2 +- 10 files changed, 144 insertions(+), 39 deletions(-) diff --git a/dashboard/backend/domain/leaderboard/service.py b/dashboard/backend/domain/leaderboard/service.py index 3d0b7174c..a408b7f19 100644 --- a/dashboard/backend/domain/leaderboard/service.py +++ b/dashboard/backend/domain/leaderboard/service.py @@ -27,7 +27,7 @@ from dashboard.backend.domain.leaderboard.strategies._common import reference_start_date from dashboard.backend.domain.leaderboard.strategies import get_strategy from dashboard.backend.infrastructure.llm import backtest_harness as llm_harness -from dashboard.backend.infrastructure.market_data.sessions import session_windows +from dashboard.backend.infrastructure.market_data.sessions import session_close_time from dashboard.backend.infrastructure.market_data.alpaca_bars import ( MarketDataUnavailableError, configured_feed_name, @@ -89,8 +89,6 @@ # Daily board window is the last *completed* US cash session, not UTC-yesterday. _US_EASTERN = ZoneInfo("America/New_York") -# 16:00 America/New_York regular-session close, from the one owner of the bounds. -_US_CASH_CLOSE = session_windows("US")[-1][1] # H6 integrity threshold: an LLM entry must have decided at least this fraction # of its steps with the model itself. Below it, the curve is mostly a rule-based @@ -132,9 +130,9 @@ def _coerce_as_of_eastern(as_of: Optional[Union[date, datetime]]) -> datetime: - ``None`` → now in Eastern. - aware ``datetime`` → converted to Eastern. - naive ``datetime`` → interpreted as already Eastern. - - bare ``date`` → that Eastern calendar day at the cash close (16:00), so a - weekday date means "that session is eligible" without callers inventing a - clock time. + - bare ``date`` → that Eastern calendar day at its cash close (16:00, 13:00 + on an NYSE early close), so a weekday date means "that session is + eligible" without callers inventing a clock time. """ if as_of is None: return datetime.now(_US_EASTERN) @@ -142,21 +140,18 @@ def _coerce_as_of_eastern(as_of: Optional[Union[date, datetime]]) -> datetime: if as_of.tzinfo is None: return as_of.replace(tzinfo=_US_EASTERN) return as_of.astimezone(_US_EASTERN) - return datetime( - as_of.year, as_of.month, as_of.day, - _US_CASH_CLOSE.hour, _US_CASH_CLOSE.minute, - tzinfo=_US_EASTERN, - ) + return datetime.combine(as_of, session_close_time(as_of, "US"), tzinfo=_US_EASTERN) def daily_window_dates(as_of: Optional[Union[date, datetime]] = None) -> Tuple[str, str]: """Most recently completed US cash equity session (Mon–Fri). Anchored on America/New_York so the board flips to **today** once the - regular session has closed (16:00 ET). Before the close on a weekday — and - all weekend — the window rolls back to the previous weekday. That is what - makes a post-close cron (e.g. 22:30 UTC ≈ 18:30 EDT) refresh the session - that just finished, instead of UTC-yesterday / last Friday. + regular session has closed (16:00 ET, 13:00 on an NYSE early close). + Before the close on a weekday — and all weekend — the window rolls back to + the previous weekday. That is what makes a post-close cron (e.g. 22:30 UTC + ≈ 18:30 EDT) refresh the session that just finished, instead of + UTC-yesterday / last Friday. Market holidays are not special-cased: an exchange holiday still selects that weekday after 16:00 ET (bars may be empty/sparse). @@ -164,7 +159,9 @@ def daily_window_dates(as_of: Optional[Union[date, datetime]] = None) -> Tuple[s now_et = _coerce_as_of_eastern(as_of) d = now_et.date() # Wall-clock compare in Eastern (DST already applied by ZoneInfo). - session_complete = now_et.weekday() < 5 and now_et.time() >= _US_CASH_CLOSE + session_complete = ( + now_et.weekday() < 5 and now_et.time() >= session_close_time(d, "US") + ) if not session_complete: d -= timedelta(days=1) while d.weekday() >= 5: # Sat/Sun → Friday diff --git a/dashboard/backend/domain/leaderboard/strategies/_yahoo.py b/dashboard/backend/domain/leaderboard/strategies/_yahoo.py index 3d4264a49..70651fdc8 100644 --- a/dashboard/backend/domain/leaderboard/strategies/_yahoo.py +++ b/dashboard/backend/domain/leaderboard/strategies/_yahoo.py @@ -16,6 +16,7 @@ from dashboard.backend.infrastructure.market_data.sessions import ( session_close_time, session_windows, + timezone_for_market, ) from dashboard.backend.infrastructure.market_data.us_market_calendar import ( is_early_close, @@ -23,7 +24,15 @@ _CHART_URL = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}" _HEADERS = {"User-Agent": "Mozilla/5.0"} -_US_EXCHANGE_ZONE = "America/New_York" +# Yahoo's ``meta.exchangeTimezoneName`` spells New York the IANA way; the US +# profile registers the "US/Eastern" alias. Either names the NYSE's clock, so the +# restamp accepts both rather than comparing against one spelling. +_YAHOO_NEW_YORK = "America/New_York" +_NEW_YORK = ZoneInfo(_YAHOO_NEW_YORK) +# ERRORs already printed this process. fetch_index_hourly runs on every plot +# render and baseline refresh, and a broken half-day shape is a standing +# condition, so each (kind, symbol, day) is reported once rather than per call. +_reported: set[Tuple[str, ...]] = set() class YahooChartError(requests.RequestException): @@ -131,12 +140,35 @@ def fetch_index_hourly( if close is None or not (start_e <= ts < end_e): continue points.append((dt.datetime.fromtimestamp(ts, dt.timezone.utc), float(close))) - exchange_zone = (res.get("meta") or {}).get("exchangeTimezoneName") + meta = res.get("meta") + exchange_zone = meta.get("exchangeTimezoneName") if isinstance(meta, dict) else None + if not isinstance(exchange_zone, str) or not exchange_zone: + # Absent is not "some other exchange": a renamed or dropped field would + # otherwise skip the restamp and the missing-level check together, and + # every early close would end on its 11:30 bar with nothing logged. + _report_once( + ("zone", symbol), + f"ERROR: yahoo.exchange_zone_missing symbol={symbol} " + "(meta.exchangeTimezoneName absent; NYSE early-close levels are not " + "restamped, so those days end on their 11:30 bar)", + ) + return points points = _restamp_close_levels(points, exchange_zone) _report_missing_close_levels(symbol, points, exchange_zone) return points +def _report_once(key: Tuple[str, ...], line: str) -> None: + if key in _reported: + return + _reported.add(key) + print(line, flush=True) + + +def _is_new_york(exchange_zone: Optional[str]) -> bool: + return exchange_zone in (_YAHOO_NEW_YORK, timezone_for_market("US")) + + def _last_slot_open(local: dt.datetime) -> dt.datetime: """The open of the last grid slot ending at or before ``local``, on Yahoo's hourly grid from the 09:30 open (12:30 for a 13:00 close, 15:30 for 16:00).""" @@ -176,9 +208,9 @@ def _restamp_close_levels( response, inside the window whenever a fetch reaches today; moving it would mark today on the official close and a replay of the same day pre-auction. """ - if exchange_zone != _US_EXCHANGE_ZONE: + if not _is_new_york(exchange_zone): return points - zone = ZoneInfo(_US_EXCHANGE_ZONE) + zone = _NEW_YORK by_stamp: Dict[dt.datetime, float] = {} for stamp, level in points: local = stamp.astimezone(zone) @@ -196,7 +228,8 @@ def _report_missing_close_levels( exchange_zone: Optional[str], now: Optional[dt.datetime] = None, ) -> None: - """One flushed ERROR per settled NYSE early close that has no closing level. + """One flushed ERROR per settled NYSE early close that has no closing level, + once per (symbol, day) per process. The restamp depends on Yahoo's half-day shape. If Yahoo stops sending the 13:00 point, or stamps it a second off, the day ends on its 11:30 bar and @@ -204,17 +237,17 @@ def _report_missing_close_levels( nothing else to notice it. A day still within an hour of its close is skipped, since Yahoo may not have sent the level yet. """ - if exchange_zone != _US_EXCHANGE_ZONE: + if not _is_new_york(exchange_zone): return - zone = ZoneInfo(_US_EXCHANGE_ZONE) + zone = _NEW_YORK now = now or dt.datetime.now(dt.timezone.utc) stamps = {stamp.astimezone(zone) for stamp, _level in points} for day in sorted({stamp.date() for stamp in stamps if is_early_close(stamp.date())}): close = dt.datetime.combine(day, session_close_time(day, "US"), tzinfo=zone) if now < close + dt.timedelta(hours=1) or _last_slot_open(close) in stamps: continue - print( + _report_once( + ("early_close_level", symbol, day.isoformat()), f"ERROR: yahoo.early_close_level_missing symbol={symbol} date={day} " "(no level at the last half bar; the index curve ends on its 11:30 bar)", - flush=True, ) diff --git a/dashboard/backend/infrastructure/market_data/sessions.py b/dashboard/backend/infrastructure/market_data/sessions.py index 6e401a7da..108d43822 100644 --- a/dashboard/backend/infrastructure/market_data/sessions.py +++ b/dashboard/backend/infrastructure/market_data/sessions.py @@ -78,9 +78,10 @@ def day_session_windows(day: date, market: object) -> tuple[tuple[time, time], . from the 12:30-12:59 minutes, nothing filled it, and an early-close day lost its closing decision. The close-stamped rule (:func:`time_in_session`) does not read it and still admits a bar stamped up to 16:00 (D19 in the US - conformance design, owned by the engine rewrite). No US source this repo - loads is close-stamped, and aggregation no longer produces a decision bar - after 13:00 on an early close, so nothing reaches that gap today. + conformance design, owned by the engine rewrite). Alpaca bars are + open-stamped and aggregation no longer makes a decision bar after 13:00 + on an early close, so only a frame loaded close-stamped (the + ``vnpy_simulation`` source, test doubles) still reaches that gap. """ if _trades_us_sessions(market) and is_early_close(day): return ((_US_SESSIONS[0][0], EARLY_CLOSE_TIME),) diff --git a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py index d711baffb..23406e017 100644 --- a/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py +++ b/dashboard/backend/tests/infrastructure/market_data/test_us_market_calendar.py @@ -210,12 +210,14 @@ def test_day_session_windows(day, market, windows): def test_is_session_close_reads_the_days_close(): from datetime import datetime - def close(hour, day): - stamp = datetime(2026, 11, day, hour, 0) - return sessions.is_session_close(stamp, market="US", timezone="US/Eastern") - - assert close(13, 27) and not close(16, 27) - assert close(16, 25) and not close(13, 25) + closes = { + (day, hour): sessions.is_session_close( + datetime(2026, 11, day, hour, 0), market="US", timezone="US/Eastern" + ) + for day in (25, 27) + for hour in (13, 16) + } + assert closes == {(25, 13): False, (25, 16): True, (27, 13): True, (27, 16): False} @pytest.mark.parametrize( diff --git a/dashboard/backend/tests/test_frontend_fast_boot.py b/dashboard/backend/tests/test_frontend_fast_boot.py index 5551c90d2..0968817a2 100644 --- a/dashboard/backend/tests/test_frontend_fast_boot.py +++ b/dashboard/backend/tests/test_frontend_fast_boot.py @@ -197,7 +197,7 @@ def test_cache_busters_bumped(): assert "app.js?v=159" in APP_HTML assert "js/agent-editor.js?v=32" in APP_HTML assert "styles.css?v=159" in APP_HTML - assert "js/leaderboard.js?v=37" in APP_HTML + assert "js/leaderboard.js?v=38" in APP_HTML assert "home-page.js?v=55" in APP_HTML assert "js/credit-format.js?v=1" in APP_HTML assert "js/credits.js?v=8" in APP_HTML diff --git a/dashboard/backend/tests/test_frontend_live_leaderboard.py b/dashboard/backend/tests/test_frontend_live_leaderboard.py index b4f4a6c6f..20d957a66 100644 --- a/dashboard/backend/tests/test_frontend_live_leaderboard.py +++ b/dashboard/backend/tests/test_frontend_live_leaderboard.py @@ -75,6 +75,16 @@ def test_live_chart_hint_names_the_early_close(): assert "10:00–13:00 on early closes" in view +def test_live_board_copy_names_the_early_close(): + """The header cadence and the About text sit on the same view as the chart + hint; neither may say the session ends at 16:00 on a 13:00 day.""" + leaderboard_js = (_FRONTEND / "js" / "leaderboard.js").read_text(encoding="utf-8") + assert "after the 16:00 ET cash session" not in leaderboard_js + assert "13:00 on early closes" in leaderboard_js + assert "Each night after the 16:00 ET cash close" not in _APP_HTML + assert "13:00 on an NYSE early close" in _APP_HTML + + def _y_bounds_source() -> str: start = _LIVE_JS.index("const LIVE_Y_PAD_RATIO") end = _LIVE_JS.index("function renderLiveChart") diff --git a/dashboard/backend/tests/test_leaderboard_api.py b/dashboard/backend/tests/test_leaderboard_api.py index bcb4c7a78..26018f876 100644 --- a/dashboard/backend/tests/test_leaderboard_api.py +++ b/dashboard/backend/tests/test_leaderboard_api.py @@ -258,6 +258,24 @@ def test_daily_window_dates_skips_weekend(): assert start == end == "2026-07-14" +def test_daily_window_dates_reads_an_early_close(): + """2026-11-27 closes at 13:00 ET: the session is complete from then, not + from 16:00, and a bare date resolves to that close. The close has one + owner (``sessions.session_close_time``), shared with the Live board.""" + from datetime import date, datetime + from zoneinfo import ZoneInfo + + et = ZoneInfo("America/New_York") + start, _ = lb_service.daily_window_dates(as_of=datetime(2026, 11, 27, 12, 59, tzinfo=et)) + assert start == "2026-11-26" + start, _ = lb_service.daily_window_dates(as_of=datetime(2026, 11, 27, 13, 0, tzinfo=et)) + assert start == "2026-11-27" + assert lb_service._coerce_as_of_eastern(date(2026, 11, 27)) == datetime( + 2026, 11, 27, 13, 0, tzinfo=et + ) + assert not hasattr(lb_service, "_US_CASH_CLOSE") + + def test_daily_window_dates_cron_moment_is_todays_session(): """22:30 UTC after a weekday close must select that US session, not Friday.""" from datetime import datetime, timezone diff --git a/dashboard/backend/tests/test_yahoo_chart.py b/dashboard/backend/tests/test_yahoo_chart.py index d16c979e3..2497d5802 100644 --- a/dashboard/backend/tests/test_yahoo_chart.py +++ b/dashboard/backend/tests/test_yahoo_chart.py @@ -183,6 +183,16 @@ def test_yahoo_chart_error_is_a_request_exception(): _NEW_YORK = ZoneInfo("America/New_York") +@pytest.fixture(autouse=True) +def _fresh_error_reports(): + """The module reports each condition once per process; each test starts clean.""" + from dashboard.backend.domain.leaderboard.strategies import _yahoo + + _yahoo._reported.clear() + yield + _yahoo._reported.clear() + + def _recorded(name): return json.loads((_FIXTURES / name).read_text()) @@ -255,6 +265,39 @@ def test_a_missing_early_close_level_is_reported(monkeypatch, capsys): ) +def test_a_missing_early_close_level_is_reported_once_per_process(monkeypatch, capsys): + """Every plot render and refresh fetches again; one standing condition must + not print one ERROR per call.""" + payload = _without_the_close_point(_recorded("yahoo-chart-dji-1h-2025-11-28.json")) + _stub_response(monkeypatch, payload) + fetch_index_hourly("^DJI", "2025-11-28", "2025-11-28") + fetch_index_hourly("^DJI", "2025-11-28", "2025-11-28") + assert capsys.readouterr().out.count("yahoo.early_close_level_missing") == 1 + + +@pytest.mark.parametrize("meta", [{}, None, {"exchangeTimezoneName": ""}]) +def test_a_missing_exchange_zone_is_reported_not_skipped_silently(monkeypatch, capsys, meta): + """Absent is not "another exchange". Without the field the restamp and the + missing-level check would both skip and every early close would end on its + 11:30 bar with nothing logged, so absence is an ERROR (once per symbol).""" + payload = _recorded("yahoo-chart-dji-1h-2025-11-28.json") + payload["chart"]["result"][0]["meta"] = meta + _stub_response(monkeypatch, payload) + points = fetch_index_hourly("^DJI", "2025-11-28", "2025-11-28") + fetch_index_hourly("^DJI", "2025-11-28", "2025-11-28") + assert _clock(points)[-1] == ("13:00", 47716.421875) + out = capsys.readouterr().out + assert out.count("ERROR: yahoo.exchange_zone_missing symbol=^DJI") == 1 + + +def test_the_profiles_spelling_of_new_york_also_restamps(monkeypatch): + payload = _recorded("yahoo-chart-dji-1h-2025-11-28.json") + payload["chart"]["result"][0]["meta"]["exchangeTimezoneName"] = "US/Eastern" + _stub_response(monkeypatch, payload) + points = fetch_index_hourly("^DJI", "2025-11-28", "2025-11-28") + assert _clock(points)[-1] == ("12:30", 47716.421875) + + def test_an_early_close_within_its_settle_hour_is_not_reported(capsys): from dashboard.backend.domain.leaderboard.strategies._yahoo import ( _report_missing_close_levels, @@ -327,9 +370,10 @@ def test_the_close_stamped_level_wins_over_a_half_bar_yahoo_also_filled(monkeypa assert len(points) == 4 -def test_a_non_us_index_is_not_restamped_on_an_nyse_early_close(monkeypatch): +def test_a_non_us_index_is_not_restamped_on_an_nyse_early_close(monkeypatch, capsys): payload = _recorded("yahoo-chart-dji-1h-2025-11-28.json") payload["chart"]["result"][0]["meta"]["exchangeTimezoneName"] = "Asia/Shanghai" _stub_response(monkeypatch, payload) points = fetch_index_hourly("^DJI", "2025-11-28", "2025-11-28") assert _clock(points)[-1] == ("13:00", 47716.421875) + assert "ERROR" not in capsys.readouterr().out diff --git a/dashboard/frontend/app.html b/dashboard/frontend/app.html index 5b6a085a3..b40dc8dff 100644 --- a/dashboard/frontend/app.html +++ b/dashboard/frontend/app.html @@ -1590,7 +1590,7 @@

Competition Overview

Live Trading Leaderboard

This board runs forward, one calendar month at a time, on NYSE trading days. “Live” describes the direction, not the money: this is simulated trading on real market data — no broker account, no real capital, and no order ever leaves this system.

-

Each night after the 16:00 ET cash close, each entry is advanced one trading day: the model sees that day's real bars for the first time and its portfolio carries forward from where the previous day left it. Nobody — including us — knows what the next bar does when an entry locks, which is what separates this board from the fixed-window Competition one.

+

Each night after the US cash close (16:00 ET, or 13:00 on an NYSE early close), each entry is advanced one trading day: the model sees that day's real bars for the first time and its portfolio carries forward from where the previous day left it. Nobody — including us — knows what the next bar does when an entry locks, which is what separates this board from the fixed-window Competition one.

Each entry is one continuous book that starts the month flat at $100,000 and ranks only the days inside it; the board resets on the 1st. Entries are a curated roster of AI models and baselines — there are no user submissions.

We are in Season 0, the shakedown season, so read the standings as a preview rather than an official result. An entry more than one trading session behind is shown but not ranked.

@@ -2519,7 +2519,7 @@

Refund Credits purchase

- +