Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 32 additions & 18 deletions loopx/capabilities/native_chat/session_queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,14 @@
from ...chat_store import CHAT_SESSION_MODE_ATTACHED, TERMINAL_TURN_STATES


def _is_observation_only_monitor_turn(turn: dict[str, Any] | None) -> bool:
"""Return True only for native no-model monitor turns."""
if not isinstance(turn, dict) or turn.get("loopx_execution") is not True:
return False
request = turn.get("loopx_request")
return isinstance(request, dict) and request.get("work_kind") == "monitor"


def drain_session_queue(controller: Any, *, session_id: str, work_dir: Path, objective: str) -> None:
wakeup = threading.Event()
with controller.turn_release_wakeup(wakeup):
Expand Down Expand Up @@ -76,19 +84,20 @@ def _drain_session_queue_events(controller: Any, *, session_id: str, work_dir: P
# normal preparation and execution fences below.
turn = active
else:
with controller.lock:
attached = controller.adapters.get(session_id)
if attached is None or not attached.healthcheck():
try:
controller._ensure_adapter(
session,
work_dir=work_dir,
objective=objective,
)
except Exception as exc:
controller._fail_queue_preparation(session_id, active_turn_id, exc)
else:
continue
if not _is_observation_only_monitor_turn(active):
with controller.lock:
attached = controller.adapters.get(session_id)
if attached is None or not attached.healthcheck():
try:
controller._ensure_adapter(
session,
work_dir=work_dir,
objective=objective,
)
except Exception as exc:
controller._fail_queue_preparation(session_id, active_turn_id, exc)
else:
continue
active = controller.store.load_turn(session_id, active_turn_id)
if active and active.get("status") not in TERMINAL_TURN_STATES:
controller.closed.wait(0.05)
Expand Down Expand Up @@ -132,11 +141,16 @@ def _drain_session_queue_events(controller: Any, *, session_id: str, work_dir: P
current = controller.store.load_session(session_id)
if current is None or current.get("active_turn_id") != turn_id:
return
adapter = controller._ensure_adapter(
current,
work_dir=work_dir,
objective=objective,
)
if _is_observation_only_monitor_turn(turn):
from .steward_monitor import StewardObservationAdapter

adapter: Any = StewardObservationAdapter(current.get("upstream_thread_id"))
else:
adapter = controller._ensure_adapter(
current,
work_dir=work_dir,
objective=objective,
)
if (controller.store.load_session(session_id) or {}).get("active_turn_id") != turn_id:
return
except Exception as exc:
Expand Down
44 changes: 39 additions & 5 deletions loopx/capabilities/native_chat/steward_turn.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,48 @@

from collections.abc import Mapping, Sequence
from contextlib import contextmanager, ExitStack, nullcontext
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import re
import time
from typing import Any

from ...control_plane.goals.activation import goal_is_stopped
from ...control_plane.goals.first_party_host_admission import FirstPartyHostRuntimeRejected
from ...file_lock import exclusive_file_lock
from .steward_managed_mode import ManagedSessionTurn, ManagedTaskValidationUnavailable, StewardManagedMode

MONITOR_RECOVERY_RETRY_COOLDOWN_SECONDS = 300
_WORKER_EXITED_ERROR = (
"The LoopX Chat execution worker exited before this turn finished. "
"Your conversation is saved; execution may have occurred. This turn was not retried automatically."
)
_POST_CONTEXT_WATCH_ERRORS = frozenset({
"steward monitor external signal source temporarily unavailable",
"disk I/O error",
_WORKER_EXITED_ERROR,
})
_USER_ACTION_DEDUPE_KEY = re.compile(r"^[a-z0-9][a-z0-9._:@/-]{0,119}$")
_REPORT_VALIDATOR_PREFIX = (
"from pathlib import Path; from loopx.capabilities.periodic_report.private_generation import validate_private_generation; "
"validate_private_generation(Path("
)


def _iso_timestamp_seconds(value: Any) -> float | None:
if not isinstance(value, str) or not value.strip():
return None
try:
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
except ValueError:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.timestamp()


def _same_report_validator_command(frozen_cmd: Any, request_cmd: Any) -> bool:
"""Allow report validation commands whose only drift is the host Python path."""
return (isinstance(frozen_cmd, list) and isinstance(request_cmd, list)
Expand Down Expand Up @@ -209,18 +233,27 @@ def monitor_restart_basis(self, request: Mapping[str, Any]) -> dict[str, Any]:
"event_kinds": [row["kind"] for row in events], "poll_absent": not present,
"contexts": [row["payload"] for row in events if row["kind"] == "manager.context"],
"phases": [row["payload"] for row in events if row["kind"] == "agent.phase"],
"old_worker_excluded": True, "lease_available": available}}
"old_worker_excluded": True, "lease_available": available,
"observed_at": time.time()}}

@staticmethod
def monitor_restart_request(original: Mapping[str, Any]) -> dict[str, Any] | None:
def monitor_restart_request(original: Mapping[str, Any], *, now: float | None = None) -> dict[str, Any] | None:
"""Prepare correlation only; current typed admission decides recovery."""
prior = original.get("loopx_request") or {}
error = original.get("error_code")
if (original.get("status") != "failed"
or error not in {"server_restarted", "runtime_error", "runtime_unavailable"}
or prior.get("work_kind") != "monitor"
or (prior.get("monitor_restart_recovery") is not None and error != "server_restarted")):
or prior.get("work_kind") != "monitor"):
return None
if prior.get("monitor_restart_recovery") is not None and error != "server_restarted":
post_context_error = error == "runtime_unavailable" or (
error == "runtime_error" and original.get("error") in _POST_CONTEXT_WATCH_ERRORS
)
completed_ts = _iso_timestamp_seconds(original.get("completed_at"))
current_ts = time.time() if now is None else now
if (not post_context_error or completed_ts is None
or current_ts - completed_ts < MONITOR_RECOVERY_RETRY_COOLDOWN_SECONDS):
return None
return {**prior, "monitor_restart_recovery": {"original_turn_id": original["turn_id"]}}

def monitor_restart_carrier(self, original: dict[str, Any], client_id: str) -> dict[str, Any] | None:
Expand Down Expand Up @@ -248,7 +281,6 @@ def report_quota_basis(self, request: Mapping[str, Any]) -> dict[str, Any]:
from ...control_plane.turn_driver import inspect_loopx_turn_journal
from ..periodic_report.private_generation import attempt_directory
from .steward_reports import report_service
import time
actor = request["actor"]
original = self.store.load_turn(actor["session_id"], recovery["original_turn_id"])
result = ((original or {}).get("response") or {}).get("native_loopx") or {}
Expand Down Expand Up @@ -615,6 +647,8 @@ def _prepare_submission(self, binding: dict[str, Any] | None, *, turn_instance_i
return {"turn": existing, "quota_probe": {"original": existing, "binding": binding}}
quota_request = quota_observation["request"]
if quota_request is not None:
if _same_report_validator_command(quota_request.get("validation_command"), request.get("validation_command")):
quota_request = {**quota_request, "validation_command": request["validation_command"]}
request = quota_request
client_id = native_steward_client_id(actor, request["turn_instance_id"])
existing = None
Expand Down
61 changes: 49 additions & 12 deletions loopx/control_plane/collaboration/steward_managed_mode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,42 @@ import {projectTurnBoundary, turnPlanWorkSource} from "../quota/turn_envelope.ts
import {projectGoalCreationExecutionRequest} from "../goals/creation_execution.ts";

/** A stopped observation may get a new Chat carrier for the same native
* identity. After a recovery starts, only another proven process restart can
* continue it. The host reads the stopped worker lock, journals and native poll
* receipts. Normal admission still checks ownership, deadline, permissions and user work. */
* identity. After a recovery starts, only another proven process restart or a
* cooldown-bounded pre-poll transient failure can continue it. The host reads
* the stopped worker lock, journals and native poll receipts. Normal admission
* still checks ownership, deadline, permissions and user work. */
const WORKER_EXITED_ERROR =
"The LoopX Chat execution worker exited before this turn finished. Your conversation is saved; execution may have occurred. This turn was not retried automatically.";
const MONITOR_RECOVERY_RETRY_COOLDOWN_MS = 300_000;
const REPORT_VALIDATOR_PREFIX =
"from pathlib import Path; from loopx.capabilities.periodic_report.private_generation import validate_private_generation; "
+ "validate_private_generation(Path(";

function sameReportValidatorCommand(frozenCmd: unknown, requestCmd: unknown): boolean {
return Array.isArray(frozenCmd) && Array.isArray(requestCmd)
&& frozenCmd.length === 3 && requestCmd.length === 3
&& frozenCmd.every(part => typeof part === "string" && part.length > 0 && !part.includes("\0"))
&& requestCmd.every(part => typeof part === "string" && part.length > 0 && !part.includes("\0"))
&& frozenCmd[1] === "-c" && requestCmd[1] === "-c"
&& frozenCmd[2] === requestCmd[2]
&& frozenCmd[2].startsWith(REPORT_VALIDATOR_PREFIX)
&& frozenCmd[2].endsWith("))");
}

function sameReportRequestField(key: string, priorValue: unknown, requestValue: unknown): boolean {
if (key === "validation_command" && sameReportValidatorCommand(priorValue, requestValue)) {
return true;
}
return JSON.stringify(priorValue) === JSON.stringify(requestValue);
}

function sameReportRequest(priorValue: unknown, request: JsonObject): boolean {
const prior = jsonObject(priorValue);
if (!prior) return false;
const keys = Object.keys(request);
return Object.keys(prior).length === keys.length
&& keys.every(key => Object.hasOwn(prior, key) && sameReportRequestField(key, prior[key], request[key]));
}

function hasSingleManagerContextEvidence(basis: JsonObject, prior: JsonObject, events: unknown): boolean {
if (requireJsonObject(prior.actor, "original watch actor").schema_version !== "loopx_steward_managed_actor_v0"
Expand Down Expand Up @@ -80,8 +111,9 @@ function admitMonitorRestartRecovery(params: JsonObject, request: JsonObject, se
&& Array.isArray(events) && events.filter(kind => kind === "agent.phase").length === 1;
// Post-context monitor failures (external signal outage, daemon worker exit,
// storage I/O error, or runtime unavailability) before any poll receipt or
// upstream model turn may recover at most once when strict single-context
// and zero-effect journal proofs hold; pre-context unknown worker launches remain closed.
// upstream model turn may recover once immediately and retry only after a
// bounded cooldown when strict single-context and zero-effect journal proofs
// hold; pre-context unknown worker launches remain closed.
const postContextWatchRecoverable = hasSingleManagerContextEvidence(basis, prior, events) && (
original.error_code === "runtime_unavailable"
|| original.error_code === "runtime_error" && (
Expand All @@ -90,6 +122,12 @@ function admitMonitorRestartRecovery(params: JsonObject, request: JsonObject, se
|| original.error === WORKER_EXITED_ERROR
)
);
const completedAtMs = typeof original.completed_at === "string" ? Date.parse(original.completed_at) : NaN;
const observedAtMs = typeof basis.observed_at === "number" && Number.isFinite(basis.observed_at)
? basis.observed_at * 1000 : NaN;
const chainedPostContextWatchRecoverable = postContextWatchRecoverable
&& Number.isFinite(completedAtMs) && Number.isFinite(observedAtMs)
&& observedAtMs - completedAtMs >= MONITOR_RECOVERY_RETRY_COOLDOWN_MS;
if (Object.keys(recovery).join() !== "original_turn_id"
|| recovery.original_turn_id !== original.turn_id || typeof original.turn_id !== "string"
|| !/^[a-f0-9]{32}$/.test(original.turn_id) || original.session_id !== session.session_id
Expand All @@ -98,7 +136,7 @@ function admitMonitorRestartRecovery(params: JsonObject, request: JsonObject, se
|| original.origin !== "native_loopx" || original.loopx_execution !== true
|| original.response !== null || original.upstream_turn_id !== null || original.delta_count !== 0
|| prior.work_kind !== "monitor" || request.work_kind !== "monitor" || request.validation_command !== null
|| (predecessor !== null && (original.error_code !== "server_restarted"
|| (predecessor !== null && (!(original.error_code === "server_restarted" || chainedPostContextWatchRecoverable)
|| Object.keys(predecessor).join() !== "original_turn_id"
|| typeof predecessor.original_turn_id !== "string" || !/^[a-f0-9]{32}$/.test(predecessor.original_turn_id)
|| predecessor.original_turn_id === original.turn_id))
Expand Down Expand Up @@ -154,9 +192,9 @@ function admitReportQuotaRecovery(params: JsonObject, request: JsonObject, sessi
|| original.origin !== "native_loopx" || original.loopx_execution !== true || original.status !== "completed"
|| original.delta_count !== 0 || typeof original.upstream_turn_id !== "string" || !original.upstream_turn_id
|| Object.keys(prior).length !== keys.length
|| keys.some(key => key === "turn_instance_id"
|| keys.some(key => !Object.hasOwn(prior, key) || (key === "turn_instance_id"
? request[key] !== String(prior[key]) + ".quota-recovery"
: JSON.stringify(request[key]) !== JSON.stringify(prior[key]))
: !sameReportRequestField(key, prior[key], request[key])))
|| result.schema_version !== "loopx_turn_execution_v0" || result.ok !== false || result.mode !== "run_once"
|| result.status !== "failed" || result.result_kind !== "host_failure"
|| failure.schema_version !== "loopx_turn_host_failure_v0" || failure.attempt !== 1
Expand Down Expand Up @@ -194,8 +232,7 @@ function admitReportSelectionRecovery(params: JsonObject, request: JsonObject, s
if (original.session_id !== sessionId || original.origin !== "native_loopx"
|| original.loopx_execution !== true || original.status !== "completed"
|| original.delta_count !== 0 || original.upstream_turn_id !== null
|| Object.keys(prior).length !== Object.keys(request).length
|| Object.keys(request).some(key => JSON.stringify(prior[key]) !== JSON.stringify(request[key]))
|| !sameReportRequest(prior, request)
|| result.schema_version !== "loopx_turn_execution_v0" || result.ok !== false
|| result.mode !== "run_once" || result.effects_scope !== "current_invocation"
|| result.error_code !== "turn_todo_selection_deferred" || result.resume_turn_key != null
Expand Down Expand Up @@ -228,7 +265,7 @@ function admitReportSelectionRecovery(params: JsonObject, request: JsonObject, s
|| !/^[a-f0-9]{32}$/.test(observed.turn_id) || observed.turn_id === original.turn_id
|| observed.origin !== "native_loopx" || observed.loopx_execution !== true
|| observed.error_code != null || observed.error != null
|| JSON.stringify(observed.loopx_request) !== JSON.stringify(request)
|| !sameReportRequest(observed.loopx_request, request)
|| !(runningHere || observed.status === "completed"
&& JSON.stringify(requireJsonObject(observed.response, "probe response").native_loopx) === JSON.stringify(result))
|| journal.schema_version !== "loopx_turn_journal_v0" || !ENVELOPED_SHA256_PATTERN.test(String(journal.turn_key))
Expand Down Expand Up @@ -638,7 +675,7 @@ export function admitStewardManagedTurn(params: JsonObject): JsonObject {
|| original.loopx_execution !== true || original.status !== "completed"
|| original.delta_count !== 0 || original.upstream_turn_id !== null
|| Object.keys(prior).length !== keys.length
|| keys.some(key => JSON.stringify(prior[key]) !== JSON.stringify(request[key]))
|| keys.some(key => !Object.hasOwn(prior, key) || !sameReportRequestField(key, prior[key], request[key]))
|| result.schema_version !== "loopx_turn_execution_v0" || result.ok !== false
|| result.mode !== "run_once" || result.status !== "failed" || result.result_kind !== "host_failure"
|| result.reason !== "first-party Host runtime rejected: stale_goal_authority"
Expand Down
Loading
Loading