From 96b8be60088b23b790be03def2961861186a113d Mon Sep 17 00:00:00 2001 From: root Date: Mon, 7 Sep 2026 18:30:40 +0800 Subject: [PATCH 1/4] optimize skills --- README.md | 76 +++++-- README_CN.md | 71 ++++-- docs/guides/custom-environment.md | 21 +- docs/guides/custom-environment_CN.md | 21 +- skills/safactory-workflows/SKILL.md | 154 +++++++++---- .../references/environment-integration.md | 214 ++++++++++++------ 6 files changed, 394 insertions(+), 163 deletions(-) diff --git a/README.md b/README.md index 79419bd9..f958ab0f 100644 --- a/README.md +++ b/README.md @@ -51,33 +51,81 @@ https://github.com/user-attachments/assets/4c551b27-ce4d-4fc8-8df6-d6dc8100cc88 ## 🧩 Agent Skill Quick Start -This repository includes a lightweight Agent skill that helps agents use SAfactory through the standard workflows: +This repository includes a lightweight Agent skill that helps an Agent onboard a benchmark and run a minimal evaluation through the standard workflow: ```text skills/safactory-workflows/SKILL.md ``` -It covers three common requests: +This skill currently covers onboarding a new benchmark and running a minimal single-case evaluation in Docker or RJob mode. -- onboard a new benchmark or custom environment into SAfactory; -- run Docker-mode evaluation for a selected environment; -- start GRPO / RL training for a selected environment. +### Benchmark Onboarding Prompt -When working with an Agent, use prompts such as: +Before onboarding, prepare: -```text -Use skills/safactory-workflows to help me onboard this benchmark into SAfactory. -``` +1. Prepare **1–2 test cases** and make sure the benchmark's native single-case command works independently. +2. Choose the mode to onboard: `docker` (local image) or `rjob` (cluster RJob). +3. Prepare and provide the Agent with: -```text -Use the safactory-workflows skill to run geo3k evaluation in Docker mode. -``` + - environment name; + - local benchmark source path or link; + - path to the test dataset; + - the native single-case command, or the corresponding section in the benchmark README; + - an accessible Docker image address; + - native benchmark result/output file path or naming rule; + - native score/reward location, range, and pass condition. + +4. We recommend filling in the provided prompt and sending it to the Agent. The Agent will inspect the benchmark source/README and SAfactory docs, then implement the adapter, configs, and evaluator. +5. The Agent runs a minimal smoke test with 1–2 cases. Onboarding is complete when the runner result JSON, native benchmark output file, Gateway trajectory, and final `0–10` reward are all present and traceable to the same case. + +RJob users also need to prepare a Gateway URL. Do not use `localhost` or `127.0.0.1` as the Gateway address from an RJob container. + +
+Expand to get the Benchmark Onboarding Prompt ```text -Use the safactory-workflows skill to start GRPO training for my_env. +Use skills/safactory-workflows to onboard the following benchmark into SAfactory. + +[Execution mode] (required; choose one) +- mode: [docker / rjob] + +[Benchmark] +- environment name (for example, mybench): ____________________ +- benchmark source or checkout path/repository: ____________________ +- dataset path: ____________________ +- one dataset-row schema/field description: ____________________ +- 1–2 smoke-test case IDs or dataset rows: ____________________ +- native single-case benchmark command: ____________________ +- if the command is defined in a README, file and section: ____________________ +- Docker image, if one already exists: ____________________ + +[Results and scoring] +- native benchmark result/output file path or naming rule: ____________________ +- field or file containing the native score/reward: ____________________ +- score/reward range, meaning, and pass condition: ____________________ + +[Scope] +- Start with only the 1–2 cases above. +- Implement the SAfactory adapter boundary: read the request, take + env_params.dataset, call the model through the Gateway, invoke the existing + native single-case command, read its result, and return a + SimulationStartResult JSON object. +- Do not rewrite the benchmark single-case execution or evaluation logic + already inside the Docker image. +- Report and verify the runner result JSON, native benchmark output file, + Gateway trajectory, and final 0–10 reward. + +First inspect the benchmark source/README and +docs/guides/custom-environment.md, then edit the files required by the selected +mode. If information is missing, ask only for that field; do not guess the +benchmark command or scoring rule. ``` -The skill does not replace the docs. It guides the Agent to read `docs/guides/`, `docs/reference/`, and the root README as needed, while using the standard `env/geo3k/` environment as the reference implementation. If your Agent supports local skill discovery, add `skills/safactory-workflows/` to its skill search path; otherwise mention this path explicitly in the request. +
+ +The Agent owns the SAfactory adapter boundary, not the benchmark's internal single-case logic. See [Custom Environments](docs/guides/custom-environment.md) and the skill's [integration reference](skills/safactory-workflows/references/environment-integration.md) for file responsibilities, the runner/result contract, and Docker/RJob differences. + +When you use this skill, the Agent reads `docs/guides/`, `docs/reference/`, and the root README as needed, using the standard `env/geo3k/` environment as its reference implementation. You only need to provide the benchmark information listed above; if your Agent cannot discover local skills automatically, include `skills/safactory-workflows/` explicitly in the prompt. ## 🚀 Quick Start diff --git a/README_CN.md b/README_CN.md index 7e1c1405..b4acf995 100644 --- a/README_CN.md +++ b/README_CN.md @@ -51,33 +51,76 @@ https://github.com/user-attachments/assets/4c551b27-ce4d-4fc8-8df6-d6dc8100cc88 ## 🧩 Agent Skill 快速上手 -仓库内置了一个轻量 Agent skill，用于帮助 Agent 按标准 workflow 使用 SAfactory： +仓库内置了一个轻量 Agent skill，用于帮助 Agent 按标准 workflow 接入 benchmark 并完成最小评测： ```text skills/safactory-workflows/SKILL.md ``` -它覆盖三类高频请求： +### Benchmark 接入 Prompt -- 接入新的 benchmark 或自定义环境到 SAfactory； -- 用 Docker 模式运行指定环境的测评； -- 启动指定环境的 GRPO / RL 训练。 +接入前需要准备： -使用 Agent 时，可以直接这样提问： +1. 准备 **1–2 个测试 case**，并确保 benchmark 原生的单 case 命令可以独立运行。 +2. 选择需要接入的模式：`docker`（本地镜像）或 `rjob`（集群 RJob）。 +3. 准备并告知Agent下方信息： -```text -请使用 skills/safactory-workflows，帮我把这个 benchmark 接入 SAfactory。 -``` + - environment name； + - benchmark 源码本地路径或链接； + - 测试 dataset 所在路径； + - benchmark 原生单 case 执行命令，或源码 README 中对应的章节； + - 可访问的Docker image地址； + - benchmark 原生结果输出文件的路径/命名规则； + - 原生 score/reward 所在位置、取值范围和通过条件。 -```text -请使用 safactory-workflows skill，用 Docker 模式跑 geo3k 测评。 -``` +4. 建议使用提供的Prompt将信息填写好后发送给 Agent。Agent 会先检查 benchmark 源码/README 和 SAfactory 文档，再实现 adapter、配置文件和评测器。 +5. Agent 用 1–2 个 case 运行最小 smoke test。完成接入的验收标准是：runner result JSON、benchmark 原生结果文件、Gateway 轨迹和最终 `0–10` reward 都能找到并相互对应。 + +RJob 用户还需要准备 Gateway 地址；不要把 `localhost` 或 `127.0.0.1` 作为 RJob 容器访问 Gateway 的地址。 + +
+展开获取接入 Prompt ```text -请使用 safactory-workflows skill，启动 my_env 环境的 GRPO 训练。 +请使用 skills/safactory-workflows，将下面的 benchmark 接入 SAfactory。 + +【接入模式】（必填，只能选一个） +- mode: [docker / rjob] + +【Benchmark 信息】 +- environment name（例如 mybench）: ____________________ +- Benchmark 源码或 checkout 路径/仓库地址: ____________________ +- 数据集路径: ____________________ +- 单条 dataset row 的格式/字段说明: ____________________ +- 用于 smoke test 的 1–2 个 case ID 或 dataset row: ____________________ +- Benchmark 原生单 case 执行命令: ____________________ +- 如果命令来自 README，请填写文件和章节: ____________________ +- 对应的 Docker image（如已有）: ____________________ + +【结果与评分】 +- Benchmark 原生测评结果输出文件路径/命名规则: ____________________ +- 原生 score/reward 所在字段或文件: ____________________ +- score/reward 的取值范围、含义和通过条件: ____________________ + +【本次目标】 +- 先只接入并验证上面 1–2 个 case。 +- 请实现 SAfactory adapter 的输入/输出处理：读取 request，取出 + env_params.dataset，通过 Gateway 调用模型，调用已有的 benchmark + 单 case 命令，读取结果并返回 SimulationStartResult JSON。 +- 不要重写 Docker image 内已有的 benchmark 单 case 运行/评测逻辑。 +- 请确认并报告：runner result JSON、benchmark 原生结果文件、Gateway + 轨迹以及最终 0–10 reward 的位置和内容。 + +请先检查 benchmark 的源码/README 和 SAfactory 的 +docs/guides/custom-environment_CN.md，再开始修改。请按所选 mode 创建或适配 +所需文件；如果信息不足，请只询问缺失字段，不要猜测 benchmark 命令或评分规则。 ``` -该 skill 不替代文档，而是引导 Agent 按需读取 `docs/guides/`、`docs/reference/` 和根 README，并优先参考标准环境 `env/geo3k/`。如果你的 Agent 支持本地 skill 搜索，可以把 `skills/safactory-workflows/` 加入其 skill 搜索路径；否则在请求中显式写出该路径即可。 +
+ +Agent 的接入范围是 SAfactory adapter 的边界处理，不包括重写 benchmark 镜像内部的单 case 运行逻辑。完整的文件职责、runner/result 契约、Docker/RJob 差异见[自定义环境指南](docs/guides/custom-environment_CN.md)和 skill 的[接入参考](skills/safactory-workflows/references/environment-integration.md)。 + +使用这个 skill 时，Agent 会按需读取 `docs/guides/`、`docs/reference/` 和根 README，并优先参考标准环境 `env/geo3k/`。你只需要提供上面列出的 benchmark 信息；如果 Agent 不支持自动发现本地 skill，请在 prompt 中显式写出 `skills/safactory-workflows/` 路径。 ## 🚀 快速开始 diff --git a/docs/guides/custom-environment.md b/docs/guides/custom-environment.md index 6276036a..fd78bc7d 100644 --- a/docs/guides/custom-environment.md +++ b/docs/guides/custom-environment.md @@ -10,7 +10,7 @@ The most important scheduling rule is: For every dataset row, the launcher creates a separate `job_environments` row, `session_id`, and gateway session. It then starts the same image and runner for that single row. This keeps model calls, gateway telemetry, runtime output, and evaluation rewards tied to one session. When integrating a benchmark, do not make the runner loop over the full benchmark dataset inside one episode. Put each benchmark case in its own dataset row and let Safactory schedule the rows independently. -You usually need five pieces: +You usually need a runtime image, runner, task config, start config, and (for scored benchmarks) a rule evaluator. The integration boundary is the SAfactory adapter's input/output handling: the benchmark harness or image should already know how to execute and score one case, and onboarding should not rewrite that logic. Before adding a new environment, run the standard Geo3K Docker smoke test from the root README. That confirms the Gateway, model route, storage, Docker permissions, and evaluator flow are working. When the baseline passes, use `env/geo3k` as the reference layout for a complete runtime with dataset loading, a runner, Docker startup config, and rule evaluation. @@ -18,8 +18,8 @@ Before adding a new environment, run the standard Geo3K Docker smoke test from t |-------|-------|------|---------| | Runtime image | `env_image` in the agent config. RJob deployments can override it from the start config. | Contains the agent or benchmark dependencies, the harness, and the language runtimes needed by the runner. | `myagent-image:latest`, `mybench-image:latest` | | Runner entrypoint | Usually `env//runner.py` or `env//runner.mjs`, invoked by `container.runner_entrypoint.command`. | Adapts Safactory to the native agent or benchmark. It reads the request, extracts `env_params.dataset`, calls the target model through the gateway, runs one task or case, and returns the result JSON. | `python /tmp/safactory-mybench-runner.py` | -| Task config | `env//_config.yaml`, passed with `--agent-config`. | Defines task rows: `env_name`, `env_image`, `dataset`, `env_num`, `env_params`, and optional evaluation settings. | `env/mybench/mybench_config.yaml` | -| Start config | `env//_start.yaml`, passed with `--agent-start-config`. | Defines how the matching runtime starts: runner entrypoint, working directory, environment variables, Docker or RJob settings, and mounts. `agent_name` must match `env_name`. | `env/mybench/mybench_start.yaml` | +| Task config | `env//_config.yaml`, passed with `--agent-config`. RJob mode also provides `_config.rjob.yaml`. | Defines task rows: `env_name`, `env_image`, `dataset`, `env_num`, and `env_params`. Each dataset row is one case/episode. | `env/mybench/mybench_config.yaml`, `env/mybench/mybench_config.rjob.yaml` | +| Start config | `env//_start.yaml`, passed with `--agent-start-config`. RJob mode also provides `_start.rjob.yaml`. | Defines how the matching runtime starts: runner entrypoint, working directory, environment variables, Docker or RJob settings, and mounts. `agent_name` must match `env_name`. | `env/mybench/mybench_start.yaml`, `env/mybench/mybench_start.rjob.yaml` | | Rule evaluator | Optional, commonly `env//rule_evaluator.py`. | Converts raw runtime metrics and the gateway trajectory into a Safactory score on the 0 to 10 scale. Simple smoke tests can omit it. Benchmarks usually should provide it. | `env/mybench/rule_evaluator.py` | Agents and benchmarks mostly differ in the runner and evaluator: @@ -230,9 +230,6 @@ environments: task_family: mybench bench_root: /workspace/MyBench output_root: /workspace/Safactory/results/mybench - evaluation: - rule_evaluator: env/mybench/rule_evaluator.py - rule_evaluator_timeout_s: 60 ``` ```jsonl @@ -296,7 +293,17 @@ container: idle_command: "tail -f /dev/null" ``` -`agent_name: mybench` must match `env_name: mybench` in `mybench_config.yaml`; otherwise the launcher cannot find the startup definition for the scheduled rows. +`agent_name: mybench` must match `env_name: mybench` in the selected task config +(`mybench_config.yaml` for Docker or `mybench_config.rjob.yaml` for RJob); +otherwise the launcher cannot find the startup definition for the scheduled rows. + +For RJob mode, provide `mybench_config.rjob.yaml` and +`mybench_start.rjob.yaml`, then run with `--mode rjob`. Keep +`container.runner_entrypoint` but add the `rjob:` settings. Do not copy local +Docker bind mounts into RJob: use cluster-accessible `rjob.mount_config` or +`rjob.mount`, list local runner dependencies in `rjob.embedded_files`, use an +image and storage visible to the cluster, and use a Gateway URL other than +`127.0.0.1` or `localhost`. See [RJob Mode](../internal/rjob-mode.md). ## 6. Run A Smoke Test diff --git a/docs/guides/custom-environment_CN.md b/docs/guides/custom-environment_CN.md index 773e940a..322ec173 100644 --- a/docs/guides/custom-environment_CN.md +++ b/docs/guides/custom-environment_CN.md @@ -10,7 +10,7 @@ Launcher 会为每一行 dataset 创建独立的 `job_environments` 记录、`session_id` 和 gateway session，然后用同一个镜像和 runner 执行这一行。这样模型调用、gateway 记录、运行时输出和评测 reward 都会绑定到同一个 session。接入 benchmark 时，不要让 runner 在一个 episode 里循环整个 benchmark dataset。应该让每个 benchmark case 对应一行 dataset，由 Safactory 按行独立调度。 -通常需要准备五个组件： +通常需要准备运行时镜像、runner、任务配置、启动配置和（有评分时）rule evaluator。接入的边界是 SAfactory adapter 的输入/输出处理：benchmark 单 case 的执行和评测逻辑应当已经存在于 benchmark harness 或 Docker 镜像中，接入时不重写这部分逻辑。 接入新环境前，先运行根目录 README 中的标准 Geo3K Docker smoke test。它可以先验证 Gateway、模型 route、存储、Docker 权限和 evaluator 链路是否正常。基线跑通后，再以 `env/geo3k` 作为完整 runtime 参考：它包含 dataset 加载、runner、Docker 启动配置和 rule evaluation。 @@ -18,8 +18,8 @@ Launcher 会为每一行 dataset 创建独立的 `job_environments` 记录、`se |------|------|------|------| | 运行时镜像 | agent config 中的 `env_image`。RJob 部署可以在 start config 中覆盖。 | 包含 agent 或 benchmark 依赖、harness，以及 runner 需要的语言运行时。 | `myagent-image:latest`、`mybench-image:latest` | | Runner entrypoint | 通常是 `env//runner.py` 或 `env//runner.mjs`，由 `container.runner_entrypoint.command` 调用。 | 连接 Safactory 与原生 agent 或 benchmark。它读取 request，取出 `env_params.dataset`，通过 gateway 调用被测模型，执行一个任务或 case，并返回结果 JSON。 | `python /tmp/safactory-mybench-runner.py` | -| 任务配置 | `env//_config.yaml`，通过 `--agent-config` 传入。 | 定义任务行：`env_name`、`env_image`、`dataset`、`env_num`、`env_params`，以及可选评测配置。 | `env/mybench/mybench_config.yaml` | -| 启动配置 | `env//_start.yaml`，通过 `--agent-start-config` 传入。 | 定义同名运行时如何启动：runner entrypoint、工作目录、环境变量、Docker 或 RJob 参数以及挂载。`agent_name` 必须匹配 `env_name`。 | `env/mybench/mybench_start.yaml` | +| 任务配置 | `env//_config.yaml`，通过 `--agent-config` 传入。RJob 模式另提供 `_config.rjob.yaml`。 | 定义任务行：`env_name`、`env_image`、`dataset`、`env_num` 和 `env_params`。每行 dataset 对应一个 case/episode。 | `env/mybench/mybench_config.yaml`、`env/mybench/mybench_config.rjob.yaml` | +| 启动配置 | `env//_start.yaml`，通过 `--agent-start-config` 传入。RJob 模式另提供 `_start.rjob.yaml`。 | 定义同名运行时如何启动：runner entrypoint、工作目录、环境变量、Docker 或 RJob 参数以及挂载。`agent_name` 必须匹配 `env_name`。 | `env/mybench/mybench_start.yaml`、`env/mybench/mybench_start.rjob.yaml` | | Rule evaluator | 可选，常见路径为 `env//rule_evaluator.py`。 | 把运行时写入的原始 `metrics` 和 gateway 轨迹转换为 Safactory 的 0 到 10 分。简单冒烟测试可以省略，benchmark 通常建议提供。 | `env/mybench/rule_evaluator.py` | Agent 和 benchmark 的差别主要体现在 runner 和 evaluator： @@ -230,9 +230,6 @@ environments: task_family: mybench bench_root: /workspace/MyBench output_root: /workspace/Safactory/results/mybench - evaluation: - rule_evaluator: env/mybench/rule_evaluator.py - rule_evaluator_timeout_s: 60 ``` ```jsonl @@ -296,7 +293,17 @@ container: idle_command: "tail -f /dev/null" ``` -`agent_name: mybench` 必须与 `mybench_config.yaml` 中的 `env_name: mybench` 一致，否则 launcher 找不到这些任务行对应的启动定义。 +`agent_name: mybench` 必须与所选任务配置（Docker 的 +`mybench_config.yaml` 或 RJob 的 `mybench_config.rjob.yaml`）中的 +`env_name: mybench` 一致，否则 launcher 找不到这些任务行对应的启动定义。 + +如果选择 RJob 模式，需要分别保存 `mybench_config.rjob.yaml` 和 +`mybench_start.rjob.yaml`，并使用 `--mode rjob`。RJob start config 仍然保留 +`container.runner_entrypoint`，但增加 `rjob:` 配置；本地 Docker 的 +`container.mounts` 不应直接复制到 RJob，应改为集群可访问的 +`rjob.mount_config`/`rjob.mount`。runner 依赖的本地文件要列在 +`rjob.embedded_files` 中，镜像和结果存储必须能被 RJob 集群访问，Gateway URL +也不能使用 `127.0.0.1` 或 `localhost`。详见[RJob 模式](../internal/rjob-mode_CN.md)。 ## 6. 运行冒烟测试 diff --git a/skills/safactory-workflows/SKILL.md b/skills/safactory-workflows/SKILL.md index 5972a952..7a97c3bd 100644 --- a/skills/safactory-workflows/SKILL.md +++ b/skills/safactory-workflows/SKILL.md @@ -1,69 +1,131 @@ --- name: safactory-workflows -description: Use this skill when helping users onboard a benchmark or environment into SAfactory, run SAfactory Docker-mode evaluation, or start SAfactory GRPO/RL training. It guides Codex to inspect the SAfactory repository, read the relevant docs, follow the established Geo3K baseline and my_env placeholder workflows, configure Gateway/model routes safely, and run or prepare launcher/RL commands without leaking private credentials. +description: Use this skill to onboard a benchmark or custom environment into SAfactory, run Docker or RJob evaluation, or prepare GRPO/RL training. For benchmark onboarding it defines the required adapter files, intake fields, single-case contract, and Docker/RJob-specific checks. --- # SAfactory Workflows -This skill helps agents operate the SAfactory repository for three user intents: +Use this skill for three related workflows: -1. Add a benchmark or custom environment to SAfactory. -2. Run Docker-mode evaluation for an environment. -3. Start GRPO/RL training for an environment. +1. onboard a benchmark or custom environment; +2. run Docker or RJob evaluation; +3. prepare or start GRPO/RL training after evaluation works. -Keep the root README and `docs/` as the source of truth. Use this skill to choose the right workflow, files, and checks; do not duplicate or rewrite full documentation. +The repository documentation is the source of truth. Do not invent a second runtime contract. For a new benchmark, read [references/environment-integration.md](references/environment-integration.md) and `docs/guides/custom-environment_CN.md` (or the English guide for an English request). For RJob-specific details, also read `docs/internal/rjob-mode_CN.md` or `docs/internal/rjob-mode.md`. -## First Steps +## Benchmark onboarding: intake gate -From the repository root: +Before editing, collect these fields from the user or infer them from the benchmark source/README: -1. Inspect the current layout before assuming paths: `rg --files README* docs env rl gateway`. -2. Identify the user intent: - - Benchmark/environment onboarding: read [references/environment-integration.md](references/environment-integration.md). - - Docker evaluation: read [references/docker-evaluation.md](references/docker-evaluation.md). - - GRPO/RL training: read [references/grpo-training.md](references/grpo-training.md). -3. Load only the matching reference file and the linked SAfactory docs needed for the task. -4. Prefer existing patterns in `env/geo3k/`, current docs, and root README commands. +- `mode`: exactly `docker` or `rjob`; +- environment name, normally lowercase with underscores; +- benchmark source/check-out path or repository; +- dataset path and the shape of one dataset row; +- one or two test-case IDs/rows; +- the benchmark's native single-case command (or the README section that defines it); +- Docker image name, if already available; +- where the native benchmark writes its result/output file; +- where the native score/reward is located and its scale/meaning. -## Repository Sources +The user must have (or provide enough information to run) **1–2 test cases**. The first milestone is a single-case pipeline whose output file and reward can be inspected. If the native case command, output path, score field, or selected mode is unknown, ask for that specific missing value before implementing. -Use these documents as canonical sources: +### Scope boundary -- Root quick start: `README.md` or `README_CN.md`. -- Custom environment guide: `docs/guides/custom-environment.md` or `docs/guides/custom-environment_CN.md`. -- Evaluation guide: `docs/guides/evaluation.md` or `docs/guides/evaluation_CN.md`. -- RL guide: `docs/guides/rl-training.md` or `docs/guides/rl-training_CN.md`. -- Gateway reference: `docs/reference/gateway.md` or `docs/reference/gateway_CN.md`. -- Environment reference: `docs/reference/environments.md` or `docs/reference/environments_CN.md`. -- CLI/config reference: `docs/reference/configuration.md` or `docs/reference/configuration_CN.md`. +The skill creates and validates the SAfactory adapter only. It does not rewrite the benchmark's case-solving/evaluation logic inside an existing Docker image, add a new benchmark harness, or repair a broken native single-case command unless the user explicitly expands the scope. Assume the image/harness can already execute one case; the adapter passes the current case and model calls in, then translates the native result out. -Use Chinese docs when the user writes Chinese; otherwise use English docs. +## Adapter files and responsibilities -## Operational Rules +For `mybench`, the onboarding output is under `env/mybench/`: -- Treat `env/geo3k/` as the standard reference implementation, not as a hardcoded target. -- Use generic placeholders such as `my_env` or the user's requested environment name for new workflows. -- Do not write private model endpoints, API keys, or internal route names into committed docs or shared configs. -- Do not overwrite `gateway/config.local.yaml` without checking its existing contents and preserving user edits. -- Ensure `--llm-model` and `RL_MODEL` match a Gateway `llm_routes` key. -- Keep Launcher, Gateway, and Buffer Server storage pointing at the same backend or SQLite URI. -- For new environments, run a minimal Docker evaluation before recommending RL training. -- Read `docs/internal/` only when the user explicitly asks about RJob, Sandbox, or internal deployment modes. +| File | Required | Responsibility | +|---|---:|---| +| `runner.py`, `runner.mjs`, or `runner.sh` | yes | Read `SimulationStartRequest` from stdin or `SAFACTORY_START_REQUEST_JSON`; read the current row from `env_params.dataset`; call the model through the current Gateway session URL; invoke the already-available native single-case command; collect the native output; print one `SimulationStartResult` JSON. | +| `mybench_config.yaml` | Docker mode | Define task rows and runtime metadata: `env_name`, `env_image`, `dataset`, `env_num`, and `env_params` (plus dataset loading options when needed). One dataset row must represent one episode/case. | +| `mybench_start.yaml` | Docker mode | Define how the runtime starts: runner entrypoint, working directory, environment variables, Docker settings, and mounts. `agent_name` must equal `env_name`. | +| `rule_evaluator.py` | recommended for scored benchmarks | Read runtime `metrics` and available trajectory data and convert the native score/pass result to a SAfactory reward in the `0–10` range. SAfactory auto-discovers `env/mybench/rule_evaluator.py`; do not register its path in YAML. | +| `Dockerfile` | optional | Build a dedicated image only when an existing image is unavailable or needs adapter dependencies. Do not move benchmark case logic into the adapter. | -## When Information Is Missing +The runner/result contract, config fields, and evaluator interface must follow `docs/guides/custom-environment_CN.md`. Keep stdout limited to the machine-readable result; write diagnostics to stderr. Put native case ID, score, pass/fail, reason, and output path in `metrics` so evaluation does not rerun the case. -Ask the smallest set of concrete questions required to proceed. Common blockers: +## Runtime modes -- New benchmark source path, dataset path, or native run command is unknown. -- Desired SAfactory environment name is unknown. -- Model route key or Gateway endpoint is unknown for an actual run. -- Docker image, runtime dependencies, or scoring contract is unknown. +The user must select one mode in the intake prompt. Implement that mode first; do not silently substitute the other mode. -If the user asks only for commands and prerequisites are missing, provide a fill-in command with placeholders and state exactly what must be replaced. +### Docker mode -## Verification Expectations +Create/use: -- For file edits: check created paths with `rg --files` and inspect changed files. -- For evaluation: run the smallest feasible `launcher.py --mode docker --enable-evaluation` smoke test when credentials, image, and data are available. -- For RL: first verify or ask for evidence that Docker evaluation passes; then prepare `env.sh`; start long-running training processes only when the user requested execution. -- If a command cannot be run because credentials, Docker, data, or dependencies are unavailable, report the blocker precisely and leave the exact next command. +```text +env/mybench/runner.py # or runner.mjs / runner.sh +env/mybench/mybench_config.yaml +env/mybench/mybench_start.yaml +env/mybench/rule_evaluator.py # when scored evaluation is required +``` + +The image is local, `container.mounts` are Docker bind mounts, and the smoke test uses `--mode docker`. The runner should use `SAFACTORY_GATEWAY_SESSION_URL_CONTAINER` rather than hardcoding `localhost`. + +### RJob mode + +Create/use both mode-specific files in addition to the runner/evaluator: + +```text +env/mybench/runner.py # or runner.mjs / runner.sh +env/mybench/mybench_config.rjob.yaml +env/mybench/mybench_start.rjob.yaml +env/mybench/rule_evaluator.py +``` + +`mybench_config.rjob.yaml` is the RJob task config (normally derived from the Docker task config). `mybench_start.rjob.yaml` contains the RJob runtime settings under `rjob:`. Use cluster-accessible images and storage; use `rjob.mount_config`/`rjob.mount` rather than local Docker bind mounts; list every local runner dependency in `rjob.embedded_files`. The Gateway URL must be reachable from the cluster and must not be `127.0.0.1` or `localhost`. The smoke test must use `--mode rjob` and the appropriate `--rjob-config`. + +The RJob files change deployment mechanics, not the single-case input/output contract. Keep the same dataset-row semantics, Gateway session handling, result JSON, and evaluator across both modes unless the runtime genuinely requires a documented difference. + +## Onboarding workflow + +1. Inspect the repository layout, the benchmark README/source, `env/geo3k/`, and the custom-environment guide. +2. Confirm the selected mode and validate the native benchmark command on 1–2 cases. Record the actual output file and native score/reward before writing the evaluator. +3. Implement only the adapter boundary: request parsing, dataset-row mapping, Gateway call, native command invocation, result-file discovery, and `SimulationStartResult` serialization. +4. Add the task config and selected-mode start config. Keep `env_name`/`agent_name` identical and make all paths/mounts explicit. +5. Add `rule_evaluator.py` when the benchmark has deterministic scoring. Convert the native result to `0–10`; do not rerun the benchmark case in the evaluator. +6. Run the smallest smoke test for the selected mode with one worker and 1–2 cases. Verify the runtime result JSON, native output file, Gateway trajectory, and final reward. +7. If the user requests both modes, repeat the config/start-config and smoke-test checks for the second mode; do not assume a Docker mount works in RJob. + +Typical commands are: + +```bash +# Docker +python launcher.py \ + --mode docker \ + --agent-config env/mybench/mybench_config.yaml \ + --agent-start-config env/mybench/mybench_start.yaml \ + --gateway-base-url http://127.0.0.1:8000/v1/sessions \ + --llm-model YOUR_ROUTE_KEY \ + --enable-evaluation \ + --db-path sqlite://mybench_smoke.db \ + --job-id mybench-docker-smoke \ + --pool-size 1 --max-workers 1 --max-steps 10 + +# RJob +python launcher.py \ + --mode rjob \ + --rjob-config config.yaml \ + --agent-config env/mybench/mybench_config.rjob.yaml \ + --agent-start-config env/mybench/mybench_start.rjob.yaml \ + --gateway-base-url http://GATEWAY_HOST:8000/v1/sessions \ + --llm-model YOUR_ROUTE_KEY \ + --enable-evaluation \ + --db-path sqlite://mybench_smoke.db \ + --job-id mybench-rjob-smoke \ + --pool-size 1 --max-workers 1 --max-steps 10 +``` + +Replace placeholders with real values. `--llm-model` must match a Gateway `llm_routes` key; never commit private endpoints or credentials. + +## Other workflows and repository rules + +- Read only the reference matching the request: `references/environment-integration.md`, `references/docker-evaluation.md`, or `references/grpo-training.md`. +- Treat `env/geo3k/` as the complete reference implementation, not as a hardcoded target. +- Keep Launcher, Gateway, and Buffer Server on the same storage backend or SQLite URI. +- Do not overwrite `gateway/config.local.yaml` without preserving user edits. +- Do not read `docs/internal/` except for RJob/Sandbox/internal deployment questions; RJob onboarding is the explicit exception above. +- Before recommending RL, establish that the selected-mode single-case evaluation and reward path work. +- For edits, inspect changed paths and configs. If Docker, RJob SDK, image, data, or credentials are unavailable, report the exact blocker and leave the next command rather than claiming the smoke test passed. diff --git a/skills/safactory-workflows/references/environment-integration.md b/skills/safactory-workflows/references/environment-integration.md index bb39c24f..7d74e1a2 100644 --- a/skills/safactory-workflows/references/environment-integration.md +++ b/skills/safactory-workflows/references/environment-integration.md @@ -1,86 +1,150 @@ -# Environment Integration Workflow - -Use this reference when the user asks to add a benchmark, custom environment, or new task suite to SAfactory. - -Canonical docs: - -- `docs/guides/custom-environment.md` -- `docs/guides/custom-environment_CN.md` -- `docs/guides/evaluation.md` -- `docs/guides/evaluation_CN.md` -- `docs/reference/configuration.md` -- `docs/reference/configuration_CN.md` -- `docs/reference/environments.md` -- `docs/reference/environments_CN.md` - -Reference implementation: - -- `env/geo3k/` is the standard complete environment layout. - -## Intake - -Before editing, identify: - -- Environment name, normally lowercase with underscores, for example `my_env`. -- Benchmark source path or repository. -- Dataset path and row format. -- Native task execution command. -- Required Docker image or dependencies. -- Result format and scoring rule. -- Whether rule-based evaluation is required. - -If any of these are unknown and cannot be inferred from local files, ask focused questions before implementing. - -## Target Files - -For environment `my_env`, expect or create: - -- `env/my_env/` -- `env/my_env/my_env_config.yaml` -- `env/my_env/my_env_start.yaml` -- `env/my_env/runner.py` or equivalent runner entrypoint. -- `env/my_env/rule_evaluator.py` when `--enable-evaluation` should produce rewards. -- `env/my_env/datasets/...` when data is stored in the repo. -- Optional `env/my_env/Dockerfile` when the environment needs a dedicated image. +# Benchmark / Environment Integration Workflow + +Use this reference when the user asks to add a benchmark, custom environment, or new task suite to SAfactory. The detailed runtime contract is defined by `docs/guides/custom-environment.md` and `docs/guides/custom-environment_CN.md`; this file turns it into an agent workflow. + +## Intake gate + +Do not start implementation until the request identifies: + +- mode: `docker` or `rjob`; +- environment name; +- benchmark source/check-out path; +- dataset path and one-row shape; +- 1–2 smoke-test rows/case IDs; +- native single-case command or the benchmark README section that defines it; +- existing Docker image, if any; +- native result/output file location; +- native score/reward location and scale. + +The user-facing copy/paste prompt is in the root `README_CN.md` / `README.md` under “Benchmark 接入 / Benchmark Onboarding Prompt”. If information is missing, ask only for the missing field. The first deliverable is one working single-case pipeline, not a full benchmark batch. + +## Scope boundary + +The adapter owns only the SAfactory boundary: + +- parse `SimulationStartRequest`; +- map `env_params.dataset` to one native case; +- call the model through the session-aware Gateway; +- invoke the native command that already exists in the image/harness; +- find/read the native result; +- emit the SAfactory result JSON and `metrics`. + +Do not reimplement benchmark case-solving logic, scoring logic already provided by the benchmark, or image internals as part of onboarding. If the native single-case command does not work independently, report that as a blocker or request explicit scope expansion. + +## Files to create or adapt + +For environment `mybench`, create the selected-mode files below under `env/mybench/`. + +| File | Required | What it does | +|---|---:|---| +| `runner.py`, `runner.mjs`, or `runner.sh` | yes | Reads the request, gets `env_params.dataset`, calls the current Gateway session, runs one native case, reads the native result, and prints one `SimulationStartResult` JSON. | +| `mybench_config.yaml` | Docker mode | Defines `env_name`, `env_image`, `dataset`, `env_num`, `env_params`, and dataset loading options. Each dataset row is one episode. | +| `mybench_start.yaml` | Docker mode | Defines `container.runner_entrypoint`, workdir, env vars, Docker mounts/args, and `agent_name`; `agent_name` must equal `env_name`. | +| `mybench_config.rjob.yaml` | RJob mode | RJob variant of the task config. Keep dataset-row and `env_params` semantics aligned with the Docker config. | +| `mybench_start.rjob.yaml` | RJob mode | RJob variant of the start config, including `rjob:` resources, cleanup, embedded files, and cluster-accessible mounts. | +| `rule_evaluator.py` | scored benchmark | Converts runtime `metrics` and trajectory information into a `0–10` reward. Auto-discovered at `env/mybench/rule_evaluator.py`; do not add an evaluator path to YAML. | +| `Dockerfile` | optional | Builds a dedicated image when no suitable image exists. It is not a place to rewrite the benchmark's native case logic. | + +RJob mode still needs the runner and evaluator. The two `.rjob.yaml` files are additional mode-specific deployment configs, not replacements for the runtime contract. + +## Runner contract + +The runner must: + +1. read JSON from stdin or `SAFACTORY_START_REQUEST_JSON`; +2. use `request.session_id` and `SAFACTORY_GATEWAY_SESSION_URL_CONTAINER` (or the request's session URL) for model calls; +3. read the current row from `request.env_params.dataset`; +4. pass that row to the native single-case command without looping over the dataset; +5. capture native score/pass/failure/output path in `metrics`; +6. print exactly one result object like: + +```json +{ + "session_id": "same-session-id", + "status": "succeeded", + "total_reward": 0.0, + "step_count": 1, + "terminated": true, + "truncated": false, + "error_text": null, + "metrics": { + "bench_case_id": "case-001", + "bench_score": 0.73, + "bench_passed": true, + "bench_reason": "all required checks passed", + "bench_output_path": "/workspace/Safactory/results/mybench/case-001.json" + } +} +``` -Use the existing naming pattern: `/_config.yaml` and `/_start.yaml`. +Keep diagnostics on stderr. A controlled task failure should be represented by a failed result; a non-zero process exit is reserved for runtime/infrastructure failure in JSON result mode. -## Implementation Checklist +## Config patterns -1. Run the standard Geo3K Docker smoke test from the root README when the platform setup is unverified. -2. Inspect `env/geo3k/` and any similar existing environment before adding abstractions. -3. Implement a runner that reads the SAfactory request payload and calls the target model through the provided Gateway session root and route key. -4. Return one machine-readable result per task according to the custom environment guide. -5. Add task config with dataset, runtime metadata, evaluator settings, and environment name. -6. Add start config with runner entrypoint, working directory, Docker settings, mounts, and environment variables. -7. Ensure `env_name` and `agent_name` match where the config format requires it. -8. Add or adapt `rule_evaluator.py` only when the benchmark has deterministic reward or scoring logic. -9. Run a minimal Docker smoke test with a small dataset or small sample count. +Docker task config: -## Smoke Test Shape +```yaml +environments: + - env_name: mybench + env_image: mybench-image:latest + env_num: 1 + dataset: ./datasets/cases.jsonl + dataset_load_mode: eager + env_params: + task_family: mybench + bench_root: /workspace/MyBench + output_root: /workspace/Safactory/results/mybench +``` -Use the root README and custom environment guide as the command source. A typical shape is: +Docker start config: + +```yaml +agent_name: mybench +container: + workdir: /workspace/MyBench + runner_entrypoint: + source: ./runner.py + target: /tmp/safactory-mybench-runner.py + command: "python /tmp/safactory-mybench-runner.py" + mounts: + - source: ./results + target: /workspace/Safactory/results + mode: rw + env: + NO_PROXY: host.docker.internal,localhost,127.0.0.1,::1 + no_proxy: host.docker.internal,localhost,127.0.0.1,::1 + extra_args: + - --add-host=host.docker.internal:host-gateway + idle_command: "tail -f /dev/null" +``` -```bash -python launcher.py \ - --mode docker \ - --agent-config env/my_env/my_env_config.yaml \ - --agent-start-config env/my_env/my_env_start.yaml \ - --gateway-base-url http://127.0.0.1:8000/v1/sessions \ - --llm-model my_env_model \ - --enable-evaluation \ - --db-path sqlite:///my_env_smoke.db \ - --job-id my_env-docker-smoke +RJob start config keeps the `container.runner_entrypoint` contract but adds, for example: + +```yaml +rjob: + name_prefix: mybench + image_pull_policy: IfNotPresent + no_packaging: true + cleanup_on_finish: true + resources: + cpu: 1 + gpu: 0 + memory_in_mb: 1024 + embedded_files: + - source: ./runner.py + target: /tmp/safactory-mybench-runner.py + mount_config: + - "gpfs://CLUSTER_STORAGE/results:/workspace/Safactory/results" ``` -Replace `my_env_model` with a real Gateway route key. Do not invent real private routes. +Do not copy local Docker bind mounts into RJob. RJob images and mounted storage must be accessible from the cluster, and a local runner dependency must be listed in `rjob.embedded_files`. -## Completion Criteria +## Validation -The integration is not done until: +1. Run or otherwise verify the native command on the 1–2 supplied cases first. +2. Build/pull the selected-mode image and verify the Gateway is reachable from the runtime. +3. Run one-worker smoke evaluation with `--enable-evaluation`. +4. Verify all four artifacts: runner result JSON, native benchmark output file, Gateway trajectory/request log, and final `0–10` reward. +5. For RJob, also verify the global `--rjob-config`, cluster storage, image pull, and non-loopback Gateway URL. -- Required files exist under `env/my_env/`. -- The runner can execute one task in Docker mode. -- Gateway route and storage assumptions are documented or configured. -- Evaluation produces a result and reward when `rule_evaluator.py` is present. -- Any missing external assets, images, or credentials are clearly listed. +Use the root README commands as the command source. Replace `YOUR_ROUTE_KEY` with an actual Gateway `llm_routes` key; never commit private routes or credentials. From af0dc7352235e655fa156f0d686e91154fd0c3c3 Mon Sep 17 00:00:00 2001 From: chenxinquan Date: Wed, 16 Sep 2026 15:31:52 +0800 Subject: [PATCH 2/4] =?UTF-8?q?[skills]=20safactory-workflows=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E7=8E=AF=E5=A2=83=E8=84=9A=E6=89=8B=E6=9E=B6=E4=B8=8E?= =?UTF-8?q?=E5=86=92=E7=83=9F=E6=B5=8B=E8=AF=95=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 scaffold_environment.py 生成新环境骨架，contract_smoke.py / live_smoke.py 冒烟脚本 - assets/environment 模板：adapter/runner/rule_evaluator 与 config/start 模板 - env/prmeval 作为参考实现：新增 adapter.py、request.smoke.json、README；runner/rule_evaluator 重构为 adapter 委托 - 文档与测试同步更新（SKILL.md、environment-integration.md、custom-environment 等） 迁移自 pr-92 分支工作区未提交改动 --- README.md | 84 ++---- README_CN.md | 77 ++--- docs/guides/custom-environment.md | 107 +++++-- docs/guides/custom-environment_CN.md | 97 +++++-- docs/reference/environments.md | 3 +- docs/reference/environments_CN.md | 3 +- env/prmeval/README.md | 49 ++++ env/prmeval/adapter.py | 183 ++++++++++++ env/prmeval/prmeval_config.rjob.yaml | 3 +- env/prmeval/prmeval_config.yaml | 3 +- env/prmeval/prmeval_start.rjob.yaml | 13 +- env/prmeval/prmeval_start.yaml | 14 +- env/prmeval/request.smoke.json | 34 +++ env/prmeval/results/.gitkeep | 1 + env/prmeval/rule_evaluator.py | 76 ++--- env/prmeval/runner.py | 248 +++++----------- skills/safactory-workflows/SKILL.md | 147 +++------- .../assets/environment/adapter.py | 27 ++ .../assets/environment/config.rjob.yaml.tmpl | 12 + .../assets/environment/config.yaml.tmpl | 11 + .../environment/request.smoke.json.tmpl | 14 + .../assets/environment/rule_evaluator.py | 35 +++ .../assets/environment/runner.py | 80 ++++++ .../assets/environment/start.docker.yaml.tmpl | 28 ++ .../assets/environment/start.rjob.yaml.tmpl | 27 ++ .../references/docker-evaluation.md | 19 +- .../references/environment-integration.md | 226 ++++++--------- .../scripts/contract_smoke.py | 169 +++++++++++ .../safactory-workflows/scripts/live_smoke.py | 147 ++++++++++ .../scripts/scaffold_environment.py | 69 +++++ tests/env/prmeval/test_prmeval_config.py | 36 +++ tests/env/prmeval/test_prmeval_runner.py | 104 +++++++ tests/test_environment_skill.py | 268 ++++++++++++++++++ 33 files changed, 1787 insertions(+), 627 deletions(-) create mode 100644 env/prmeval/README.md create mode 100644 env/prmeval/adapter.py create mode 100644 env/prmeval/request.smoke.json create mode 100644 env/prmeval/results/.gitkeep create mode 100644 skills/safactory-workflows/assets/environment/adapter.py create mode 100644 skills/safactory-workflows/assets/environment/config.rjob.yaml.tmpl create mode 100644 skills/safactory-workflows/assets/environment/config.yaml.tmpl create mode 100644 skills/safactory-workflows/assets/environment/request.smoke.json.tmpl create mode 100644 skills/safactory-workflows/assets/environment/rule_evaluator.py create mode 100644 skills/safactory-workflows/assets/environment/runner.py create mode 100644 skills/safactory-workflows/assets/environment/start.docker.yaml.tmpl create mode 100644 skills/safactory-workflows/assets/environment/start.rjob.yaml.tmpl create mode 100644 skills/safactory-workflows/scripts/contract_smoke.py create mode 100644 skills/safactory-workflows/scripts/live_smoke.py create mode 100644 skills/safactory-workflows/scripts/scaffold_environment.py create mode 100644 tests/env/prmeval/test_prmeval_config.py create mode 100644 tests/env/prmeval/test_prmeval_runner.py create mode 100644 tests/test_environment_skill.py diff --git a/README.md b/README.md index 7366de39..ba7bba7b 100644 --- a/README.md +++ b/README.md @@ -51,81 +51,55 @@ https://github.com/user-attachments/assets/4c551b27-ce4d-4fc8-8df6-d6dc8100cc88 ## 🧩 Agent Skill Quick Start -This repository includes a lightweight Agent skill that helps an Agent onboard a benchmark and run a minimal evaluation through the standard workflow: +This repository includes an Agent skill for template-based environment integration, local contract tests, optional Docker/RJob evaluation, and GRPO/RL workflows: ```text skills/safactory-workflows/SKILL.md ``` -This skill currently covers onboarding a new benchmark and running a minimal single-case evaluation in Docker or RJob mode. - ### Benchmark Onboarding Prompt -Before onboarding, prepare: - -1. Prepare **1–2 test cases** and make sure the benchmark's native single-case command works independently. -2. Choose the mode to onboard: `docker` (local image) or `rjob` (cluster RJob). -3. Prepare and provide the Agent with: - - - environment name; - - local benchmark source path or link; - - path to the test dataset; - - the native single-case command, or the corresponding section in the benchmark README; - - an accessible Docker image address; - - native benchmark result/output file path or naming rule; - - native score/reward location, range, and pass condition. +Provide the benchmark source, one-row dataset schema, 1–2 representative cases, native single-case command, and native output format. Specify Docker or RJob deployment when known. **Integration alone does not require evaluation, a score/reward definition, a running Gateway, or an internal cluster.** Only provide scoring details when evaluation is requested. -4. We recommend filling in the provided prompt and sending it to the Agent. The Agent will inspect the benchmark source/README and SAfactory docs, then implement the adapter, configs, and evaluator. -5. The Agent runs a minimal smoke test with 1–2 cases. Onboarding is complete when the runner result JSON, native benchmark output file, Gateway trajectory, and final `0–10` reward are all present and traceable to the same case. - -RJob users also need to prepare a Gateway URL. Do not use `localhost` or `127.0.0.1` as the Gateway address from an RJob container. +The Agent copies fixed runner/config templates, fills the environment hooks, and runs local contract tests with an owned mock model endpoint. Native dependencies must be available locally or represented by explicit test fixtures. Live deployment is a separate check when image/data/model/runtime access is available; a helper starts and stops Gateway without a second terminal. RJob live runs additionally require an existing cluster configuration and a cluster-reachable Gateway URL.
Expand to get the Benchmark Onboarding Prompt ```text -Use skills/safactory-workflows to onboard the following benchmark into SAfactory. +Use skills/safactory-workflows to onboard this benchmark into SAfactory. -[Execution mode] (required; choose one) -- mode: [docker / rjob] +[Goal] +- evaluation: [disabled (default, integration only) / enabled] +- target deployment mode: [docker / rjob / decide after adapter inspection] +- validation: [local contract first / also run live deployment when available] [Benchmark] -- environment name (for example, mybench): ____________________ -- benchmark source or checkout path/repository: ____________________ -- dataset path: ____________________ -- one dataset-row schema/field description: ____________________ -- 1–2 smoke-test case IDs or dataset rows: ____________________ -- native single-case benchmark command: ____________________ -- if the command is defined in a README, file and section: ____________________ -- Docker image, if one already exists: ____________________ - -[Results and scoring] -- native benchmark result/output file path or naming rule: ____________________ -- field or file containing the native score/reward: ____________________ -- score/reward range, meaning, and pass condition: ____________________ - -[Scope] -- Start with only the 1–2 cases above. -- Implement the SAfactory adapter boundary: read the request, take - env_params.dataset, call the model through the Gateway, invoke the existing - native single-case command, read its result, and return a - SimulationStartResult JSON object. -- Do not rewrite the benchmark single-case execution or evaluation logic - already inside the Docker image. -- Report and verify the runner result JSON, native benchmark output file, - Gateway trajectory, and final 0–10 reward. - -First inspect the benchmark source/README and -docs/guides/custom-environment.md, then edit the files required by the selected -mode. If information is missing, ask only for that field; do not guess the -benchmark command or scoring rule. +- environment name: ____________________ +- source or checkout path/repository: ____________________ +- dataset path and one-row schema: ____________________ +- 1–2 case IDs/rows: ____________________ +- native single-case command, or README file/section: ____________________ +- Docker image, if available: ____________________ +- native result/output path and format: ____________________ + +[Only if evaluation is enabled] +- native score field/file, range, meaning, and pass condition: ____________________ + +Start from the fixed templates based on docs/guides/custom-environment.md. +Keep protocol handling in runner.py and fill adapter.py with single-case row +mapping, Gateway model routing, native command execution, and output collection. +Do not reimplement native benchmark solving/scoring logic. +Only add rule_evaluator.py and --enable-evaluation if evaluation is enabled. +Run local contract checks without requiring Gateway or RJob cluster setup. +Report fixtures used, native output mapping, and which checks actually passed; +keep local contract, live deployment, and evaluation results distinct. +Ask only for missing information needed by the next dependent step. ```
-The Agent owns the SAfactory adapter boundary, not the benchmark's internal single-case logic. See [Custom Environments](docs/guides/custom-environment.md) and the skill's [integration reference](skills/safactory-workflows/references/environment-integration.md) for file responsibilities, the runner/result contract, and Docker/RJob differences. - -When you use this skill, the Agent reads `docs/guides/`, `docs/reference/`, and the root README as needed, using the standard `env/geo3k/` environment as its reference implementation. You only need to provide the benchmark information listed above; if your Agent cannot discover local skills automatically, include `skills/safactory-workflows/` explicitly in the prompt. +See [Custom Environments](docs/guides/custom-environment.md) and the skill's [integration reference](skills/safactory-workflows/references/environment-integration.md) for the templates, single-command test helpers, and Docker/RJob deployment settings. If your Agent cannot discover local skills automatically, include `skills/safactory-workflows/` explicitly in the prompt. ## 🚀 Quick Start diff --git a/README_CN.md b/README_CN.md index 51e9543f..31d1e4b2 100644 --- a/README_CN.md +++ b/README_CN.md @@ -51,7 +51,7 @@ https://github.com/user-attachments/assets/4c551b27-ce4d-4fc8-8df6-d6dc8100cc88 ## 🧩 Agent Skill 快速上手 -仓库内置了一个轻量 Agent skill，用于帮助 Agent 按标准 workflow 接入 benchmark 并完成最小评测： +仓库内置 Agent skill，支持从固定模板接入环境、本地契约测试、可选的 Docker/RJob 评测，以及 GRPO/RL workflow： ```text skills/safactory-workflows/SKILL.md @@ -59,68 +59,45 @@ skills/safactory-workflows/SKILL.md ### Benchmark 接入 Prompt -接入前需要准备： +请提供 benchmark 源码、单条 dataset row 格式、1–2 个代表性 case、原生单 case 命令和输出格式。已知时注明 Docker 或 RJob 部署模式。**仅接入不要求评测、score/reward 定义、已启动的 Gateway 或内部集群。** 只有需要评测时才提供评分信息。 -1. 准备 **1–2 个测试 case**，并确保 benchmark 原生的单 case 命令可以独立运行。 -2. 选择需要接入的模式：`docker`（本地镜像）或 `rjob`（集群 RJob）。 -3. 准备并告知Agent下方信息： - - - environment name； - - benchmark 源码本地路径或链接； - - 测试 dataset 所在路径； - - benchmark 原生单 case 执行命令，或源码 README 中对应的章节； - - 可访问的Docker image地址； - - benchmark 原生结果输出文件的路径/命名规则； - - 原生 score/reward 所在位置、取值范围和通过条件。 - -4. 建议使用提供的Prompt将信息填写好后发送给 Agent。Agent 会先检查 benchmark 源码/README 和 SAfactory 文档，再实现 adapter、配置文件和评测器。 -5. Agent 用 1–2 个 case 运行最小 smoke test。完成接入的验收标准是：runner result JSON、benchmark 原生结果文件、Gateway 轨迹和最终 `0–10` reward 都能找到并相互对应。 - -RJob 用户还需要准备 Gateway 地址；不要把 `localhost` 或 `127.0.0.1` 作为 RJob 容器访问 Gateway 的地址。 +Agent 会复制固定 runner/config 模板，填写环境 hook，并使用自动管理的 mock 模型端点执行本地契约测试。原生依赖需要本地可用或有明确的测试 fixture。镜像、数据、模型和 runtime 可用后再做真实部署验证；helper 负责 Gateway 启停，无需另开终端。RJob 真实运行还需要已有集群配置及集群可达的 Gateway URL。
展开获取接入 Prompt ```text -请使用 skills/safactory-workflows，将下面的 benchmark 接入 SAfactory。 +请使用 skills/safactory-workflows，将以下 benchmark 接入 SAfactory。 -【接入模式】（必填，只能选一个） -- mode: [docker / rjob] +【本次目标】 +- evaluation: [disabled（默认，仅接入）/ enabled] +- 目标部署模式: [docker / rjob / 检查 adapter 后再确定] +- 验证范围: [先做本地契约验证 / 条件具备时也做真实部署验证] 【Benchmark 信息】 -- environment name（例如 mybench）: ____________________ -- Benchmark 源码或 checkout 路径/仓库地址: ____________________ -- 数据集路径: ____________________ -- 单条 dataset row 的格式/字段说明: ____________________ -- 用于 smoke test 的 1–2 个 case ID 或 dataset row: ____________________ -- Benchmark 原生单 case 执行命令: ____________________ -- 如果命令来自 README，请填写文件和章节: ____________________ -- 对应的 Docker image（如已有）: ____________________ - -【结果与评分】 -- Benchmark 原生测评结果输出文件路径/命名规则: ____________________ -- 原生 score/reward 所在字段或文件: ____________________ -- score/reward 的取值范围、含义和通过条件: ____________________ - -【本次目标】 -- 先只接入并验证上面 1–2 个 case。 -- 请实现 SAfactory adapter 的输入/输出处理：读取 request，取出 - env_params.dataset，通过 Gateway 调用模型，调用已有的 benchmark - 单 case 命令，读取结果并返回 SimulationStartResult JSON。 -- 不要重写 Docker image 内已有的 benchmark 单 case 运行/评测逻辑。 -- 请确认并报告：runner result JSON、benchmark 原生结果文件、Gateway - 轨迹以及最终 0–10 reward 的位置和内容。 - -请先检查 benchmark 的源码/README 和 SAfactory 的 -docs/guides/custom-environment_CN.md，再开始修改。请按所选 mode 创建或适配 -所需文件；如果信息不足，请只询问缺失字段，不要猜测 benchmark 命令或评分规则。 +- environment name: ____________________ +- 源码或 checkout 路径/仓库地址: ____________________ +- 数据集路径和单条 row 格式: ____________________ +- 1–2 个 case ID/row: ____________________ +- 原生单 case 命令，或 README 文件/章节: ____________________ +- Docker image（如已有）: ____________________ +- 原生结果输出路径和格式: ____________________ + +【仅在启用评测时填写】 +- 原生分数字段/文件、范围、含义、通过条件: ____________________ + +从基于 docs/guides/custom-environment.md 的固定模板开始。 +runner.py 保留协议处理，adapter.py 只填写单 case 的数据映射、Gateway 模型 +配置、原生命令调用和输出收集。不要重写 benchmark 内部的解题/评分逻辑。 +仅在启用评测时添加 rule_evaluator.py 和 --enable-evaluation。 +先运行本地契约检查，不要求手动启动 Gateway 或配置 RJob 集群。 +报告使用的 fixture、原生输出映射及实际通过的检查；区分本地契约、真实部署 +和评测结果。仅询问下一步依赖的缺失信息。 ```
-Agent 的接入范围是 SAfactory adapter 的边界处理，不包括重写 benchmark 镜像内部的单 case 运行逻辑。完整的文件职责、runner/result 契约、Docker/RJob 差异见[自定义环境指南](docs/guides/custom-environment_CN.md)和 skill 的[接入参考](skills/safactory-workflows/references/environment-integration.md)。 - -使用这个 skill 时，Agent 会按需读取 `docs/guides/`、`docs/reference/` 和根 README，并优先参考标准环境 `env/geo3k/`。你只需要提供上面列出的 benchmark 信息；如果 Agent 不支持自动发现本地 skill，请在 prompt 中显式写出 `skills/safactory-workflows/` 路径。 +固定模板、单命令测试 helper 和 Docker/RJob 配置见[自定义环境指南](docs/guides/custom-environment_CN.md)和 skill 的[接入参考](skills/safactory-workflows/references/environment-integration.md)。如果 Agent 不支持自动发现本地 skill，请在 prompt 中显式写出 `skills/safactory-workflows/` 路径。 ## 🚀 快速开始 diff --git a/docs/guides/custom-environment.md b/docs/guides/custom-environment.md index fd78bc7d..ef2466c1 100644 --- a/docs/guides/custom-environment.md +++ b/docs/guides/custom-environment.md @@ -10,26 +10,39 @@ The most important scheduling rule is: For every dataset row, the launcher creates a separate `job_environments` row, `session_id`, and gateway session. It then starts the same image and runner for that single row. This keeps model calls, gateway telemetry, runtime output, and evaluation rewards tied to one session. When integrating a benchmark, do not make the runner loop over the full benchmark dataset inside one episode. Put each benchmark case in its own dataset row and let Safactory schedule the rows independently. -You usually need a runtime image, runner, task config, start config, and (for scored benchmarks) a rule evaluator. The integration boundary is the SAfactory adapter's input/output handling: the benchmark harness or image should already know how to execute and score one case, and onboarding should not rewrite that logic. +You usually need a runtime image, runner, task config, and start config. A rule evaluator is optional and is needed only when evaluation is requested. The integration boundary is the SAfactory adapter's input/output handling: the benchmark harness or image should already know how to execute one case and, when evaluation is requested, produce its native score, and onboarding should not rewrite that logic. -Before adding a new environment, run the standard Geo3K Docker smoke test from the root README. That confirms the Gateway, model route, storage, Docker permissions, and evaluator flow are working. When the baseline passes, use `env/geo3k` as the reference layout for a complete runtime with dataset loading, a runner, Docker startup config, and rule evaluation. +Start with local adapter contract tests; they do not require Docker, a running Gateway, model credentials, or an internal RJob cluster. Run a live smoke test when the deployment prerequisites are available. The Geo3K live baseline can help diagnose shared infrastructure, but it is not an onboarding prerequisite. Use `env/geo3k` as a reference for environment-specific behavior. | Piece | Where | Role | Example | |-------|-------|------|---------| -| Runtime image | `env_image` in the agent config. RJob deployments can override it from the start config. | Contains the agent or benchmark dependencies, the harness, and the language runtimes needed by the runner. | `myagent-image:latest`, `mybench-image:latest` | +| Runtime image | `env_image` in the task config. The RJob config variant normally names a cluster-pullable image. | Contains the agent or benchmark dependencies, the harness, and the language runtimes needed by the runner. | `myagent-image:latest`, `mybench-image:latest` | | Runner entrypoint | Usually `env//runner.py` or `env//runner.mjs`, invoked by `container.runner_entrypoint.command`. | Adapts Safactory to the native agent or benchmark. It reads the request, extracts `env_params.dataset`, calls the target model through the gateway, runs one task or case, and returns the result JSON. | `python /tmp/safactory-mybench-runner.py` | | Task config | `env//_config.yaml`, passed with `--agent-config`. RJob mode also provides `_config.rjob.yaml`. | Defines task rows: `env_name`, `env_image`, `dataset`, `env_num`, and `env_params`. Each dataset row is one case/episode. | `env/mybench/mybench_config.yaml`, `env/mybench/mybench_config.rjob.yaml` | | Start config | `env//_start.yaml`, passed with `--agent-start-config`. RJob mode also provides `_start.rjob.yaml`. | Defines how the matching runtime starts: runner entrypoint, working directory, environment variables, Docker or RJob settings, and mounts. `agent_name` must match `env_name`. | `env/mybench/mybench_start.yaml`, `env/mybench/mybench_start.rjob.yaml` | -| Rule evaluator | Optional, commonly `env//rule_evaluator.py`. | Converts raw runtime metrics and the gateway trajectory into a Safactory score on the 0 to 10 scale. Simple smoke tests can omit it. Benchmarks usually should provide it. | `env/mybench/rule_evaluator.py` | +| Rule evaluator | Optional, commonly `env//rule_evaluator.py`. | Converts raw runtime metrics and the gateway trajectory into a Safactory score on the 0 to 10 scale. Integration-only runs can omit it even when the native benchmark produces scores. Enable evaluation explicitly with `--enable-evaluation`. | `env/mybench/rule_evaluator.py` | Agents and benchmarks mostly differ in the runner and evaluator: - An agent runtime usually turns `env_params.dataset` into a prompt, tool task, or interaction flow. Evaluation uses a custom rule evaluator. -- A benchmark runtime usually wraps an existing benchmark harness. The runner handles only the current dataset row, writes native score, pass/fail status, reason, and output paths into `metrics`, and lets `rule_evaluator.py` normalize those details into a Safactory reward. +- A benchmark runtime usually wraps an existing benchmark harness. The runner handles only the current dataset row, writes available native outputs and paths into `metrics`, and optionally preserves score/pass details for `rule_evaluator.py` when evaluation is requested. -## 1. Write A Runner +## 1. Start From The Fixed Templates -Create `env/myagent/runner.py`: +Create the guide-derived files with: + +```bash +python skills/safactory-workflows/scripts/scaffold_environment.py myagent --mode docker +# Use --mode rjob to mark RJob as the first deployment target. +# Both Docker and RJob config pairs are generated; add --enable-evaluation only +# when evaluation is requested. +``` + +The [templates](../../skills/safactory-workflows/assets/environment/) keep protocol handling in `runner.py` and environment logic in `adapter.py:run_case`. Fill that hook and the YAML values; keep the protocol shell unchanged. For benchmarks, replace the generated greeting with the existing native single-case command and output mapping. The scaffolder refuses to overwrite existing directories. Its sample request/dataset verifies the scaffold only; replace those examples with representative cases before claiming integration success. + +The optional `rule_evaluator.py` template isolates scoring in `score_metrics`. It deliberately fails until the native score mapping is supplied. Scoring data is not required for integration-only work. See the [integration workflow](../../skills/safactory-workflows/references/environment-integration.md) for the complete file map and validation commands. + +The following compact example illustrates the underlying runner protocol; use the templates for new integration files: ```python #!/usr/bin/env python3 @@ -170,14 +183,14 @@ Fields: |-------|----------|---------| | `session_id` | yes | Must match the request session ID. | | `status` | yes | Use `succeeded` when the runner completed normally, even if the task score is low. Use `failed` for runtime errors. | -| `total_reward` | yes | Runtime-reported reward before any optional evaluator override. | +| `total_reward` | yes | Runtime-reported reward before any optional evaluator override. Use `0.0` for ungraded integration-only runs; this does not imply an evaluation result. | | `step_count` | yes | Number of steps reported by the runtime. | | `terminated` | yes | Whether the episode reached a normal stopping point. | | `truncated` | yes | Whether the episode stopped because of a timeout or step limit. | | `error_text` | no | Failure detail for runtime errors. | | `metrics` | no | Adapter-specific JSON object. This is the best place to keep benchmark outputs and file paths. | -For benchmarks, `metrics` is the main interface between the runner and `rule_evaluator.py`. Store enough information for evaluation to score the case without rerunning it: +When evaluation is requested, `metrics` is the main interface between the runner and `rule_evaluator.py`. Store enough information to score the case without rerunning it; these scoring fields are optional for integration-only work: ```json { @@ -243,6 +256,8 @@ Those two rows become two independent episodes, each with its own `session_id`, The start config describes how to execute the runner after Safactory allocates the image. `container.runner_entrypoint.command` runs once per dataset row. It must read the request JSON and return the result JSON. +Docker `container.mounts[].source` paths are resolved from the launcher's current working directory. The repository workflows run from the repository root, so generated templates use `./env//...` for environment-local adapter and dataset mounts. `runner_entrypoint.source` and RJob `embedded_files[].source` are instead resolved relative to their start-config file; keep those path bases distinct. + When `container.runner_entrypoint.source` points to a local file, the path is resolved relative to the start config file. Docker adds it as a mount at `target`; RJob embeds or stages the file through the RJob runtime config. The `command` should execute the file at the target path. Create `env/myagent/myagent_start.yaml`: @@ -305,34 +320,80 @@ Docker bind mounts into RJob: use cluster-accessible `rjob.mount_config` or image and storage visible to the cluster, and use a Gateway URL other than `127.0.0.1` or `localhost`. See [RJob Mode](../internal/rjob-mode.md). -## 6. Run A Smoke Test +### Environment parameter transport + +`env_params` is the benchmark's runtime configuration. The launcher includes +the fully expanded object (including the current dataset row) in +`SimulationStartRequest` on stdin and in `SAFACTORY_START_REQUEST_JSON`; the +fixed runner passes it unchanged to `adapter.py`. Use this channel for paths, +flags, timeouts, native command arguments, and benchmark-specific settings. + +Values under `container.env`/`rjob.env` are static process environment values +only (for example `NO_PROXY`). They are merged with launcher-provided +`SAFACTORY_*` variables at episode start. Do not duplicate the dataset or put +secrets in committed start YAML. If a native program only accepts environment +variables, have the adapter derive them from `request['env_params']` and pass +them to its subprocess. + +### The PRMEval reference layout + +`env/prmeval/` is the checked-in example of this separation: + +```text +env/prmeval/ + runner.py # fixed protocol shell + adapter.py # one-row PRMEval invocation + rule_evaluator.py # optional MSE -> 0..10 mapping + prmeval_config.yaml # Docker image + rows + env_params + prmeval_start.yaml # Docker command + mounts + prmeval_config.rjob.yaml # RJob image + rows + env_params + prmeval_start.rjob.yaml # RJob resources + embedded files + mounts + datasets/samples.jsonl +``` + +Use it as the standard when adding another benchmark: keep the runner shell +stable, put native logic in the adapter, and let the two RJob files add only +cluster-specific image/storage/resource settings. `env/prmeval/README.md` +describes the paths and the fast contract check. + +## 6. Validate Locally, Then Run A Live Smoke Test + +First fill `request.smoke.json` with one case and its environment parameters: + +```bash +python skills/safactory-workflows/scripts/contract_smoke.py \ + --runner env/myagent/runner.py \ + --request env/myagent/request.smoke.json \ + --require-model-call +``` + +If native dependencies are not installed locally, add +`--adapter path/to/adapter_fixture.py` to copy an explicit fixture beside the +runner. This validates only the SAfactory protocol and Gateway routing; it +does not claim that the native benchmark works. + +Repeat with `--input-mode env` to check environment-variable input. The helper owns a mock non-streaming chat endpoint and checks the result JSON/session identity, with no Gateway service or cluster. Native dependencies must be available locally or explicitly mocked in environment-specific tests. It does not verify image pulls, mounts, real Gateway persistence, or RJob scheduling. Include native failure and output-mapping fixtures; report precisely what was mocked. -Start the gateway first, then run one worker and one task at a time. This should mirror the Geo3K smoke-test shape, with only the environment paths and route key changed: +When a real image, dataset, route, and runtime are available, this command starts Gateway, waits for readiness, runs Launcher, and stops its processes. Run from the repository root using a config containing only 1–2 cases: ```bash -python launcher.py \ +python skills/safactory-workflows/scripts/live_smoke.py \ + --gateway-config gateway/config.local.yaml -- \ --mode docker \ --agent-config env/myagent/myagent_config.yaml \ --agent-start-config env/myagent/myagent_start.yaml \ - --gateway-base-url http://127.0.0.1:8000/v1/sessions \ --llm-model YOUR_ROUTE_KEY \ - --db-path sqlite://env_trajs.db \ --job-id myagent-docker-smoke \ - --pool-size 1 \ - --max-workers 1 \ - --max-steps 10 + --pool-size 1 --max-workers 1 --max-steps 10 ``` -Check: +The helper uses Gateway's SQLite URI when `--db-path` is omitted. If a Gateway is already running, verify its readiness, route, and storage, then run `launcher.py` directly; the helper does not take over an occupied port. RJob uses the same local contract tests, followed by a live run with `.rjob.yaml` files, the global `--rjob-config`, matching storage, and a cluster-reachable Gateway URL when a cluster is available. See the [integration workflow](../../skills/safactory-workflows/references/environment-integration.md) for the RJob command settings. -- `logs//main.log` for launcher and scheduler events. -- `logs//gateway.log` for gateway events. -- `logs//gateway_requests.jsonl` for request and response records. -- Adapter-specific output directories under `results/`. +For a live integration-only run, check the runtime result, native output, completed rows, Launcher logs, and Gateway request/trajectory records. Gateway helper output is in `logs/smoke/gateway.log`; other logs follow the configured paths. Omit `--enable-evaluation` and do not require a final normalized reward. Report local contract, live deployment, and evaluation results separately. ## Optional Evaluation -Add `env/myagent/rule_evaluator.py` and start the launcher with +When evaluation is requested, copy the [evaluator template](../../skills/safactory-workflows/assets/environment/rule_evaluator.py), fill `score_metrics`, and start the launcher with `--enable-evaluation`. The file is discovered from `agent_root` and `env_name`; no evaluator registration is read from `env_params`. diff --git a/docs/guides/custom-environment_CN.md b/docs/guides/custom-environment_CN.md index 322ec173..18a3e253 100644 --- a/docs/guides/custom-environment_CN.md +++ b/docs/guides/custom-environment_CN.md @@ -10,26 +10,36 @@ Launcher 会为每一行 dataset 创建独立的 `job_environments` 记录、`session_id` 和 gateway session，然后用同一个镜像和 runner 执行这一行。这样模型调用、gateway 记录、运行时输出和评测 reward 都会绑定到同一个 session。接入 benchmark 时，不要让 runner 在一个 episode 里循环整个 benchmark dataset。应该让每个 benchmark case 对应一行 dataset，由 Safactory 按行独立调度。 -通常需要准备运行时镜像、runner、任务配置、启动配置和（有评分时）rule evaluator。接入的边界是 SAfactory adapter 的输入/输出处理：benchmark 单 case 的执行和评测逻辑应当已经存在于 benchmark harness 或 Docker 镜像中，接入时不重写这部分逻辑。 +通常需要准备运行时镜像、runner、任务配置和启动配置。只有本次明确需要评测时才添加可选的 rule evaluator；原生 benchmark 有分数并不意味着接入必须包含评测。接入的边界是 SAfactory adapter 的输入/输出处理：benchmark 单 case 的执行和评测逻辑应当已经存在于 benchmark harness 或 Docker 镜像中，接入时不重写这部分逻辑。 -接入新环境前，先运行根目录 README 中的标准 Geo3K Docker smoke test。它可以先验证 Gateway、模型 route、存储、Docker 权限和 evaluator 链路是否正常。基线跑通后，再以 `env/geo3k` 作为完整 runtime 参考：它包含 dataset 加载、runner、Docker 启动配置和 rule evaluation。 +先进行本地 adapter 契约测试，无需 Docker、已启动的 Gateway、模型凭据或内部 RJob 集群。部署条件具备后再进行真实 smoke test。Geo3K 基线可用于排查公共基础设施问题，不是接入的前置门槛；`env/geo3k` 可作为环境特定逻辑的参考。 | 组件 | 位置 | 作用 | 示例 | |------|------|------|------| -| 运行时镜像 | agent config 中的 `env_image`。RJob 部署可以在 start config 中覆盖。 | 包含 agent 或 benchmark 依赖、harness，以及 runner 需要的语言运行时。 | `myagent-image:latest`、`mybench-image:latest` | +| 运行时镜像 | 任务配置中的 `env_image`。RJob 配置通常改为集群可拉取的镜像。 | 包含 agent 或 benchmark 依赖、harness，以及 runner 需要的语言运行时。 | `myagent-image:latest`、`mybench-image:latest` | | Runner entrypoint | 通常是 `env//runner.py` 或 `env//runner.mjs`，由 `container.runner_entrypoint.command` 调用。 | 连接 Safactory 与原生 agent 或 benchmark。它读取 request，取出 `env_params.dataset`，通过 gateway 调用被测模型，执行一个任务或 case，并返回结果 JSON。 | `python /tmp/safactory-mybench-runner.py` | | 任务配置 | `env//_config.yaml`，通过 `--agent-config` 传入。RJob 模式另提供 `_config.rjob.yaml`。 | 定义任务行：`env_name`、`env_image`、`dataset`、`env_num` 和 `env_params`。每行 dataset 对应一个 case/episode。 | `env/mybench/mybench_config.yaml`、`env/mybench/mybench_config.rjob.yaml` | | 启动配置 | `env//_start.yaml`，通过 `--agent-start-config` 传入。RJob 模式另提供 `_start.rjob.yaml`。 | 定义同名运行时如何启动：runner entrypoint、工作目录、环境变量、Docker 或 RJob 参数以及挂载。`agent_name` 必须匹配 `env_name`。 | `env/mybench/mybench_start.yaml`、`env/mybench/mybench_start.rjob.yaml` | -| Rule evaluator | 可选，常见路径为 `env//rule_evaluator.py`。 | 把运行时写入的原始 `metrics` 和 gateway 轨迹转换为 Safactory 的 0 到 10 分。简单冒烟测试可以省略，benchmark 通常建议提供。 | `env/mybench/rule_evaluator.py` | +| Rule evaluator | 可选，常见路径为 `env//rule_evaluator.py`。 | 把运行时写入的原始 `metrics` 和 gateway 轨迹转换为 Safactory 的 0 到 10 分。仅接入时可以省略，即使 benchmark 自带分数；通过 `--enable-evaluation` 显式开启评测。 | `env/mybench/rule_evaluator.py` | Agent 和 benchmark 的差别主要体现在 runner 和 evaluator： - Agent 运行时通常把 `env_params.dataset` 转换为 prompt、工具任务或交互流程。评测使用自定义 rule evaluator。 -- Benchmark 运行时通常包装已有 benchmark harness。runner 只处理当前 dataset 行对应的单个 case，把原生分数、通过状态、原因和输出路径写入 `metrics`，再由 `rule_evaluator.py` 统一换算成 Safactory reward。 +- Benchmark 运行时通常包装已有 benchmark harness。runner 只处理当前 dataset 行对应的单个 case，把可用的原生输出和路径写入 `metrics`；需要评测时再保留评分信息，供 `rule_evaluator.py` 换算 reward。 -## 1. 编写 Runner +## 1. 从固定模板开始 -创建 `env/myagent/runner.py`： +```bash +python skills/safactory-workflows/scripts/scaffold_environment.py myagent --mode docker +# 如果首个目标是 RJob，可改用 --mode rjob；两种模式的 config/start 文件都会生成。 +# 只有需要评测时才追加 --enable-evaluation。 +``` + +[模板目录](../../skills/safactory-workflows/assets/environment/)将协议处理固定在 `runner.py`，环境逻辑放在 `adapter.py:run_case`。填写 hook 和 YAML 参数，保留协议外壳。接入 benchmark 时，把示例问候替换成已有的原生单 case 命令和输出映射。脚手架拒绝覆盖已有目录；示例 request/dataset 只能验证脚手架，必须换成有代表性的 case 才能证明接入有效。 + +可选的 `rule_evaluator.py` 只需填写 `score_metrics`；没有配置评分映射时会明确失败。仅接入不需要评分字段。完整文件职责和命令见[接入 workflow](../../skills/safactory-workflows/references/environment-integration.md)。 + +下面的紧凑示例用于解释 runner 协议；新接入文件应从模板创建： ```python #!/usr/bin/env python3 @@ -177,7 +187,7 @@ Safactory 会通过 stdin 和 `SAFACTORY_START_REQUEST_JSON` 同时传入 `Simul | `error_text` | 否 | 运行时错误的详细信息。 | | `metrics` | 否 | 适配器自定义 JSON 对象。benchmark 输出和文件路径建议放在这里。 | -对于 benchmark，`metrics` 是 runner 和 `rule_evaluator.py` 之间最主要的接口。请保存足够信息，让评测阶段不需要重新运行 case 就能打分： +需要评测时，`metrics` 是 runner 和 `rule_evaluator.py` 的主要接口。请保存足够信息，让评测不必重新运行 case；仅接入时不要求这些评分字段： ```json { @@ -243,6 +253,8 @@ environments: 启动配置描述 Safactory 分配镜像后如何执行 runner。`container.runner_entrypoint.command` 会针对每一行 dataset 执行一次。它必须读取 request JSON，并返回 result JSON。 +Docker 的 `container.mounts[].source` 按启动 Launcher 时的当前工作目录解析；本仓库的命令从仓库根目录执行，因此模板会使用 `./env//...` 指向环境目录。`runner_entrypoint.source` 以及 RJob 的 `embedded_files[].source` 则按各自 start 配置文件所在目录解析，二者不要混用。 + 当 `container.runner_entrypoint.source` 指向本地文件时，该路径会相对 start config 文件解析。Docker 会把它挂载到 `target`，RJob 会通过 RJob runtime config 嵌入或分发该文件。`command` 应该执行 target 路径上的文件。 创建 `env/myagent/myagent_start.yaml`： @@ -305,34 +317,75 @@ container: `rjob.embedded_files` 中，镜像和结果存储必须能被 RJob 集群访问，Gateway URL 也不能使用 `127.0.0.1` 或 `localhost`。详见[RJob 模式](../internal/rjob-mode_CN.md)。 -## 6. 运行冒烟测试 +### 环境参数如何透传 + +`env_params` 是 benchmark 的运行时配置。Launcher 会把展开后的完整对象 +（包括当前 dataset 行）放入 `SimulationStartRequest` 的 stdin 和 +`SAFACTORY_START_REQUEST_JSON`，固定的 runner 再原样传给 `adapter.py`。 +路径、开关、超时、原生命令参数和 benchmark 特有配置都应放在这里。 + +`container.env`/`rjob.env` 只用于静态进程环境变量（例如 `NO_PROXY`），episode +启动时会和 Launcher 注入的 `SAFACTORY_*` 变量合并。不要在提交的 start YAML +中重复 dataset 或写入密钥。如果原生程序只能读取环境变量，应由 adapter 从 +`request['env_params']` 派生后传给子进程。 + +### PRMEval 标准目录 + +`env/prmeval/` 是这套分层的仓库内示例： + +```text +env/prmeval/ + runner.py # 固定协议外壳 + adapter.py # 单行 PRMEval 调用 + rule_evaluator.py # 可选的 MSE -> 0..10 映射 + prmeval_config.yaml # Docker 镜像、任务行和 env_params + prmeval_start.yaml # Docker 命令和挂载 + prmeval_config.rjob.yaml # RJob 镜像、任务行和 env_params + prmeval_start.rjob.yaml # RJob 资源、嵌入文件和挂载 + datasets/samples.jsonl +``` + +接入其他 benchmark 时保持 runner 外壳稳定，把原生逻辑放到 adapter；RJob +的两个文件只增加集群相关的镜像、存储和资源设置。`env/prmeval/README.md` +说明了各文件职责和快速契约检查命令。 + +## 6. 本地验证与真实 Smoke Test + +先将 `request.smoke.json` 填成单个 case 及其环境参数： + +```bash +python skills/safactory-workflows/scripts/contract_smoke.py \ + --runner env/myagent/runner.py \ + --request env/myagent/request.smoke.json \ + --require-model-call +``` + +如果本地没有原生依赖，可追加 +`--adapter path/to/adapter_fixture.py`，把明确的 fixture 复制到 runner 旁边。 +这只验证 SAfactory 协议和 Gateway 路由，不代表 benchmark 原生逻辑可用。 + +追加 `--input-mode env` 再验证环境变量输入。helper 自动启动本地非流式 chat mock，检查结果 JSON 和 session 一致性，不依赖 Gateway 服务或集群。原生依赖需要本地可用，或在环境测试中明确用 fixture 替代；这不能证明镜像、挂载、真实 Gateway 落库或 RJob 调度可用。应补充原生命令失败和输出映射测试，并报告 mock 的范围。 -先启动 gateway，然后以单 worker、单并发运行最小测试。命令形态应与 Geo3K smoke test 一致，只替换环境路径和 route key： +真实镜像、数据、route 和 runtime 可用后，下面的单条命令负责启动 Gateway、等待 ready、运行 Launcher、清理自身进程。请在仓库根目录执行，任务配置只包含 1–2 个 case： ```bash -python launcher.py \ +python skills/safactory-workflows/scripts/live_smoke.py \ + --gateway-config gateway/config.local.yaml -- \ --mode docker \ --agent-config env/myagent/myagent_config.yaml \ --agent-start-config env/myagent/myagent_start.yaml \ - --gateway-base-url http://127.0.0.1:8000/v1/sessions \ --llm-model YOUR_ROUTE_KEY \ - --db-path sqlite://env_trajs.db \ --job-id myagent-docker-smoke \ - --pool-size 1 \ - --max-workers 1 \ - --max-steps 10 + --pool-size 1 --max-workers 1 --max-steps 10 ``` -重点检查： +省略 `--db-path` 时，helper 使用 Gateway 的 SQLite URI。若 Gateway 已运行，应检查其 ready、route 和存储后直接调用 `launcher.py`；helper 不接管已占用端口。RJob 同样先执行本地契约测试；集群可用后再使用 `.rjob.yaml`、全局 `--rjob-config`、一致的存储和集群可达 Gateway URL 做真实部署验证。RJob 参数见[接入 workflow](../../skills/safactory-workflows/references/environment-integration.md)。 -- `logs//main.log`：launcher 和 scheduler 事件。 -- `logs//gateway.log`：gateway 事件。 -- `logs//gateway_requests.jsonl`：请求和响应记录。 -- `results/` 下该适配器自己的输出目录。 +仅接入的真实运行检查 runner JSON、原生输出、完成状态及 Gateway 请求/轨迹记录。helper 的 Gateway 日志为 `logs/smoke/gateway.log`，其他日志按配置路径查看。省略 `--enable-evaluation`，不要求最终归一化 reward；本地契约、真实部署和评测结果分别报告。未评分时 `total_reward: 0.0` 只满足返回协议，不代表评测结果。 ## 可选评测 -添加 `env/myagent/rule_evaluator.py`，并使用 `--enable-evaluation` 启动 +需要评测时，从[评测器模板](../../skills/safactory-workflows/assets/environment/rule_evaluator.py)创建 `env/myagent/rule_evaluator.py`，填写 `score_metrics`，再使用 `--enable-evaluation` 启动 launcher。系统根据 `agent_root` 和 `env_name` 自动发现该文件，不从 `env_params` 读取 evaluator 注册信息。 diff --git a/docs/reference/environments.md b/docs/reference/environments.md index 5dc69f1c..bc3da13c 100644 --- a/docs/reference/environments.md +++ b/docs/reference/environments.md @@ -6,13 +6,14 @@ SAfactory v2 treats each environment as an external agent runtime. A runtime is - an agent start config: Docker, RJob, or Sandbox startup details for the runtime; - an optional `rule_evaluator.py`: reward conversion after rollout. -The standard environment for onboarding, smoke tests, evaluation, and RL examples is **Geo3K**. +The standard end-to-end environment for smoke tests, evaluation, and RL examples is **Geo3K**. For new benchmark onboarding, use the template-based **PRMEval** layout as the structural reference and keep Geo3K as a behavior-specific example. ## Environment Matrix | Environment | `env_name` / `agent_name` | Domain | Config | Start config | Runtime modes | Evaluator | |-------------|----------------------------|--------|--------|--------------|---------------|-----------| | Geo3K | `geo3k` | Geometry / VLM QA | `env/geo3k/geo3k_config.yaml` | `env/geo3k/geo3k_start.yaml` | Docker; RL template | `env/geo3k/rule_evaluator.py` | +| PRMEval | `prmeval` | Progress benchmark harness | `env/prmeval/prmeval_config.yaml` / `.rjob.yaml` | `env/prmeval/prmeval_start.yaml` / `.rjob.yaml` | Docker; RJob | `env/prmeval/rule_evaluator.py` | | OpenClaw | `openclaw` | General OpenClaw CLI tasks | `env/openclaw/openclaw_config.yaml` | `env/openclaw/openclaw_start.yaml` | Docker | Optional | | OpenRT | `openrt` | Safety / red-team benchmark | `env/openrt/openrt_config.yaml` | `env/openrt/openrt_start.yaml` | Docker | `env/openrt/rule_evaluator.py` | | OpenRT RJob | `openrt` | Remote OpenRT benchmark | `env/openrt/openrt_config.rjob.yaml` | `env/openrt/openrt_start.rjob.yaml` | RJob | `env/openrt/rule_evaluator.py` | diff --git a/docs/reference/environments_CN.md b/docs/reference/environments_CN.md index 98071088..10709df7 100644 --- a/docs/reference/environments_CN.md +++ b/docs/reference/environments_CN.md @@ -6,13 +6,14 @@ SAfactory v2 将每个环境视为外部 agent runtime。一个 runtime 由以 - agent start config：Docker、RJob 或 Sandbox 的启动细节； - 可选的 `rule_evaluator.py`：rollout 后的 reward 转换。 -用于新用户上手、smoke test、评测和 RL 示例的标准环境是 **Geo3K**。 +用于端到端 smoke test、评测和 RL 示例的标准环境仍是 **Geo3K**。接入新 benchmark 时，目录结构应参考基于模板的 **PRMEval**，而环境特定行为可参考 Geo3K。 ## 环境矩阵 | 环境 | `env_name` / `agent_name` | 领域 | Config | Start config | Runtime 模式 | Evaluator | |------|----------------------------|------|--------|--------------|--------------|-----------| | Geo3K | `geo3k` | 几何 / VLM QA | `env/geo3k/geo3k_config.yaml` | `env/geo3k/geo3k_start.yaml` | Docker；RL 模板 | `env/geo3k/rule_evaluator.py` | +| PRMEval | `prmeval` | Progress benchmark harness | `env/prmeval/prmeval_config.yaml` / `.rjob.yaml` | `env/prmeval/prmeval_start.yaml` / `.rjob.yaml` | Docker；RJob | `env/prmeval/rule_evaluator.py` | | OpenClaw | `openclaw` | 通用 OpenClaw CLI 任务 | `env/openclaw/openclaw_config.yaml` | `env/openclaw/openclaw_start.yaml` | Docker | 可选 | | OpenRT | `openrt` | 安全 / red-team benchmark | `env/openrt/openrt_config.yaml` | `env/openrt/openrt_start.yaml` | Docker | `env/openrt/rule_evaluator.py` | | OpenRT RJob | `openrt` | 远程 OpenRT benchmark | `env/openrt/openrt_config.rjob.yaml` | `env/openrt/openrt_start.rjob.yaml` | RJob | `env/openrt/rule_evaluator.py` | diff --git a/env/prmeval/README.md b/env/prmeval/README.md new file mode 100644 index 00000000..e3662c8e --- /dev/null +++ b/env/prmeval/README.md @@ -0,0 +1,49 @@ +# PRMEval standard environment + +`env/prmeval` is the reference layout for onboarding a benchmark that already +has a container image, a per-case dataset row, a native runner, and optional +native metrics. + +| File | Responsibility | +| --- | --- | +| `runner.py` | Fixed SAfactory protocol shell. Reads one `SimulationStartRequest`, calls the adapter, and emits exactly one result JSON. | +| `adapter.py` | PRMEval-only logic: writes the current row to a temporary JSONL, injects the session-aware Gateway URL/model, runs `Evaluator`, and maps native artifacts into `metrics`. | +| `rule_evaluator.py` | Optional evaluation stage. Reads stored PRMEval MSE and maps it to a 0--10 reward; it never reruns PRMEval. | +| `prmeval_config.yaml` | Docker task rows, local image, and environment parameters. One row is one episode. | +| `prmeval_start.yaml` | Docker runner command, explicit read-only adapter/dataset mounts, and result mount. | +| `prmeval_config.rjob.yaml` | RJob task rows and cluster-pullable image. | +| `prmeval_start.rjob.yaml` | RJob resources, embedded adapter, and cluster-accessible result/dataset mounts. | +| `datasets/samples.jsonl` | Native PRMEval trajectory rows. Frame paths must exist at the same path inside the runtime image/mount. | + +The Docker start config assumes commands are launched from the SAfactory +repository root (as `live_smoke.py` requires). Its bind sources therefore use +`./env/prmeval/...`; the RJob config instead resolves embedded-file sources +relative to its own config file and uses cluster-visible storage. + +## Fast local contract check + +The full PRMEval wheel and model are not needed to validate the SAfactory +protocol. Use the skill helper with a temporary adapter fixture: + +```bash +python skills/safactory-workflows/scripts/contract_smoke.py \ + --runner env/prmeval/runner.py \ + --adapter skills/safactory-workflows/assets/environment/adapter.py \ + --request env/prmeval/request.smoke.json \ + --require-model-call +``` + +The `--adapter` argument is intentional: it substitutes the tiny standard +fixture because PRMEval's wheel is not required for this protocol-only check. +To validate the real PRMEval adapter, run it inside the built image (or install +its dependencies locally) and remove `--adapter`. + +For a real PRMEval run, build/pull the selected image and use a Gateway route +through the normal Launcher command. `--enable-evaluation` is opt-in; an +integration-only run can succeed with `total_reward: 0.0` while still exposing +the native MSE/output paths in `metrics`. + +Before an RJob run, replace the example `gpfs://...` sources in +`prmeval_start.rjob.yaml` with storage visible to the cluster and use a +cluster-reachable Gateway URL. Local Docker bind mounts cannot be reused by +RJob. diff --git a/env/prmeval/adapter.py b/env/prmeval/adapter.py new file mode 100644 index 00000000..593b6673 --- /dev/null +++ b/env/prmeval/adapter.py @@ -0,0 +1,183 @@ +"""PRMEval-specific adapter for the fixed SAfactory runner protocol. + +The runner owns the SAfactory contract; this module owns only the native +PRMEval invocation. It deliberately evaluates the one dataset row supplied +in ``request['env_params']['dataset']`` and never loops over the source file. +""" + +from __future__ import annotations + +import copy +import json +import os +from pathlib import Path +import tempfile +from typing import Any + + +def run_case( + request: dict[str, Any], + task: dict[str, Any], + session_url: str, +) -> tuple[dict[str, Any], int]: + """Run one PRMEval trajectory and return ``(metrics, step_count)``. + + ``prmeval`` is imported lazily so the protocol shell can be imported and + contract-tested on a workstation without the benchmark wheel installed. + """ + if not isinstance(task, dict): + raise TypeError("env_params.dataset must be a JSON object") + env_params = request.get("env_params") + if not isinstance(env_params, dict): + raise TypeError("env_params must be a JSON object") + + native_config = env_params.get("prmeval") + if not isinstance(native_config, dict): + raise ValueError("env_params.prmeval must be a JSON object") + + prepared_task = _prepare_trajectory(task, env_params) + session_id = _required_text(request.get("session_id"), "session_id") + job_id = _required_text(request.get("job_id"), "job_id") + results_root = _first_text( + env_params.get("results_root"), + os.environ.get("SAFACTORY_RESULTS_ROOT"), + "/tmp/safactory-prmeval-results", + ) + run_name = _safe_path_part(f"{job_id}-{session_id}") + + # Keep the temporary source row private to this episode. A fixed path + # would race when env_num/pool-size is greater than one. + with tempfile.TemporaryDirectory(prefix="safactory-prmeval-") as temp_dir: + source_path = Path(temp_dir) / "trajectory.jsonl" + source_path.write_text( + json.dumps(prepared_task, ensure_ascii=False) + "\n", encoding="utf-8" + ) + + config = copy.deepcopy(native_config) + sampling = config.setdefault("sampling", {}) + if not isinstance(sampling, dict): + raise TypeError("env_params.prmeval.sampling must be a JSON object") + sampling["paths"] = [str(source_path)] + sampling["max_trajectories"] = 1 + sampling.setdefault("eval_types", ["progress"]) + + infer = config.setdefault("infer", {}) + if not isinstance(infer, dict): + raise TypeError("env_params.prmeval.infer must be a JSON object") + infer["model_id"] = _first_text( + os.environ.get("SAFACTORY_ROUTE_MODEL"), request.get("model") + ) + infer["base_url"] = session_url.rstrip("/") + infer.setdefault("api_key", os.environ.get("OPENAI_API_KEY", "EMPTY")) + if request.get("temperature") is not None: + infer["temperature"] = float(request["temperature"]) + options = infer.setdefault("options", {}) + if not isinstance(options, dict): + raise TypeError("env_params.prmeval.infer.options must be a JSON object") + options.setdefault("keep_base_url", True) + + output_dir = Path(results_root) / run_name + config["output_dir"] = str(output_dir.parent) + config["run_name"] = output_dir.name + config.setdefault("resume", False) + + # Import only after the request/config has been validated. This keeps + # malformed requests deterministic and makes local protocol tests cheap. + from prmeval.core import EvalConfig, Evaluator # type: ignore + + summary = Evaluator(EvalConfig.model_validate(config)).run() + + metrics = _flatten_summary(summary) + metrics.update( + { + "bench": "prmeval", + "case_id": task.get("id") or task.get("sample_id") or task.get("task_id"), + "native_output_dir": str(output_dir), + "session_id": session_id, + } + ) + progress = metrics.get("mse") + step_count = _step_count(summary, default=1) + if progress is not None: + metrics["native_score_available"] = True + return metrics, step_count + + +def _prepare_trajectory(task: dict[str, Any], env_params: dict[str, Any]) -> dict[str, Any]: + """Resolve relative media references without changing the source row.""" + prepared = copy.deepcopy(task) + frames = prepared.get("frames") + if isinstance(frames, str) and not Path(frames).is_absolute(): + media_root = _first_text(env_params.get("media_root"), env_params.get("frames_root")) + if not media_root: + raise ValueError("relative frames path requires env_params.media_root") + prepared["frames"] = str((Path(media_root) / frames).resolve()) + return prepared + + +def _flatten_summary(summary: Any) -> dict[str, Any]: + if not isinstance(summary, dict): + raise TypeError("PRMEval summary must be a JSON object") + metrics = summary.get("metrics") if isinstance(summary.get("metrics"), dict) else {} + progress = metrics.get("progress") if isinstance(metrics.get("progress"), dict) else {} + flattened: dict[str, Any] = { + "native_summary": summary, + "coverage": summary.get("coverage", {}), + "mse": progress.get("mse"), + "pearson": progress.get("pearson"), + "num_samples": progress.get("num_samples"), + "bench_output_path": summary.get("details") or summary.get("predictions"), + } + details = summary.get("details") + if details: + prediction = _first_prediction(Path(str(details))) + if prediction is not None: + flattened["predicted_progress"] = prediction + return {key: value for key, value in flattened.items() if value is not None} + + +def _first_prediction(path: Path) -> list[float] | None: + try: + with path.open(encoding="utf-8") as handle: + for line in handle: + if not line.strip(): + continue + row = json.loads(line) + values = ((row.get("prediction") or {}).get("values")) + if isinstance(values, list): + return [float(value) for value in values] + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return None + return None + + +def _step_count(summary: Any, *, default: int) -> int: + coverage = summary.get("coverage") if isinstance(summary, dict) else {} + for key in ("executed", "successful"): + try: + value = int(coverage.get(key)) + except (AttributeError, TypeError, ValueError): + continue + if value >= 0: + return value + return default + + +def _required_text(value: Any, name: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError(f"SimulationStartRequest missing {name}") + return text + + +def _first_text(*values: Any) -> str: + for value in values: + text = str(value or "").strip() + if text: + return text + return "" + + +def _safe_path_part(value: Any) -> str: + text = "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in str(value)) + return text.strip("._") or "episode" diff --git a/env/prmeval/prmeval_config.rjob.yaml b/env/prmeval/prmeval_config.rjob.yaml index 27391f4a..b198e70e 100644 --- a/env/prmeval/prmeval_config.rjob.yaml +++ b/env/prmeval/prmeval_config.rjob.yaml @@ -10,7 +10,8 @@ environments: prmeval: sampling: dataset_name: rbm_1m_ood_test - paths: [/tmp/safactory-prmeval-tempfile/temp_sample.jsonl] + # adapter.py replaces paths with a temporary one-row JSONL per episode. + paths: [] base_frames: 8 progress_type: absolute_first_frame infer: diff --git a/env/prmeval/prmeval_config.yaml b/env/prmeval/prmeval_config.yaml index 17751579..d7eb9d26 100644 --- a/env/prmeval/prmeval_config.yaml +++ b/env/prmeval/prmeval_config.yaml @@ -14,7 +14,8 @@ environments: prmeval: sampling: dataset_name: rbm_1m_ood_test - paths: [/tmp/safactory-prmeval-tempfile/temp_sample.jsonl] + # adapter.py replaces paths with a temporary one-row JSONL per episode. + paths: [] base_frames: 8 progress_type: absolute_first_frame infer: diff --git a/env/prmeval/prmeval_start.rjob.yaml b/env/prmeval/prmeval_start.rjob.yaml index b64061d4..39af25ca 100644 --- a/env/prmeval/prmeval_start.rjob.yaml +++ b/env/prmeval/prmeval_start.rjob.yaml @@ -1,6 +1,6 @@ agent_name: prmeval container: - workdir: /tmp + workdir: /workspace runner_entrypoint: source: ./runner.py target: /tmp/safactory-prmeval/runner.py @@ -27,6 +27,11 @@ rjob: gpu: 0 memory_in_mb: 4096 mount_config: - - "gpfs://gpfs1/liuyicong/SAfactory/results:/app/results" - # Mount each GPFS source once; the startup command links the alternate frame path. - - "gpfs://gpfs1/liuyicong/SAfactory/env/prmeval/datasets:/tmp/safactory-prmeval/datasets" + # Replace CLUSTER_STORAGE with a location visible to the RJob cluster. + - "gpfs://CLUSTER_STORAGE/SAfactory/results:/app/results" + # Mount the dataset directory at the path stored in samples.jsonl. + - "gpfs://CLUSTER_STORAGE/SAfactory/env/prmeval/datasets:/tmp/safactory-prmeval/datasets" + embedded_files: + # runner.py is embedded automatically from runner_entrypoint.source. + - source: ./adapter.py + target: /tmp/safactory-prmeval/adapter.py diff --git a/env/prmeval/prmeval_start.yaml b/env/prmeval/prmeval_start.yaml index 1e4fcd9b..7fdce737 100644 --- a/env/prmeval/prmeval_start.yaml +++ b/env/prmeval/prmeval_start.yaml @@ -5,17 +5,25 @@ container: # Build from the repository root: docker build -t safactory-prmeval:0.2.0 env/prmeval workdir: /workspace runner_entrypoint: - source: ./ - target: /tmp/safactory-prmeval + source: ./runner.py + target: /tmp/safactory-prmeval/runner.py command: "python /tmp/safactory-prmeval/runner.py" mounts: + # The runner is mounted automatically from runner_entrypoint.source. + # Docker bind sources are resolved from the repository root by the + # launcher/live_smoke workflow. + - source: ./env/prmeval/adapter.py + target: /tmp/safactory-prmeval/adapter.py + mode: ro + - source: ./env/prmeval/datasets + target: /tmp/safactory-prmeval/datasets + mode: ro - source: ./results target: /workspace/Safactory/results mode: rw env: - # The Gateway accepts this placeholder; do not put a provider key here. PYTHONDONTWRITEBYTECODE: "1" NO_PROXY: host.docker.internal,localhost,127.0.0.1,::1 no_proxy: host.docker.internal,localhost,127.0.0.1,::1 diff --git a/env/prmeval/request.smoke.json b/env/prmeval/request.smoke.json new file mode 100644 index 00000000..dd4592af --- /dev/null +++ b/env/prmeval/request.smoke.json @@ -0,0 +1,34 @@ +{ + "job_id": "prmeval-contract", + "session_id": "prmeval-case-001", + "agent_name": "prmeval", + "agent_id": "case-001", + "group_id": "", + "gateway_base_url": "http://127.0.0.1:8000/v1/sessions", + "model": "contract-model", + "temperature": 0.0, + "max_steps": 10, + "storage_type": "sqlite", + "storage_config": {}, + "env_params": { + "dataset": { + "schema_version": "bench.record.v1", + "id": "prmeval-contract-001", + "task": "Move the object.", + "frames": [ + [[[0, 0, 0]]], + [[[127, 127, 127]]], + [[[255, 255, 255]]] + ], + "quality_label": "successful", + "partial_success": 1.0, + "target_progress": [0.0, 0.5, 1.0] + }, + "results_root": "/tmp/safactory-prmeval-results", + "prmeval": { + "sampling": {"base_frames": 3, "eval_types": ["progress"]}, + "infer": {"name": "openai_compatible"}, + "metrics": ["progress"] + } + } +} diff --git a/env/prmeval/results/.gitkeep b/env/prmeval/results/.gitkeep new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/env/prmeval/results/.gitkeep @@ -0,0 +1 @@ + diff --git a/env/prmeval/rule_evaluator.py b/env/prmeval/rule_evaluator.py index d1b9972d..30a45dc4 100644 --- a/env/prmeval/rule_evaluator.py +++ b/env/prmeval/rule_evaluator.py @@ -1,64 +1,60 @@ +"""PRMEval score adapter for SAfactory's optional evaluation stage. + +The native PRMEval run happens once in ``adapter.py``. This file only reads +the stored metrics and normalizes progress MSE to SAfactory's 0--10 reward; +it must never invoke PRMEval or the model again. +""" + from __future__ import annotations import math from numbers import Real from typing import Any -from evaluator.eval_types import ( - EvalRequest, - EvalResult, - EvalSpec, - EvalStatus, - Trajectory, -) - - async def evaluate_rule( *, - request: EvalRequest, - spec: EvalSpec, - trajectory: Trajectory, -) -> EvalResult: - """Normalize the stored native MSE without rerunning the benchmark.""" - metrics = _start_metrics(request) + request: Any, + spec: Any, + trajectory: Any, +) -> Any: + # Keep score helpers importable in lightweight local tooling. The full + # evaluator package is only needed when Launcher actually enables eval. + from evaluator.eval_types import EvalResult, EvalStatus + metrics = _start_metrics(request) try: - score = _float_or_none(metrics["mse"]) - if score is None: - return EvalResult.failed( - session_id=request.session_id, - eval_id=spec.eval_id, - method=spec.method.value, - reason="prmeval metrics did not contain a numeric score", - artifacts={"bench": "prmeval", "metrics": metrics}, - ) - reward = mse_to_reward(score) + raw_score = _numeric_progress_mse(metrics) + reward = mse_to_reward(raw_score) except ValueError as exc: return EvalResult.failed( session_id=request.session_id, eval_id=spec.eval_id, method=spec.method.value, reason=str(exc), + artifacts={"bench": "prmeval", "metrics": metrics}, ) + return EvalResult( session_id=request.session_id, eval_id=spec.eval_id, method=spec.method.value, status=EvalStatus.SUCCEEDED.value, normalized_score_10=reward, - raw_score=score, + raw_score=raw_score, reason="progress_mse_log_v1: clip(7 * ln(0.18 / mse) / ln(4.5), 0, 10)", artifacts={ - "mse": score, + "bench": "prmeval", + "mse": raw_score, "reward_10": reward, "successful_gateway_steps": len(trajectory.steps), - "bench_output_path": metrics.get("bench_output_path"), + "bench_output_path": metrics.get("bench_output_path") + or metrics.get("native_output_dir"), }, ) def mse_to_reward(mse: float) -> float: - """0.18 -> 0, 0.04 -> 7; approximately 0.021 -> 10 (capped).""" + """Map PRMEval progress MSE in [0, 1] to SAfactory's [0, 10] scale.""" if ( isinstance(mse, bool) or not isinstance(mse, Real) @@ -68,21 +64,25 @@ def mse_to_reward(mse: float) -> float: raise ValueError("progress MSE must be a finite number in [0, 1]") if mse == 0: return 10.0 - return round( - max(0.0, min(10.0, 7 * (math.log(0.18) - math.log(mse)) / math.log(4.5))), 6 - ) + return round(max(0.0, min(10.0, 7 * math.log(0.18 / mse) / math.log(4.5))), 6) -def _start_metrics(request: EvalRequest) -> dict[str, Any]: +def _start_metrics(request: Any) -> dict[str, Any]: start_result = getattr(request, "start_result", None) metrics = getattr(start_result, "metrics", None) return dict(metrics) if isinstance(metrics, dict) else {} -def _float_or_none(value: Any) -> float | None: +def _numeric_progress_mse(metrics: dict[str, Any]) -> float: + progress = metrics.get("progress") + if isinstance(progress, dict) and "mse" in progress: + value = progress["mse"] + else: + native = metrics.get("native_summary") + native_metrics = native.get("metrics") if isinstance(native, dict) else None + native_progress = native_metrics.get("progress") if isinstance(native_metrics, dict) else None + value = native_progress.get("mse") if isinstance(native_progress, dict) else metrics.get("mse") try: - if value is None: - return None return float(value) - except (TypeError, ValueError): - return None + except (TypeError, ValueError) as exc: + raise ValueError("prmeval metrics did not contain a numeric progress MSE") from exc diff --git a/env/prmeval/runner.py b/env/prmeval/runner.py index af109d90..5eceafaa 100644 --- a/env/prmeval/runner.py +++ b/env/prmeval/runner.py @@ -1,127 +1,78 @@ -"""PRMEval runner 模板：第一步，只读取 SAfactory request。 +#!/usr/bin/env python3 +"""Fixed SAfactory protocol shell for the PRMEval environment. -本文件可独立运行，只依赖 Python 标准库。当前 succeeded 仅表示请求读取 -成功；metrics.evaluation_executed=False 表示尚未执行 PRMEval 评测。 +Keep this file benchmark-agnostic. PRMEval-specific behavior lives in +``adapter.py`` so the same contract can be copied to another benchmark. """ from __future__ import annotations +import contextlib import json import os -import sys from pathlib import Path +import sys from typing import Any -from urllib.parse import urlsplit, urlunsplit - -from prmeval.core import EvalConfig, Evaluator # type: ignore - -RESULT_JSON_PREFIX = "SAFACTORY_RESULT_JSON " -RESULT_PATH_ENV = "SAFACTORY_RESULT_PATH" def read_request() -> dict[str, Any]: raw = sys.stdin.read().strip() or os.environ.get("SAFACTORY_START_REQUEST_JSON", "") if not raw: - raise RuntimeError("missing SimulationStartRequest JSON") - data = json.loads(raw) - if not isinstance(data, dict): - raise TypeError("SimulationStartRequest must be a JSON object") - return data - + raise ValueError("missing SimulationStartRequest JSON") + request = json.loads(raw) + if not isinstance(request, dict): + raise ValueError("SimulationStartRequest must be a JSON object") + return request -def test_read_request(): - import json - with open( - "./config.json", - "r", - encoding="utf-8", - ) as f: - data = json.load(f) - - return data - - -def post_process_result(result: dict[str, Any], session_id, job_id) -> dict[str, Any]: - """在写入 stdout 之前，可在此处对 result 做最后处理，例如： - - 补充 metrics 中的评测结果； - - 对 result 中的敏感信息做脱敏处理； - - 对 result 中的浮点数做精度截断。 - """ +def run_episode(request: dict[str, Any]) -> dict[str, Any]: + session_id = _required_text(request.get("session_id"), "session_id") + env_params = request.get("env_params") + if not isinstance(env_params, dict): + raise TypeError("env_params must be a JSON object") + task = env_params.get("dataset") + if not isinstance(task, dict): + raise TypeError("env_params.dataset must be a JSON object") - # 下一步：使用 dataset、prmeval_settings 和上述模型参数构建评测输入。 - # 完整的其他字段仍可从 request 获取，例如 request.get("metadata", {})。 - # 此处只返回输入摘要，避免把完整图像数组写进日志。 + session_url = _first_text( + os.environ.get("SAFACTORY_GATEWAY_SESSION_URL_CONTAINER"), + _gateway_session_url(request, session_id), + os.environ.get("OPENROUTER_BASE_URL"), + os.environ.get("OPENAI_BASE_URL"), + ) + if not session_url: + raise ValueError("cannot resolve Gateway session URL") + + # Import the environment hook lazily: protocol checks remain runnable on a + # workstation without PRMEval's optional dependencies. + with contextlib.redirect_stdout(sys.stderr): + from adapter import run_case + + metrics, step_count = run_case(request, task, session_url) + if not isinstance(metrics, dict): + raise TypeError("adapter.run_case metrics must be a JSON object") + if type(step_count) is not int or step_count < 0: + raise ValueError("adapter.run_case step_count must be a nonnegative integer") + json.dumps(metrics, ensure_ascii=False, allow_nan=False) return { "session_id": session_id, "status": "succeeded", "total_reward": 0.0, - "step_count": 0, + "step_count": step_count, "terminated": True, "truncated": False, "error_text": None, - "metrics": result["metrics"]["progress"], + "metrics": metrics, } -def post_process_config(config: dict[str, Any], request) -> dict[str, Any]: - # Use the session route so the gateway can associate inference with this episode. - - session_id = _required_text(request.get("session_id"), "session_id") - base_url = _resolve_base_url(request, session_id) - model = os.environ.get("SAFACTORY_ROUTE_MODEL") or request["model"] - - config["infer"]["model_id"] = model - config["infer"]["base_url"] = base_url - - return config - - -def run_episode(request: dict[str, Any]) -> dict[str, Any]: - """从完整 request 取出业务输入；后续在这里接入 PRMEval。""" - # 1. 调度信息：由 SAfactory 为本次任务生成。 - job_id = request["job_id"] - session_id = request["session_id"] - if not isinstance(session_id, str) or not session_id.strip(): - raise ValueError("session_id must be a non-empty string") - - # 2. 任务输入：dataset 是当前 JSONL 行，不是整个数据集。 - env_params = request.get("env_params") - if not isinstance(env_params, dict): - raise TypeError("env_params must be a JSON object") - dataset = env_params["dataset"] - output_path = Path("/tmp/safactory-prmeval-tempfile/temp_sample.jsonl") - output_path.parent.mkdir(parents=True, exist_ok=True) - - try: - with output_path.open("w", encoding="utf-8") as f: - f.write(json.dumps(dataset, ensure_ascii=False) + "\n") - # 3. PRMEval 执行 - prmeval_config = env_params.get("prmeval", {}) - if not isinstance(dataset, dict) or not isinstance(prmeval_config, dict): - raise TypeError( - "env_params.dataset and env_params.prmeval must be JSON objects" - ) - prmeval_config = post_process_config(prmeval_config, request) - prmeval_config = EvalConfig.model_validate(prmeval_config) - summary = Evaluator(prmeval_config).run() - result = post_process_result(summary, session_id, job_id) - - return result - - finally: - output_path.unlink(missing_ok=True) - - def main() -> int: session_id = os.environ.get("SAFACTORY_SESSION_ID", "") try: request = read_request() - # request = test_read_request() - # print(request) - session_id = request.get("session_id", session_id) + session_id = str(request.get("session_id") or session_id) result = run_episode(request) - except Exception as exc: + except Exception as exc: # Runtime failures are represented in the result JSON. result = { "session_id": session_id, "status": "failed", @@ -130,43 +81,42 @@ def main() -> int: "terminated": True, "truncated": False, "error_text": str(exc), - "metrics": {"stage": "request_received", "evaluation_executed": False}, + "metrics": {}, } - # stdout 专用于 SAfactory 结果协议；调试信息应写到 stderr。 _write_result(result) return 0 def _write_result(result: dict[str, Any]) -> None: - _persist_result_artifact(result) - print(RESULT_JSON_PREFIX + json.dumps(result, ensure_ascii=False), flush=True) - + artifact = str(os.environ.get("SAFACTORY_RESULT_PATH") or "").strip() + if artifact: + try: + path = Path(artifact) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(result, ensure_ascii=False, allow_nan=False) + "\n", + encoding="utf-8", + ) + except Exception as exc: + print(f"SAFACTORY_RUNNER_DIAGNOSTIC result_artifact_write_failed: {exc}", file=sys.stderr) + print(json.dumps(result, ensure_ascii=False, allow_nan=False), flush=True) -def _persist_result_artifact(result: dict[str, Any]) -> None: - raw_path = str(os.environ.get(RESULT_PATH_ENV) or "").strip() - if not raw_path: - return - try: - path = Path(raw_path) - path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = path.with_name(path.name + ".tmp") - tmp_path.write_text( - json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" - ) - tmp_path.replace(path) - except Exception as exc: - print( - f"SAFACTORY_RUNNER_DIAGNOSTIC result_artifact_write_failed: {exc}", - file=sys.stderr, - flush=True, - ) +def _gateway_session_url(request: dict[str, Any], session_id: str) -> str: + base = str(request.get("gateway_base_url") or "").rstrip("/") + if not base: + return "" + # request_env normally performs this rewrite. Keep the fallback useful + # when the runner is invoked by hand or by a local contract test. + from urllib.parse import urlsplit, urlunsplit -def _required_text(value: Any, name: str) -> str: - text = str(value or "").strip() - if not text: - raise RuntimeError(f"SimulationStartRequest missing {name}") - return text + parts = urlsplit(base) + if parts.hostname in {"127.0.0.1", "localhost", "::1"}: + netloc = "host.docker.internal" + if parts.port is not None: + netloc = f"{netloc}:{parts.port}" + base = urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)).rstrip("/") + return f"{base}/{session_id}" def _first_text(*values: Any) -> str: @@ -177,58 +127,12 @@ def _first_text(*values: Any) -> str: return "" -def _containerize_local_gateway_url(url: str) -> str: - try: - parts = urlsplit(str(url)) - except Exception: - return str(url) - if parts.hostname not in {"127.0.0.1", "localhost", "::1"}: - return str(url) - netloc = "host.docker.internal" - if parts.port is not None: - netloc = f"{netloc}:{parts.port}" - return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)) - - -def _gateway_session_url(request: dict[str, Any], session_id: str) -> str: - base = str(request.get("gateway_base_url") or "").rstrip("/") - if not base: - return "" - return _containerize_local_gateway_url(f"{base}/{session_id}") - - -def _resolve_base_url(request: dict[str, Any], session_id: str) -> str: - base_url = _first_text( - os.environ.get("SAFACTORY_GATEWAY_SESSION_URL_CONTAINER"), - _gateway_session_url(request, session_id), - os.environ.get("OPENROUTER_BASE_URL"), - os.environ.get("OPENAI_BASE_URL"), - ) - if not base_url: - raise RuntimeError( - "geo3k runner could not resolve an OpenAI-compatible base URL" - ) - return base_url +def _required_text(value: Any, name: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError(f"SimulationStartRequest missing {name}") + return text if __name__ == "__main__": - try: - raise SystemExit(main()) - except Exception as exc: - print( - json.dumps( - { - "session_id": os.environ.get("SAFACTORY_SESSION_ID", ""), - "status": "failed", - "total_reward": 0.0, - "step_count": 0, - "terminated": True, - "truncated": False, - "error_text": str(exc), - "metrics": {}, - }, - ensure_ascii=False, - ), - flush=True, - ) - raise SystemExit(0) + raise SystemExit(main()) diff --git a/skills/safactory-workflows/SKILL.md b/skills/safactory-workflows/SKILL.md index 7a97c3bd..ba255c17 100644 --- a/skills/safactory-workflows/SKILL.md +++ b/skills/safactory-workflows/SKILL.md @@ -1,131 +1,64 @@ --- name: safactory-workflows -description: Use this skill to onboard a benchmark or custom environment into SAfactory, run Docker or RJob evaluation, or prepare GRPO/RL training. For benchmark onboarding it defines the required adapter files, intake fields, single-case contract, and Docker/RJob-specific checks. +description: Integrate a benchmark or custom environment into SAfactory using fixed adapter templates and local contract tests, optionally run Docker/RJob evaluation, or prepare GRPO/RL training. Use for SAfactory onboarding and runtime workflows. --- # SAfactory Workflows -Use this skill for three related workflows: +For environment onboarding, read [references/environment-integration.md](references/environment-integration.md) and `docs/guides/custom-environment.md` (or its Chinese translation). The guide defines the runtime contract; the [assets/environment/](assets/environment/) templates implement it. Use `env/geo3k/` for examples of environment-specific behavior, not as boilerplate to copy wholesale. -1. onboard a benchmark or custom environment; -2. run Docker or RJob evaluation; -3. prepare or start GRPO/RL training after evaluation works. +For evaluation requests read [references/docker-evaluation.md](references/docker-evaluation.md). For GRPO/RL requests read [references/grpo-training.md](references/grpo-training.md). Read `docs/internal/rjob-mode.md` or its Chinese translation only when working on RJob deployment. -The repository documentation is the source of truth. Do not invent a second runtime contract. For a new benchmark, read [references/environment-integration.md](references/environment-integration.md) and `docs/guides/custom-environment_CN.md` (or the English guide for an English request). For RJob-specific details, also read `docs/internal/rjob-mode_CN.md` or `docs/internal/rjob-mode.md`. +## Scope and intake -## Benchmark onboarding: intake gate +Infer available values from the request and benchmark source before asking: -Before editing, collect these fields from the user or infer them from the benchmark source/README: +- environment name, source/check-out path, dataset row schema, and 1–2 representative cases; +- native single-case command and native output location, when wrapping a benchmark; +- target deployment mode (`docker` or `rjob`) and image, when known; +- whether evaluation is requested; **default to integration only**. -- `mode`: exactly `docker` or `rjob`; -- environment name, normally lowercase with underscores; -- benchmark source/check-out path or repository; -- dataset path and the shape of one dataset row; -- one or two test-case IDs/rows; -- the benchmark's native single-case command (or the README section that defines it); -- Docker image name, if already available; -- where the native benchmark writes its result/output file; -- where the native score/reward is located and its scale/meaning. +Only evaluation or training requires the native score field, scale, and pass condition. Do not block integration on missing rewards, evaluator code, a running Gateway, or internal cluster access. If mode is unknown, implement and test the shared adapter first; clarify mode before generating deployment configs. Ask about an unknown native command or output format before implementing the dependent hook, while continuing the protocol scaffold and tests. -The user must have (or provide enough information to run) **1–2 test cases**. The first milestone is a single-case pipeline whose output file and reward can be inspected. If the native case command, output path, score field, or selected mode is unknown, ask for that specific missing value before implementing. +The adapter runs one dataset row through the existing native harness. It does not reimplement the benchmark's case-solving/scoring logic or repair image internals unless that work is requested. Preserve native scores in metrics when available; their presence does not enable evaluation. -### Scope boundary +## Template-based implementation -The skill creates and validates the SAfactory adapter only. It does not rewrite the benchmark's case-solving/evaluation logic inside an existing Docker image, add a new benchmark harness, or repair a broken native single-case command unless the user explicitly expands the scope. Assume the image/harness can already execute one case; the adapter passes the current case and model calls in, then translates the native result out. +1. Scaffold new files with `scripts/scaffold_environment.py` as documented in the integration reference. For an existing directory, adapt the relevant templates without overwriting user files. +2. Keep the protocol shell in `runner.py` fixed. Fill `adapter.py:run_case` with row mapping, model/harness configuration, native execution, and result collection. The generated greeting example only verifies the scaffold; replace it for a real integration. +3. Fill the selected-mode task/start YAML templates with the image, dataset, workdir, mounts, and dependencies. `agent_name` must equal `env_name`; one row is one episode. +4. Create `rule_evaluator.py` only when evaluation is requested. Fill its `score_metrics` hook with the agreed normalization; keep the evaluator interface and failure handling fixed. Do not rerun cases in the evaluator or register its path in YAML. +5. Python is the default template language. If the native runtime needs Node/shell, retain a thin Python wrapper when possible. If another runner language is required, port the same protocol shell and document why; keep native logic separate and cover the same contract tests. -## Adapter files and responsibilities +Change shared shell behavior only for an actual runtime-contract requirement, and update the template and its tests together. New files should come from templates, not a fresh implementation of the protocol. -For `mybench`, the onboarding output is under `env/mybench/`: +## Standard reference: `env/prmeval` -| File | Required | Responsibility | -|---|---:|---| -| `runner.py`, `runner.mjs`, or `runner.sh` | yes | Read `SimulationStartRequest` from stdin or `SAFACTORY_START_REQUEST_JSON`; read the current row from `env_params.dataset`; call the model through the current Gateway session URL; invoke the already-available native single-case command; collect the native output; print one `SimulationStartResult` JSON. | -| `mybench_config.yaml` | Docker mode | Define task rows and runtime metadata: `env_name`, `env_image`, `dataset`, `env_num`, and `env_params` (plus dataset loading options when needed). One dataset row must represent one episode/case. | -| `mybench_start.yaml` | Docker mode | Define how the runtime starts: runner entrypoint, working directory, environment variables, Docker settings, and mounts. `agent_name` must equal `env_name`. | -| `rule_evaluator.py` | recommended for scored benchmarks | Read runtime `metrics` and available trajectory data and convert the native score/pass result to a SAfactory reward in the `0–10` range. SAfactory auto-discovers `env/mybench/rule_evaluator.py`; do not register its path in YAML. | -| `Dockerfile` | optional | Build a dedicated image only when an existing image is unavailable or needs adapter dependencies. Do not move benchmark case logic into the adapter. | +When an existing benchmark is supplied, compare its integration against +`env/prmeval/` before inventing a new layout. The reference keeps the fixed +`runner.py` protocol shell separate from `adapter.py`, uses one dataset row per +episode, and keeps the optional score mapping in `rule_evaluator.py`. -The runner/result contract, config fields, and evaluator interface must follow `docs/guides/custom-environment_CN.md`. Keep stdout limited to the machine-readable result; write diagnostics to stderr. Put native case ID, score, pass/fail, reason, and output path in `metrics` so evaluation does not rerun the case. +The Docker pair (`*_config.yaml`, `*_start.yaml`) is the base contract. The +RJob pair (`*_config.rjob.yaml`, `*_start.rjob.yaml`) repeats task parameters +but adds only cluster image, resources, embedded files, and cluster-accessible +mounts. `env_params` is delivered in the request JSON; static `container.env` +or `rjob.env` values are not a replacement for it. -## Runtime modes +## Validation and completion -The user must select one mode in the intake prompt. Implement that mode first; do not silently substitute the other mode. +Use the staged checks in the integration reference: -### Docker mode +1. **Local contract check (default for both target modes):** use `scripts/contract_smoke.py` or equivalent environment-specific fixtures to exercise request input, session URL routing, native output mapping, exact result JSON, and controlled failures. The helper owns its mock endpoint and needs no Gateway, Docker daemon, model credentials, or RJob SDK. Native dependencies still require local availability or explicit test fixtures. +2. **Live deployment check:** when requested or the needed runtime is available, run 1–2 cases with one worker. `scripts/live_smoke.py` starts Gateway, waits for readiness, runs Launcher, and cleans up its processes. Do not require a user to keep a separate terminal open. If an existing Gateway is used, verify it and invoke Launcher directly. +3. **Evaluation check (opt-in):** validate native score conversion and final reward only when evaluation/training is in scope. Pass `--enable-evaluation` explicitly; omit it for integration-only runs. `total_reward: 0.0` satisfies the ungraded runtime result contract and is not an evaluation outcome. -Create/use: +Report the adapter/config changes, the cases and fixtures used, which validation level passed, and remaining deployment prerequisites. Local tests can complete adapter validation without an internal cluster; do not claim they validate image pulls, mounts, real Gateway persistence, or RJob scheduling. If a requested live check is blocked, complete independent local checks and report the specific blocker and next command. -```text -env/mybench/runner.py # or runner.mjs / runner.sh -env/mybench/mybench_config.yaml -env/mybench/mybench_start.yaml -env/mybench/rule_evaluator.py # when scored evaluation is required -``` +## Repository rules -The image is local, `container.mounts` are Docker bind mounts, and the smoke test uses `--mode docker`. The runner should use `SAFACTORY_GATEWAY_SESSION_URL_CONTAINER` rather than hardcoding `localhost`. - -### RJob mode - -Create/use both mode-specific files in addition to the runner/evaluator: - -```text -env/mybench/runner.py # or runner.mjs / runner.sh -env/mybench/mybench_config.rjob.yaml -env/mybench/mybench_start.rjob.yaml -env/mybench/rule_evaluator.py -``` - -`mybench_config.rjob.yaml` is the RJob task config (normally derived from the Docker task config). `mybench_start.rjob.yaml` contains the RJob runtime settings under `rjob:`. Use cluster-accessible images and storage; use `rjob.mount_config`/`rjob.mount` rather than local Docker bind mounts; list every local runner dependency in `rjob.embedded_files`. The Gateway URL must be reachable from the cluster and must not be `127.0.0.1` or `localhost`. The smoke test must use `--mode rjob` and the appropriate `--rjob-config`. - -The RJob files change deployment mechanics, not the single-case input/output contract. Keep the same dataset-row semantics, Gateway session handling, result JSON, and evaluator across both modes unless the runtime genuinely requires a documented difference. - -## Onboarding workflow - -1. Inspect the repository layout, the benchmark README/source, `env/geo3k/`, and the custom-environment guide. -2. Confirm the selected mode and validate the native benchmark command on 1–2 cases. Record the actual output file and native score/reward before writing the evaluator. -3. Implement only the adapter boundary: request parsing, dataset-row mapping, Gateway call, native command invocation, result-file discovery, and `SimulationStartResult` serialization. -4. Add the task config and selected-mode start config. Keep `env_name`/`agent_name` identical and make all paths/mounts explicit. -5. Add `rule_evaluator.py` when the benchmark has deterministic scoring. Convert the native result to `0–10`; do not rerun the benchmark case in the evaluator. -6. Run the smallest smoke test for the selected mode with one worker and 1–2 cases. Verify the runtime result JSON, native output file, Gateway trajectory, and final reward. -7. If the user requests both modes, repeat the config/start-config and smoke-test checks for the second mode; do not assume a Docker mount works in RJob. - -Typical commands are: - -```bash -# Docker -python launcher.py \ - --mode docker \ - --agent-config env/mybench/mybench_config.yaml \ - --agent-start-config env/mybench/mybench_start.yaml \ - --gateway-base-url http://127.0.0.1:8000/v1/sessions \ - --llm-model YOUR_ROUTE_KEY \ - --enable-evaluation \ - --db-path sqlite://mybench_smoke.db \ - --job-id mybench-docker-smoke \ - --pool-size 1 --max-workers 1 --max-steps 10 - -# RJob -python launcher.py \ - --mode rjob \ - --rjob-config config.yaml \ - --agent-config env/mybench/mybench_config.rjob.yaml \ - --agent-start-config env/mybench/mybench_start.rjob.yaml \ - --gateway-base-url http://GATEWAY_HOST:8000/v1/sessions \ - --llm-model YOUR_ROUTE_KEY \ - --enable-evaluation \ - --db-path sqlite://mybench_smoke.db \ - --job-id mybench-rjob-smoke \ - --pool-size 1 --max-workers 1 --max-steps 10 -``` - -Replace placeholders with real values. `--llm-model` must match a Gateway `llm_routes` key; never commit private endpoints or credentials. - -## Other workflows and repository rules - -- Read only the reference matching the request: `references/environment-integration.md`, `references/docker-evaluation.md`, or `references/grpo-training.md`. -- Treat `env/geo3k/` as the complete reference implementation, not as a hardcoded target. -- Keep Launcher, Gateway, and Buffer Server on the same storage backend or SQLite URI. -- Do not overwrite `gateway/config.local.yaml` without preserving user edits. -- Do not read `docs/internal/` except for RJob/Sandbox/internal deployment questions; RJob onboarding is the explicit exception above. -- Before recommending RL, establish that the selected-mode single-case evaluation and reward path work. -- For edits, inspect changed paths and configs. If Docker, RJob SDK, image, data, or credentials are unavailable, report the exact blocker and leave the next command rather than claiming the smoke test passed. +- Preserve the user's selected deployment mode. Local contract testing does not substitute Docker deployment for RJob deployment. +- RJob needs the same runner, plus `_config.rjob.yaml` and `_start.rjob.yaml`. The evaluator remains optional. Use cluster-accessible images/storage and `rjob.mount_config`/`rjob.mount`; include all runner dependencies in `rjob.embedded_files`. Do not use loopback Gateway addresses for live cluster runs. +- Keep Launcher, Gateway, and Buffer Server on the same storage backend/SQLite URI. +- Preserve local Gateway configuration and never commit private endpoints or credentials. +- Establish a working reward path before starting RL; this requirement does not apply to integration-only work. diff --git a/skills/safactory-workflows/assets/environment/adapter.py b/skills/safactory-workflows/assets/environment/adapter.py new file mode 100644 index 00000000..236a14c2 --- /dev/null +++ b/skills/safactory-workflows/assets/environment/adapter.py @@ -0,0 +1,27 @@ +"""Environment-specific hook. Replace the guide's greeting example with one native case. + +This example is runnable for scaffold verification; it is not a benchmark integration. +Use only `task` (the current dataset row), never loop over the dataset here. +For a native subprocess, capture stdout or redirect it to stderr explicitly. +Pass session_url and request['model'] to the harness's model client configuration. +Return JSON metrics (including native output paths when available) and a step count. +""" +import json +from urllib.request import Request, urlopen + + +def run_case(request, task, session_url): + prompt = task.get("prompt") or task.get("question") or "Say hello from Safactory." + payload = { + "model": request["model"], + "messages": [{"role": "user", "content": prompt}], + "temperature": request.get("temperature", 0.3), + } + call = Request( + f"{session_url.rstrip('/')}/chat/completions", + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + ) + with urlopen(call, timeout=300) as response: + body = json.load(response) + return {"answer": body["choices"][0]["message"].get("content", "")}, 1 diff --git a/skills/safactory-workflows/assets/environment/config.rjob.yaml.tmpl b/skills/safactory-workflows/assets/environment/config.rjob.yaml.tmpl new file mode 100644 index 00000000..6db954ad --- /dev/null +++ b/skills/safactory-workflows/assets/environment/config.rjob.yaml.tmpl @@ -0,0 +1,12 @@ +environments: + - env_name: __ENV_NAME__ + # Replace with an image tag that the RJob cluster can pull. + env_image: __ENV_NAME__-image:latest + env_num: 1 + dataset: ./datasets/smoke.jsonl + dataset_load_mode: eager + # env_params is serialized into SimulationStartRequest and passed to the + # runner for every dataset row. Keep runtime configuration here. + env_params: + task_family: __ENV_NAME__ + output_root: /app/results/__ENV_NAME__ diff --git a/skills/safactory-workflows/assets/environment/config.yaml.tmpl b/skills/safactory-workflows/assets/environment/config.yaml.tmpl new file mode 100644 index 00000000..fb88f61f --- /dev/null +++ b/skills/safactory-workflows/assets/environment/config.yaml.tmpl @@ -0,0 +1,11 @@ +environments: + - env_name: __ENV_NAME__ + env_image: __ENV_NAME__-image:latest + env_num: 1 + dataset: ./datasets/smoke.jsonl + dataset_load_mode: eager + # env_params is serialized into SimulationStartRequest and passed to the + # runner for every dataset row. Keep runtime configuration here. + env_params: + task_family: __ENV_NAME__ + output_root: /workspace/Safactory/results/__ENV_NAME__ diff --git a/skills/safactory-workflows/assets/environment/request.smoke.json.tmpl b/skills/safactory-workflows/assets/environment/request.smoke.json.tmpl new file mode 100644 index 00000000..edd73de9 --- /dev/null +++ b/skills/safactory-workflows/assets/environment/request.smoke.json.tmpl @@ -0,0 +1,14 @@ +{ + "job_id": "__ENV_NAME__-contract", + "session_id": "__ENV_NAME__-case-001", + "agent_name": "__ENV_NAME__", + "agent_id": "case-001", + "group_id": "", + "gateway_base_url": "http://unused.invalid/v1/sessions", + "model": "contract-model", + "temperature": 0.3, + "max_steps": 10, + "storage_type": "sqlite", + "storage_config": {}, + "env_params": {"dataset": {"task_id": "hello-001", "prompt": "Write one short greeting."}} +} diff --git a/skills/safactory-workflows/assets/environment/rule_evaluator.py b/skills/safactory-workflows/assets/environment/rule_evaluator.py new file mode 100644 index 00000000..10046ada --- /dev/null +++ b/skills/safactory-workflows/assets/environment/rule_evaluator.py @@ -0,0 +1,35 @@ +"""Optional evaluator shell; fill only score_metrics after agreeing on scoring. + +Discovered by SAfactory only when --enable-evaluation is enabled. +No benchmark execution or runner/image dependencies belong in this file. +""" +import math + + +def score_metrics(metrics, dataset, trajectory): + """Return (raw_score, score_0_to_10, reason) using the native scoring contract. + + This is the environment-specific hook. Reject missing/invalid native scores; + never treat successful adapter execution as a perfect benchmark score. + """ + raise NotImplementedError("fill score_metrics using the agreed native score and scale") + + +def evaluate_rule(*, request, spec, trajectory): + metrics = getattr(request.start_result, "metrics", {}) or {} + try: + raw, score, reason = score_metrics(metrics, request.env_params.get("dataset", {}), trajectory) + raw, score = float(raw), float(score) + if not math.isfinite(raw) or not math.isfinite(score) or not 0 <= score <= 10: + raise ValueError("scores must be finite and normalized score must be in 0–10") + return { + "session_id": request.session_id, "eval_id": spec.eval_id, + "status": "succeeded", "raw_score": raw, + "normalized_score_10": score, "reason": str(reason), "artifacts": {"metrics": metrics}, + } + except Exception as exc: + return { + "session_id": request.session_id, "eval_id": spec.eval_id, + "status": "failed", "normalized_score_10": 0.0, + "reason": str(exc), "error_text": str(exc), "artifacts": {"metrics": metrics}, + } diff --git a/skills/safactory-workflows/assets/environment/runner.py b/skills/safactory-workflows/assets/environment/runner.py new file mode 100644 index 00000000..126eed0d --- /dev/null +++ b/skills/safactory-workflows/assets/environment/runner.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Fixed SAfactory protocol shell, based on docs/guides/custom-environment.md. + +Put environment-specific behavior in adapter.py; keep this shell unchanged. +""" +from __future__ import annotations + +import contextlib +import json +import os +from pathlib import Path +import sys +from urllib.parse import urlsplit, urlunsplit + + +def read_request(): + raw = sys.stdin.read().strip() or os.environ.get("SAFACTORY_START_REQUEST_JSON", "") + if not raw: + raise ValueError("missing SimulationStartRequest JSON") + request = json.loads(raw) + if not isinstance(request, dict): + raise ValueError("SimulationStartRequest must be a JSON object") + return request + + +def main(): + result = { + "session_id": os.environ.get("SAFACTORY_SESSION_ID", ""), + "status": "failed", "total_reward": 0.0, "step_count": 0, + "terminated": True, "truncated": False, "error_text": None, "metrics": {}, + } + try: + request = read_request() + result["session_id"] = str(request["session_id"]) + session_url = os.environ.get("SAFACTORY_GATEWAY_SESSION_URL_CONTAINER") or _gateway_session_url( + request, result["session_id"] + ) + if not session_url: + raise ValueError("cannot resolve Gateway session URL") + task = (request.get("env_params") or {}).get("dataset") or {} + with contextlib.redirect_stdout(sys.stderr): + from adapter import run_case + + metrics, step_count = run_case(request, task, session_url) + if not isinstance(metrics, dict): + raise ValueError("run_case metrics must be a JSON object") + if type(step_count) is not int or step_count < 0: + raise ValueError("run_case step_count must be a nonnegative integer") + json.dumps(metrics, allow_nan=False) + result.update(status="succeeded", metrics=metrics, step_count=step_count) + except Exception as exc: + result["error_text"] = str(exc) + + artifact = os.environ.get("SAFACTORY_RESULT_PATH") + if artifact: + try: + path = Path(artifact) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(result, ensure_ascii=False, allow_nan=False), encoding="utf-8") + except Exception as exc: + result.update(status="failed", error_text=f"cannot write result artifact: {exc}") + print(json.dumps(result, ensure_ascii=False, allow_nan=False), flush=True) + return 0 # Controlled failures are result JSON, not process failures. + + +def _gateway_session_url(request, session_id): + base = str(request.get("gateway_base_url") or "").rstrip("/") + if not base: + return "" + parts = urlsplit(base) + if parts.hostname in {"127.0.0.1", "localhost", "::1"}: + netloc = "host.docker.internal" + if parts.port is not None: + netloc = f"{netloc}:{parts.port}" + base = urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)).rstrip("/") + return f"{base}/{session_id}" + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/safactory-workflows/assets/environment/start.docker.yaml.tmpl b/skills/safactory-workflows/assets/environment/start.docker.yaml.tmpl new file mode 100644 index 00000000..bdad79ba --- /dev/null +++ b/skills/safactory-workflows/assets/environment/start.docker.yaml.tmpl @@ -0,0 +1,28 @@ +agent_name: __ENV_NAME__ +container: + workdir: /workspace + runner_entrypoint: + source: ./runner.py + target: /tmp/safactory-__ENV_NAME__/runner.py + command: "python /tmp/safactory-__ENV_NAME__/runner.py" + mounts: + # Docker bind sources are resolved from the repository root when the + # launcher/live_smoke helper is run there. + - source: __ENV_ROOT__/__ENV_NAME__/adapter.py + target: /tmp/safactory-__ENV_NAME__/adapter.py + mode: ro + # Mount native benchmark assets read by the current case. If the dataset + # is remote, replace this with the image's own path or remove the mount. + - source: __ENV_ROOT__/__ENV_NAME__/datasets + target: /workspace/Safactory/datasets/__ENV_NAME__ + mode: ro + - source: ./results + target: /workspace/Safactory/results + mode: rw + env: + PYTHONDONTWRITEBYTECODE: "1" + NO_PROXY: host.docker.internal,localhost,127.0.0.1,::1 + no_proxy: host.docker.internal,localhost,127.0.0.1,::1 + extra_args: + - --add-host=host.docker.internal:host-gateway + idle_command: "tail -f /dev/null" diff --git a/skills/safactory-workflows/assets/environment/start.rjob.yaml.tmpl b/skills/safactory-workflows/assets/environment/start.rjob.yaml.tmpl new file mode 100644 index 00000000..5ebb15ea --- /dev/null +++ b/skills/safactory-workflows/assets/environment/start.rjob.yaml.tmpl @@ -0,0 +1,27 @@ +agent_name: __ENV_NAME__ +container: + workdir: /workspace + runner_entrypoint: + source: ./runner.py + target: /tmp/safactory-__ENV_NAME__/runner.py + command: "python /tmp/safactory-__ENV_NAME__/runner.py" + env: + PYTHONDONTWRITEBYTECODE: "1" +rjob: + name_prefix: __ENV_NAME__ + image_pull_policy: IfNotPresent + no_packaging: true + cleanup_on_finish: true + keep_failed_jobs: true + resources: + cpu: 1 + gpu: 0 + memory_in_mb: 1024 + embedded_files: + - source: ./adapter.py + target: /tmp/safactory-__ENV_NAME__/adapter.py + # Add cluster-accessible result storage via mount_config/mount for live runs. + mount_config: + # Replace CLUSTER_STORAGE with a path visible to the RJob cluster. + - "gpfs://CLUSTER_STORAGE/SAfactory/results:/app/results" + - "gpfs://CLUSTER_STORAGE/SAfactory/env/__ENV_NAME__/datasets:/tmp/safactory-__ENV_NAME__/datasets" diff --git a/skills/safactory-workflows/references/docker-evaluation.md b/skills/safactory-workflows/references/docker-evaluation.md index f0e8e067..9d5c2c9f 100644 --- a/skills/safactory-workflows/references/docker-evaluation.md +++ b/skills/safactory-workflows/references/docker-evaluation.md @@ -1,6 +1,6 @@ # Docker Evaluation Workflow -Use this reference when the user asks to run evaluation for an SAfactory environment in Docker mode. +Use this reference when the user asks to run evaluation for an SAfactory environment in Docker mode. Integration-only requests use `environment-integration.md` and omit evaluation. Canonical docs: @@ -28,7 +28,7 @@ Check these before running: - Environment config exists: `env//_config.yaml`. - Start config exists: `env//_start.yaml`. - Required image, dataset, and runner files are available. -- Gateway is running at a session root such as `http://127.0.0.1:8000/v1/sessions`. +- A Gateway config is ready for the helper to start, or an existing Gateway is reachable at its session root. - The requested model route key exists in Gateway `llm_routes`. - Gateway storage and Launcher `--db-path` point to the same backend when telemetry/results are expected in the same DB. - `rule_evaluator.py` exists if the user expects `--enable-evaluation` reward output. @@ -40,15 +40,16 @@ Do not add private `base_url` or `api_key` values to committed files. Use placeh Use the root README as the command source. For a generic environment: ```bash -python launcher.py \ +python skills/safactory-workflows/scripts/live_smoke.py \ + --gateway-config gateway/config.local.yaml -- \ --mode docker \ --agent-config env//_config.yaml \ --agent-start-config env//_start.yaml \ --gateway-base-url http://127.0.0.1:8000/v1/sessions \ --llm-model \ --enable-evaluation \ - --db-path sqlite:///_eval.db \ - --job-id -docker-smoke + --job-id -docker-smoke \ + --pool-size 1 --max-workers 1 ``` Use a small dataset, small sample count, or smoke-test config when available. Avoid launching a full benchmark unless the user explicitly asks for it. @@ -70,13 +71,7 @@ llm_routes: 4. Ensure `--llm-model` equals the selected route key. -Start Gateway with: - -```bash -python -m gateway --config gateway/config.local.yaml -``` - -If Gateway is already running, verify the session root and route key instead of starting a second server on the same port. +The helper command above starts Gateway, waits for `/readyz`, runs Launcher, and cleans up its own processes on completion/failure/timeout. It uses the configured SQLite URI unless an identical `--db-path` is supplied. No separate terminal is required. If Gateway is already running, verify readiness, the session root, route key, and storage, then invoke `launcher.py` directly. The helper refuses to take over an occupied port. ## Result Checks diff --git a/skills/safactory-workflows/references/environment-integration.md b/skills/safactory-workflows/references/environment-integration.md index 7d74e1a2..c88b0c3b 100644 --- a/skills/safactory-workflows/references/environment-integration.md +++ b/skills/safactory-workflows/references/environment-integration.md @@ -1,150 +1,102 @@ -# Benchmark / Environment Integration Workflow - -Use this reference when the user asks to add a benchmark, custom environment, or new task suite to SAfactory. The detailed runtime contract is defined by `docs/guides/custom-environment.md` and `docs/guides/custom-environment_CN.md`; this file turns it into an agent workflow. - -## Intake gate - -Do not start implementation until the request identifies: - -- mode: `docker` or `rjob`; -- environment name; -- benchmark source/check-out path; -- dataset path and one-row shape; -- 1–2 smoke-test rows/case IDs; -- native single-case command or the benchmark README section that defines it; -- existing Docker image, if any; -- native result/output file location; -- native score/reward location and scale. - -The user-facing copy/paste prompt is in the root `README_CN.md` / `README.md` under “Benchmark 接入 / Benchmark Onboarding Prompt”. If information is missing, ask only for the missing field. The first deliverable is one working single-case pipeline, not a full benchmark batch. - -## Scope boundary - -The adapter owns only the SAfactory boundary: - -- parse `SimulationStartRequest`; -- map `env_params.dataset` to one native case; -- call the model through the session-aware Gateway; -- invoke the native command that already exists in the image/harness; -- find/read the native result; -- emit the SAfactory result JSON and `metrics`. - -Do not reimplement benchmark case-solving logic, scoring logic already provided by the benchmark, or image internals as part of onboarding. If the native single-case command does not work independently, report that as a blocker or request explicit scope expansion. - -## Files to create or adapt - -For environment `mybench`, create the selected-mode files below under `env/mybench/`. - -| File | Required | What it does | -|---|---:|---| -| `runner.py`, `runner.mjs`, or `runner.sh` | yes | Reads the request, gets `env_params.dataset`, calls the current Gateway session, runs one native case, reads the native result, and prints one `SimulationStartResult` JSON. | -| `mybench_config.yaml` | Docker mode | Defines `env_name`, `env_image`, `dataset`, `env_num`, `env_params`, and dataset loading options. Each dataset row is one episode. | -| `mybench_start.yaml` | Docker mode | Defines `container.runner_entrypoint`, workdir, env vars, Docker mounts/args, and `agent_name`; `agent_name` must equal `env_name`. | -| `mybench_config.rjob.yaml` | RJob mode | RJob variant of the task config. Keep dataset-row and `env_params` semantics aligned with the Docker config. | -| `mybench_start.rjob.yaml` | RJob mode | RJob variant of the start config, including `rjob:` resources, cleanup, embedded files, and cluster-accessible mounts. | -| `rule_evaluator.py` | scored benchmark | Converts runtime `metrics` and trajectory information into a `0–10` reward. Auto-discovered at `env/mybench/rule_evaluator.py`; do not add an evaluator path to YAML. | -| `Dockerfile` | optional | Builds a dedicated image when no suitable image exists. It is not a place to rewrite the benchmark's native case logic. | - -RJob mode still needs the runner and evaluator. The two `.rjob.yaml` files are additional mode-specific deployment configs, not replacements for the runtime contract. - -## Runner contract - -The runner must: - -1. read JSON from stdin or `SAFACTORY_START_REQUEST_JSON`; -2. use `request.session_id` and `SAFACTORY_GATEWAY_SESSION_URL_CONTAINER` (or the request's session URL) for model calls; -3. read the current row from `request.env_params.dataset`; -4. pass that row to the native single-case command without looping over the dataset; -5. capture native score/pass/failure/output path in `metrics`; -6. print exactly one result object like: - -```json -{ - "session_id": "same-session-id", - "status": "succeeded", - "total_reward": 0.0, - "step_count": 1, - "terminated": true, - "truncated": false, - "error_text": null, - "metrics": { - "bench_case_id": "case-001", - "bench_score": 0.73, - "bench_passed": true, - "bench_reason": "all required checks passed", - "bench_output_path": "/workspace/Safactory/results/mybench/case-001.json" - } -} -``` +# Environment integration + +Read `docs/guides/custom-environment.md` for the request/result and deployment contracts. This workflow separates adapter validation, live deployment, and optional evaluation. + +## Intake and scope -Keep diagnostics on stderr. A controlled task failure should be represented by a failed result; a non-zero process exit is reserved for runtime/infrastructure failure in JSON result mode. +Infer the environment name, source, one-row schema, 1–2 cases, native single-case command, and native output format. Ask only for missing information needed by the next dependent step. Deployment mode/image can remain pending during shared adapter work. Reward fields and scoring rules are needed only for evaluation or training. A benchmark with native scores can still be integrated without enabling evaluation. -## Config patterns +Keep native execution and scoring in the existing harness. The adapter maps one `env_params.dataset` row, routes model calls through the current session, invokes one native case, collects output, and emits the runtime result. Do not run a full benchmark inside a single episode. -Docker task config: +## Copy the fixed templates -```yaml -environments: - - env_name: mybench - env_image: mybench-image:latest - env_num: 1 - dataset: ./datasets/cases.jsonl - dataset_load_mode: eager - env_params: - task_family: mybench - bench_root: /workspace/MyBench - output_root: /workspace/Safactory/results/mybench +Run from the repository root. The scaffolder always emits the shared Docker +pair and the additional RJob pair so the same adapter can be promoted later: + +```bash +python skills/safactory-workflows/scripts/scaffold_environment.py mybench --mode docker +# `--mode rjob` records that the first deployment target is RJob; both pairs are still emitted. +# Append --enable-evaluation only when scoring is requested. ``` -Docker start config: - -```yaml -agent_name: mybench -container: - workdir: /workspace/MyBench - runner_entrypoint: - source: ./runner.py - target: /tmp/safactory-mybench-runner.py - command: "python /tmp/safactory-mybench-runner.py" - mounts: - - source: ./results - target: /workspace/Safactory/results - mode: rw - env: - NO_PROXY: host.docker.internal,localhost,127.0.0.1,::1 - no_proxy: host.docker.internal,localhost,127.0.0.1,::1 - extra_args: - - --add-host=host.docker.internal:host-gateway - idle_command: "tail -f /dev/null" +The generated Docker mount sources are rooted at the launcher working +directory (the repository root in the commands below). Runner sources and +RJob embedded-file sources remain relative to their start-config file. Keep +these two path bases distinct; using `./adapter.py` as a Docker mount from the +repository root would point at `SAfactory/adapter.py`, not `env/mybench/adapter.py`. + +The scaffolder refuses to overwrite an existing environment. For existing integrations, copy/adapt individual templates after inspecting the current files. + +| Output | Template / customization | +|---|---| +| `runner.py` | `assets/environment/runner.py`: fixed request parsing, session URL selection, result serialization, artifact output, and controlled-failure handling. | +| `adapter.py` | `assets/environment/adapter.py`: fill `run_case(request, task, session_url)`, returning `(metrics, step_count)`. Replace the guide's greeting example with the native single-case invocation. | +| `_config.yaml` | `assets/environment/config.yaml.tmpl`: fill the Docker image, dataset, and env parameters. The example dataset has one row. | +| `_config.rjob.yaml` | `assets/environment/config.rjob.yaml.tmpl`: repeat task rows with an image tag pullable by the RJob cluster. Keep the shared `env_params` schema. | +| `_start.yaml` | `assets/environment/start.docker.yaml.tmpl`: fill workdir and Docker mounts. Mount runner dependencies beside the runner. | +| `_start.rjob.yaml` | `assets/environment/start.rjob.yaml.tmpl`: fill resources, cluster-accessible result storage, and embedded dependencies. The runner source is staged by the runtime; `adapter.py` is explicitly embedded. | +| `request.smoke.json` | `assets/environment/request.smoke.json.tmpl`: fill one real case and its environment parameters for local tests. | +| `rule_evaluator.py` | Optional `assets/environment/rule_evaluator.py`: fill only `score_metrics`; keep discovery/interface/failed-result handling. | + +All asset paths above are relative to the skill directory. If extra adapter modules are needed, include them in Docker mounts and RJob embedded files. A Dockerfile is optional when no suitable image exists; derive it from the native harness's dependencies without moving case logic into it. + +The fixed runner has no benchmark imports until it invokes the hook. Its stdout contains exactly one result. Python diagnostic output is redirected to stderr; native subprocesses must use `capture_output=True` or explicitly send stdout to stderr. Controlled exceptions yield `status: failed`, an `error_text`, and process exit 0. Successful execution is `status: succeeded` even if an optional native score is low. Integration-only results keep `total_reward: 0.0`; there is no required score/pass field in metrics. + +## Local validation (no cluster required) + +Fill `request.smoke.json` with a representative row and configure `adapter.py` to invoke the native harness. Run the same tests for Docker and RJob targets: + +```bash +python skills/safactory-workflows/scripts/contract_smoke.py \ + --runner env/mybench/runner.py \ + --request env/mybench/request.smoke.json \ + --require-model-call + +# If native dependencies are not installed locally, copy an explicit fixture +# adapter for protocol-only validation; this does not validate native behavior. +python skills/safactory-workflows/scripts/contract_smoke.py \ + --runner env/mybench/runner.py --adapter path/to/adapter_fixture.py \ + --request env/mybench/request.smoke.json --require-model-call + +# Independently test environment-variable input with empty stdin: +python skills/safactory-workflows/scripts/contract_smoke.py \ + --runner env/mybench/runner.py \ + --request env/mybench/request.smoke.json \ + --input-mode env --require-model-call ``` -RJob start config keeps the `container.runner_entrypoint` contract but adds, for example: - -```yaml -rjob: - name_prefix: mybench - image_pull_policy: IfNotPresent - no_packaging: true - cleanup_on_finish: true - resources: - cpu: 1 - gpu: 0 - memory_in_mb: 1024 - embedded_files: - - source: ./runner.py - target: /tmp/safactory-mybench-runner.py - mount_config: - - "gpfs://CLUSTER_STORAGE/results:/workspace/Safactory/results" +The helper starts a mock HTTP endpoint on an ephemeral loopback port, routes this one request to it, runs the Python runner with a timeout, checks session identity and result types, checks the result artifact if written, and shuts down the endpoint. It emits a `local-contract-only` summary, not a Gateway trajectory or evaluation reward. It uses only Python's standard library; it does not import Launcher or the RJob SDK. + +`--response path/to/response.json` supplies a native-compatible non-streaming chat response. The built-in response is a fixture greeting. Streaming, tool protocols beyond chat JSON, other runner languages, or image-only harness dependencies need environment-specific tests/fixtures. Explicitly stub the native command in those tests and state what was mocked; do not add a production runner switch that fabricates a passing benchmark result. A scaffold greeting passing is not evidence that the benchmark integration works. + +Use 1–2 real-row-shaped fixtures and check that each maps to the intended native command and output. Include a controlled native failure (`--expect-status failed`, without requiring a model call if failure precedes it), malformed requests, and stdout isolation. Inspect mapped metrics/output paths as well as the helper summary. When evaluation is requested, add fixture tests for the scoring hook covering success, low/zero score, and missing/invalid metrics. The evaluator must not execute the native case again. + +Validate selected-mode YAML and relative source paths with the repository config loaders when dependencies are available. For RJob, inspect the rendered runtime command/embedded files without submitting; installing an internal cluster is not a prerequisite for local adapter validation. Document any unverified config checks. + +## Live validation with an owned Gateway + +Only this stage needs the actual image, dataset, model route/credentials, and Docker or an existing configured RJob cluster. Do not require a Geo3K live baseline before local work. A baseline can help diagnose shared infrastructure if a live run fails. + +Use a task config containing only 1–2 cases; `env_num`, `--pool-size`, and `--max-workers` do not limit dataset length. Preserve the user's Gateway config and verify the route and storage. This single command owns Gateway startup/readiness/shutdown and propagates Launcher failures: + +```bash +python skills/safactory-workflows/scripts/live_smoke.py \ + --gateway-config gateway/config.local.yaml \ + --run-timeout 600 -- \ + --mode docker \ + --agent-config env/mybench/mybench_config.yaml \ + --agent-start-config env/mybench/mybench_start.yaml \ + --llm-model YOUR_ROUTE_KEY \ + --job-id mybench-docker-smoke \ + --pool-size 1 --max-workers 1 --max-steps 10 ``` -Do not copy local Docker bind mounts into RJob. RJob images and mounted storage must be accessible from the cluster, and a local runner dependency must be listed in `rjob.embedded_files`. +The helper passes the Gateway's SQLite URI to Launcher if `--db-path` is omitted. Explicit storage and route mismatches fail before starting processes. It requires a free Gateway port; if a Gateway is already running, verify its readiness, routes, and storage, then use `launcher.py` directly. It never stops a Gateway it did not start. Local helper process cleanup does not replace Launcher/Docker/RJob resource cleanup; inspect runtime resources if a timeout interrupts a job. + +For a live RJob run, use `--mode rjob`, `--rjob-config` and both `.rjob.yaml` files, set matching `--storage-type`, and pass `--gateway-base-url http://GATEWAY_HOST:8000/v1/sessions` reachable from the cluster. The helper can start a Gateway on the current host; that host must already be reachable from the cluster. Check that any `gateway_base_url` in the global RJob config agrees, as it can override the CLI value. See `docs/internal/rjob-mode.md` for cluster settings. Local tests do not validate cluster networking, mounts, image pulls, or submission. -## Validation +Integration-only live checks inspect runner JSON, native outputs, Gateway request/trajectory records, and completed runtime rows. Omit `--enable-evaluation`; do not demand an evaluator or final normalized reward. For evaluation requests, implement/test the optional evaluator and append `--enable-evaluation` to the Launcher arguments, then inspect the final `0–10` reward as well. -1. Run or otherwise verify the native command on the 1–2 supplied cases first. -2. Build/pull the selected-mode image and verify the Gateway is reachable from the runtime. -3. Run one-worker smoke evaluation with `--enable-evaluation`. -4. Verify all four artifacts: runner result JSON, native benchmark output file, Gateway trajectory/request log, and final `0–10` reward. -5. For RJob, also verify the global `--rjob-config`, cluster storage, image pull, and non-loopback Gateway URL. +## Report the evidence -Use the root README commands as the command source. Replace `YOUR_ROUTE_KEY` with an actual Gateway `llm_routes` key; never commit private routes or credentials. +State which cases ran, which dependencies/model responses were fixtures, and which level passed: local contract, live deployment, and (if requested) evaluation. Local adapter validation can finish while cluster verification is pending. If the user requested live deployment, list its precise blocker and ready-to-run command without presenting local success as live success. diff --git a/skills/safactory-workflows/scripts/contract_smoke.py b/skills/safactory-workflows/scripts/contract_smoke.py new file mode 100644 index 00000000..a1aed900 --- /dev/null +++ b/skills/safactory-workflows/scripts/contract_smoke.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Check a runner locally with an owned mock model endpoint, without Launcher/cluster. + +Only non-streaming OpenAI chat completions are emulated. Native harness dependencies +must be locally available or replaced by an environment-specific test fixture. +This does not validate Gateway persistence, images, mounts, or cluster submission. +""" +import argparse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import math +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import threading + +from live_smoke import stop_process + + +def validate_result(result, session_id, expected_status): + if not isinstance(result, dict) or result.get("session_id") != session_id: + raise ValueError("result must be an object with the request session_id") + if result.get("status") != expected_status: + raise ValueError(f"unexpected result status: {result.get('status')}: {result.get('error_text')}") + reward = result.get("total_reward") + if type(reward) not in (int, float) or not math.isfinite(reward): + raise ValueError("total_reward must be a finite number (0.0 is valid without evaluation)") + if type(result.get("step_count")) is not int or result["step_count"] < 0: + raise ValueError("step_count must be a nonnegative integer") + if any(type(result.get(key)) is not bool for key in ("terminated", "truncated")): + raise ValueError("terminated/truncated must be booleans") + if not isinstance(result.get("metrics", {}), dict): + raise ValueError("metrics must be an object") + if result.get("error_text") is not None and not isinstance(result["error_text"], str): + raise ValueError("error_text must be null or a string") + if expected_status == "failed" and not result.get("error_text"): + raise ValueError("controlled failure must include error_text") + + +def run_smoke(runner, request, *, response=None, timeout=30, input_mode="stdin", + expected_status="succeeded", require_model_call=False, adapter=None): + request = dict(request) + session_id = request["session_id"] + calls, errors = [], [] + response_body = response if response is not None else { + "id": "contract-response", "object": "chat.completion", "created": 0, + "model": request["model"], + "choices": [{"index": 0, "message": {"role": "assistant", "content": "fixture answer"}, + "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + try: + if self.path != f"/v1/sessions/{session_id}/chat/completions": + raise ValueError(f"unexpected model path: {self.path}") + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0)))) + if body.get("model") != request["model"]: + raise ValueError("model route differs from request.model") + if body.get("stream"): + raise ValueError("streaming needs an environment-specific fixture") + calls.append(body) + payload, status = response_body, 200 + except Exception as exc: + errors.append(str(exc)) + payload, status = {"error": str(exc)}, 400 + raw = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with tempfile.TemporaryDirectory(prefix="safactory-contract-") as temp: + # An optional adapter copy lets a protocol test replace native + # dependencies with an explicit fixture without changing the + # production environment directory. + runner_path = Path(runner).resolve() + if adapter is not None: + isolated = Path(temp) / "runner" + isolated.mkdir() + shutil.copy2(runner_path, isolated / "runner.py") + shutil.copy2(Path(adapter).resolve(), isolated / "adapter.py") + runner_path = isolated / "runner.py" + root_url = f"http://127.0.0.1:{server.server_port}/v1/sessions" + request["gateway_base_url"] = root_url + request["storage_config"] = {"db_url": f"sqlite://{temp}/contract.db"} + session_url = f"{root_url}/{session_id}" + env = dict(os.environ) + env.update({ + "SAFACTORY_SESSION_ID": session_id, + "SAFACTORY_START_REQUEST_JSON": json.dumps(request) if input_mode == "env" else "", + "SAFACTORY_GATEWAY_BASE_URL": root_url, + "SAFACTORY_GATEWAY_SESSION_URL": session_url, + "SAFACTORY_GATEWAY_SESSION_URL_CONTAINER": session_url, + "SAFACTORY_ROUTE_MODEL": request["model"], + "SAFACTORY_MODEL_REF": f"safactory/{request['model']}", + "OPENROUTER_BASE_URL": session_url, + "SAFACTORY_RESULT_PATH": f"{temp}/result.json", + "NO_PROXY": "127.0.0.1,localhost", "no_proxy": "127.0.0.1,localhost", + "PYTHONDONTWRITEBYTECODE": "1", + }) + process = subprocess.Popen( + [sys.executable, str(runner_path)], + env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, start_new_session=True, + ) + try: + stdout, stderr = process.communicate( + json.dumps(request) if input_mode == "stdin" else "", timeout=timeout + ) + finally: + stop_process(process) + for pipe in (process.stdin, process.stdout, process.stderr): + pipe.close() + if stderr: + print(stderr, file=sys.stderr, end="") + if process.returncode: + raise ValueError(f"runner process failed: exit {process.returncode}") + # json.loads deliberately rejects extra stdout, even extra JSON objects. + result = json.loads(stdout) + validate_result(result, session_id, expected_status) + artifact = Path(env["SAFACTORY_RESULT_PATH"]) + if artifact.exists() and json.loads(artifact.read_text()) != result: + raise ValueError("stdout and result artifact differ") + if errors: + raise ValueError("; ".join(errors)) + if require_model_call and not calls: + raise ValueError("runner did not use the mock session endpoint") + return {"validation": "local-contract-only", "model_calls": len(calls), "result": result} + finally: + server.shutdown() + server.server_close() + thread.join() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--runner", type=Path, required=True) + parser.add_argument("--adapter", type=Path, help="Optional fixture adapter copied beside the runner") + parser.add_argument("--request", type=Path, required=True) + parser.add_argument("--response", type=Path, help="Mock non-streaming chat response JSON") + parser.add_argument("--timeout", type=float, default=30) + parser.add_argument("--input-mode", choices=["stdin", "env"], default="stdin") + parser.add_argument("--expect-status", choices=["succeeded", "failed"], default="succeeded") + parser.add_argument("--require-model-call", action="store_true") + args = parser.parse_args() + summary = run_smoke( + args.runner, json.loads(args.request.read_text()), + response=json.loads(args.response.read_text()) if args.response else None, + timeout=args.timeout, input_mode=args.input_mode, expected_status=args.expect_status, + require_model_call=args.require_model_call, adapter=args.adapter, + ) + print(json.dumps(summary, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/skills/safactory-workflows/scripts/live_smoke.py b/skills/safactory-workflows/scripts/live_smoke.py new file mode 100644 index 00000000..7adaf550 --- /dev/null +++ b/skills/safactory-workflows/scripts/live_smoke.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Own a Gateway for one bounded Launcher run; no second terminal required.""" +import argparse +import json +import os +from pathlib import Path +import signal +import socket +import subprocess +import sys +import time +from urllib.parse import urlsplit +from urllib.request import ProxyHandler, build_opener + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def stop_process(process): + if process is None: + return + # Each child owns a new process group, including any descendants it starts. + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + pass + finally: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + + +def run_live(gateway_command, launcher_command, ready_url, log_path, *, + startup_timeout=60, run_timeout=600): + address = urlsplit(ready_url) + try: + with socket.create_connection((address.hostname, address.port or 80), timeout=1): + raise ValueError("Gateway port already occupied; verify that Gateway and use launcher.py directly") + except (ConnectionRefusedError, TimeoutError): + pass + log_path = Path(log_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + gateway = launcher = None + opener = build_opener(ProxyHandler({})) + env = dict(os.environ, SAFACTORY_GATEWAY_LOG_PATH=str(log_path.resolve()) + ".events") + try: + with log_path.open("w", encoding="utf-8") as log: + gateway = subprocess.Popen(gateway_command, stdout=log, stderr=subprocess.STDOUT, + env=env, start_new_session=True) + deadline = time.monotonic() + startup_timeout + while True: + if gateway.poll() is not None: + raise RuntimeError(f"Gateway exited before readiness; see {log_path}") + try: + with opener.open(ready_url, timeout=1) as response: + if response.status == 200 and json.load(response).get("status") == "ready": + break + except (OSError, ValueError): + pass + if time.monotonic() >= deadline: + raise TimeoutError(f"Gateway readiness timed out; see {log_path}") + time.sleep(0.1) + launcher = subprocess.Popen(launcher_command, start_new_session=True) + deadline = time.monotonic() + run_timeout + while launcher.poll() is None: + if gateway.poll() is not None: + raise RuntimeError(f"Gateway exited during Launcher run; see {log_path}") + if time.monotonic() >= deadline: + raise TimeoutError("Launcher smoke run timed out") + time.sleep(0.1) + return launcher.returncode + finally: + stop_process(launcher) + stop_process(gateway) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--gateway-config", required=True) + parser.add_argument("--gateway-log", default="logs/smoke/gateway.log") + parser.add_argument("--startup-timeout", type=float, default=60) + parser.add_argument("--run-timeout", type=float, default=600) + parser.add_argument("launcher_args", nargs=argparse.REMAINDER, help="-- followed by launcher.py flags") + options = parser.parse_args() + argv = options.launcher_args + if argv[:1] == ["--"]: + argv = argv[1:] + if not argv or options.startup_timeout <= 0 or options.run_timeout <= 0: + parser.error("provide Launcher flags after -- and positive timeouts") + if Path.cwd().resolve() != REPO_ROOT: + parser.error("run this helper from the SAfactory repository root") + sys.path.insert(0, str(REPO_ROOT)) + from args import parse_simulation_args + from gateway.config import load_gateway_config + + cfg = load_gateway_config(options.gateway_config) + args = parse_simulation_args(argv) + if args.llm_model not in cfg.llm_routes: + parser.error("--llm-model must match a Gateway llm_routes key") + if args.storage_type != cfg.storage_type: + parser.error("Launcher and Gateway storage_type must match") + if cfg.storage_type == "sqlite": + db_url = cfg.storage_config["db_url"] + if args.db_path and args.db_path != db_url: + parser.error("Launcher --db-path must match Gateway storage_config.db_url") + if not args.db_path: + argv += ["--db-path", db_url] + host = cfg.listen_host + if host in {"0.0.0.0", "::"}: + host = "127.0.0.1" if host == "0.0.0.0" else "::1" + local_host = f"[{host}]" if ":" in host else host + local_root = f"http://{local_host}:{cfg.listen_port}" + explicit_gateway_url = any( + value == "--gateway-base-url" or value.startswith("--gateway-base-url=") for value in argv + ) + if not explicit_gateway_url: + if args.mode == "rjob": + parser.error("RJob requires explicit --gateway-base-url reachable from the cluster") + argv += ["--gateway-base-url", local_root + cfg.base_session_path] + else: + address = urlsplit(args.gateway_base_url) + if (address.port or 80) != cfg.listen_port or address.path.rstrip('/') != cfg.base_session_path.rstrip('/'): + parser.error("Gateway URL port and session path must match the owned Gateway") + if args.mode == "rjob" and address.hostname in {"localhost", "127.0.0.1", "::1"}: + parser.error("RJob Gateway URL must be reachable from the cluster") + # SIGINT already raises KeyboardInterrupt; make SIGTERM run the same cleanup path. + def interrupted(signum, frame): + raise KeyboardInterrupt + + previous_handler = signal.signal(signal.SIGTERM, interrupted) + try: + return run_live( + [sys.executable, "-m", "gateway", "--config", options.gateway_config], + [sys.executable, "launcher.py", *argv], local_root + "/readyz", options.gateway_log, + startup_timeout=options.startup_timeout, run_timeout=options.run_timeout, + ) + finally: + signal.signal(signal.SIGTERM, previous_handler) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/safactory-workflows/scripts/scaffold_environment.py b/skills/safactory-workflows/scripts/scaffold_environment.py new file mode 100644 index 00000000..ae15059f --- /dev/null +++ b/skills/safactory-workflows/scripts/scaffold_environment.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Copy the guide-derived templates without overwriting an existing environment.""" +import argparse +from pathlib import Path +import re + +ASSETS = Path(__file__).resolve().parents[1] / "assets" / "environment" + + +def scaffold(name, mode, env_root, enable_evaluation=False): + if not re.fullmatch(r"[a-z][a-z0-9_]*", name): + raise ValueError("name must start with a lowercase letter and contain only a-z, 0-9, _") + if mode not in {"docker", "rjob"}: + raise ValueError("mode must be docker or rjob") + env_root = Path(env_root) + env_root.mkdir(parents=True, exist_ok=True) + destination = env_root / name + # Docker bind-mount sources are resolved by SAfactory from the launcher's + # current working directory (the repository root for the live-smoke + # workflow), while runner/embedded sources are resolved from the config + # file. Preserve that distinction in generated Docker YAML. + env_root_ref = env_root.expanduser().as_posix() + if not Path(env_root_ref).is_absolute() and not env_root_ref.startswith("."): + env_root_ref = f"./{env_root_ref}" + sources = { + "runner.py": "runner.py", "adapter.py": "adapter.py", + "request.smoke.json": "request.smoke.json.tmpl", + # Docker is the base contract. RJob adds two files; generating both + # pairs up front keeps an environment portable between deployment modes. + f"{name}_config.yaml": "config.yaml.tmpl", + f"{name}_start.yaml": "start.docker.yaml.tmpl", + f"{name}_config.rjob.yaml": "config.rjob.yaml.tmpl", + f"{name}_start.rjob.yaml": "start.rjob.yaml.tmpl", + } + if enable_evaluation: + sources["rule_evaluator.py"] = "rule_evaluator.py" + contents = {target: (ASSETS / source).read_text(encoding="utf-8") + .replace("__ENV_NAME__", name) + .replace("__ENV_ROOT__", env_root_ref) + for target, source in sources.items()} + destination.mkdir(parents=True, exist_ok=False) + for target, content in contents.items(): + (destination / target).write_text(content, encoding="utf-8") + (destination / "datasets").mkdir() + (destination / "datasets" / "smoke.jsonl").write_text( + '{"task_id": "hello-001", "prompt": "Write one short greeting."}\n', encoding="utf-8" + ) + (destination / "results").mkdir() + return destination + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("name") + parser.add_argument("--mode", required=True, choices=["docker", "rjob"]) + parser.add_argument("--env-root", type=Path, default=Path("env")) + parser.add_argument("--enable-evaluation", action="store_true") + args = parser.parse_args() + path = scaffold(args.name, args.mode, args.env_root, args.enable_evaluation) + print( + f"Created {path} for {args.mode} (Docker and RJob config pairs included). " + "Fill adapter.py and deployment values; replace the example dataset." + ) + if args.enable_evaluation: + print("Fill score_metrics in rule_evaluator.py before running evaluation.") + + +if __name__ == "__main__": + main() diff --git a/tests/env/prmeval/test_prmeval_config.py b/tests/env/prmeval/test_prmeval_config.py new file mode 100644 index 00000000..243b1985 --- /dev/null +++ b/tests/env/prmeval/test_prmeval_config.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +ROOT = Path(__file__).parents[3] +ENV_ROOT = ROOT / "env/prmeval" + + +def _read_yaml_text(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def test_standard_file_set_and_matching_names() -> None: + expected = { + "runner.py", + "adapter.py", + "rule_evaluator.py", + "prmeval_config.yaml", + "prmeval_start.yaml", + "prmeval_config.rjob.yaml", + "prmeval_start.rjob.yaml", + "request.smoke.json", + } + assert expected.issubset({path.name for path in ENV_ROOT.iterdir()}) + for path in expected - {"request.smoke.json", "runner.py", "adapter.py", "rule_evaluator.py"}: + text = _read_yaml_text(ENV_ROOT / path) + assert "env_name: prmeval" in text or "agent_name: prmeval" in text + + +def test_smoke_request_has_one_dataset_row_and_native_config() -> None: + request = json.loads((ENV_ROOT / "request.smoke.json").read_text(encoding="utf-8")) + assert request["env_params"]["dataset"]["id"] == "prmeval-contract-001" + assert request["env_params"]["prmeval"]["infer"]["name"] == "openai_compatible" + assert len(request["env_params"]["dataset"]["frames"]) == 3 diff --git a/tests/env/prmeval/test_prmeval_runner.py b/tests/env/prmeval/test_prmeval_runner.py new file mode 100644 index 00000000..7348bae5 --- /dev/null +++ b/tests/env/prmeval/test_prmeval_runner.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import subprocess +import sys + + +ROOT = Path(__file__).parents[3] + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +runner = _load("prmeval_runner_under_test", ROOT / "env/prmeval/runner.py") +adapter = _load("prmeval_adapter_under_test", ROOT / "env/prmeval/adapter.py") +evaluator = _load("prmeval_rule_evaluator_under_test", ROOT / "env/prmeval/rule_evaluator.py") + + +def _request() -> dict: + return { + "job_id": "prmeval-smoke", + "session_id": "session-1", + "model": "route", + "gateway_base_url": "http://127.0.0.1:8000/v1/sessions", + "env_params": { + "dataset": { + "id": "trajectory-1", + "task": "Move the object.", + "frames": "episode.npz", + }, + "media_root": "/tmp/media", + "results_root": "/tmp/results", + "prmeval": {"sampling": {"base_frames": 3}, "infer": {"name": "openai_compatible"}}, + }, + } + + +def test_runner_module_import_does_not_require_prmeval() -> None: + assert callable(runner.read_request) + assert callable(runner.run_episode) + + +def test_relative_frame_path_requires_an_explicit_media_root() -> None: + request = _request() + request["env_params"].pop("media_root") + try: + adapter._prepare_trajectory(request["env_params"]["dataset"], request["env_params"]) + except ValueError as exc: + assert "media_root" in str(exc) + else: + raise AssertionError("relative frames path should require media_root") + + +def test_relative_frame_path_is_resolved_from_media_root() -> None: + task = adapter._prepare_trajectory( + {"id": "one", "task": "Move", "frames": "episode.npz"}, + {"media_root": "/tmp/media"}, + ) + assert task["frames"] == str((Path("/tmp/media") / "episode.npz").resolve()) + + +def test_summary_is_flattened_for_rule_evaluator() -> None: + summary = { + "coverage": {"successful": 1}, + "metrics": {"progress": {"mse": 0.125, "pearson": 0.75, "num_samples": 1}}, + } + metrics = adapter._flatten_summary(summary) + assert metrics["mse"] == 0.125 + assert metrics["pearson"] == 0.75 + assert evaluator.mse_to_reward(0.0) == 10.0 + assert 0.0 < evaluator.mse_to_reward(0.125) < 10.0 + + +def test_malformed_request_emits_one_controlled_failure() -> None: + completed = subprocess.run( + [sys.executable, str(ROOT / "env/prmeval/runner.py")], + input="[1]", + capture_output=True, + text=True, + env={"SAFACTORY_SESSION_ID": "fallback"}, + check=False, + ) + assert completed.returncode == 0 + result = json.loads(completed.stdout) + assert result["session_id"] == "fallback" + assert result["status"] == "failed" + + +def test_rule_evaluator_score_mapping_is_available_without_runtime_dependencies() -> None: + assert evaluator.mse_to_reward(0.125) > 0 + try: + evaluator.mse_to_reward(float("nan")) + except ValueError as exc: + assert "finite" in str(exc) + else: + raise AssertionError("invalid MSE must fail closed") diff --git a/tests/test_environment_skill.py b/tests/test_environment_skill.py new file mode 100644 index 00000000..3a42fc2d --- /dev/null +++ b/tests/test_environment_skill.py @@ -0,0 +1,268 @@ +"""Behavioral checks for the portable onboarding templates and owned test services.""" +import contextlib +import importlib.util +import io +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +REPO_ROOT = Path(__file__).resolve().parents[1] +SKILL = REPO_ROOT / "skills" / "safactory-workflows" +sys.path.insert(0, str(SKILL / "scripts")) +from contract_smoke import run_smoke +from live_smoke import run_live +from scaffold_environment import scaffold +import live_smoke + + +class EnvironmentSkillTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def create(self, mode="docker", evaluation=False): + path = scaffold("mybench", mode, self.root, evaluation) + return path, json.loads((path / "request.smoke.json").read_text()) + + def test_scaffold_preserves_existing_files_and_validates_name(self): + path, _ = self.create() + for name in ( + "runner.py", "adapter.py", "mybench_config.yaml", "mybench_start.yaml", + "mybench_config.rjob.yaml", "mybench_start.rjob.yaml", "request.smoke.json", + ): + self.assertTrue((path / name).exists(), name) + (path / "adapter.py").write_text("user code") + with self.assertRaises(FileExistsError): + scaffold("mybench", "rjob", self.root, True) + self.assertEqual((path / "adapter.py").read_text(), "user code") + self.assertFalse((path / "rule_evaluator.py").exists()) + rjob_config = (path / "mybench_config.rjob.yaml").read_text() + self.assertIn("RJob cluster", rjob_config) + self.assertIn("output_root: /app/results/mybench", rjob_config) + docker_start = (path / "mybench_start.yaml").read_text() + self.assertIn(f"{path / 'adapter.py'}", docker_start) + self.assertIn(f"{path / 'datasets'}", docker_start) + for name in ("../escape", "bad-name", "bad\nname"): + with self.assertRaises(ValueError): + scaffold(name, "docker", self.root) + + def test_both_input_transports_work_without_evaluation_or_cluster(self): + path, request = self.create(mode="rjob") + self.assertFalse((path / "rule_evaluator.py").exists()) + # Even a present evaluator must never be imported by integration-only execution. + (path / "rule_evaluator.py").write_text("raise RuntimeError('evaluation was invoked')") + for mode in ("stdin", "env"): + with self.subTest(mode=mode): + outcome = run_smoke(path / "runner.py", request, input_mode=mode, require_model_call=True) + self.assertEqual(outcome["result"]["metrics"]["answer"], "fixture answer") + self.assertEqual(outcome["result"]["total_reward"], 0.0) + self.assertEqual(outcome["model_calls"], 1) + + def test_contract_helper_can_inject_an_explicit_adapter_fixture(self): + path, request = self.create() + fixture = self.root / "adapter_fixture.py" + fixture.write_text( + "def run_case(request, task, session_url):\n" + " return {'fixture': True, 'task_id': task['task_id']}, 2\n" + ) + outcome = run_smoke( + path / "runner.py", request, adapter=fixture, require_model_call=False + ) + self.assertEqual(outcome["result"]["metrics"]["fixture"], True) + self.assertEqual(outcome["result"]["step_count"], 2) + + def test_two_native_case_fixtures_map_outputs_and_isolate_stdout(self): + path, request = self.create() + native = self.root / "native.py" + native.write_text( + "import json, pathlib, sys\n" + "case, output = sys.argv[1:]\n" + "print('native diagnostic')\n" + "pathlib.Path(output).write_text(json.dumps({'case_id': case, 'answer': case.upper()}))\n" + ) + (path / "adapter.py").write_text( + "import json, pathlib, subprocess, sys\n" + "def run_case(request, task, session_url):\n" + " print('adapter diagnostic')\n" + " output = pathlib.Path(request['env_params']['output_root']) / (task['case_id'] + '.json')\n" + " subprocess.run([sys.executable, request['env_params']['native'], task['case_id'], str(output)],\n" + " check=True, stdout=sys.stderr)\n" + " return {'native': json.loads(output.read_text()), 'output_path': str(output)}, 1\n" + ) + for case in ("case-a", "case-b"): + request["session_id"] = case + request["env_params"] = {"dataset": {"case_id": case}, "native": str(native), + "output_root": str(self.root)} + with contextlib.redirect_stderr(io.StringIO()) as diagnostics: + outcome = run_smoke(path / "runner.py", request) + self.assertIn("native diagnostic", diagnostics.getvalue()) + metrics = outcome["result"]["metrics"] + self.assertEqual(metrics["native"]["case_id"], case) + self.assertEqual(json.loads(Path(metrics["output_path"]).read_text())["answer"], case.upper()) + + def test_controlled_native_failure_preserves_session(self): + path, request = self.create() + (path / "adapter.py").write_text("def run_case(*args):\n raise ValueError('native fixture failed')\n") + outcome = run_smoke(path / "runner.py", request, expected_status="failed") + self.assertEqual(outcome["result"]["session_id"], request["session_id"]) + self.assertEqual(outcome["result"]["error_text"], "native fixture failed") + with self.assertRaisesRegex(ValueError, "unexpected result status"): + run_smoke(path / "runner.py", request) + + def test_malformed_request_is_one_failed_json_with_zero_exit(self): + path, _ = self.create() + for raw in ("[1]", "bad json"): + completed = subprocess.run( + [sys.executable, str(path / "runner.py")], input=raw, + capture_output=True, text=True, + env={**os.environ, "SAFACTORY_SESSION_ID": "fallback", "SAFACTORY_RESULT_PATH": ""}, + ) + self.assertEqual(completed.returncode, 0) + result = json.loads(completed.stdout) + self.assertEqual(result["session_id"], "fallback") + self.assertEqual(result["status"], "failed") + + def test_extra_stdout_is_rejected(self): + path, request = self.create() + runner = path / "runner.py" + runner.write_text("print('unstructured log')\n" + runner.read_text().replace( + "from __future__ import annotations", "" + )) + with self.assertRaises(json.JSONDecodeError): + run_smoke(runner, request) + + def test_contract_timeout_stops_runner(self): + path, request = self.create() + pid_path = self.root / "runner.pid" + (path / "adapter.py").write_text( + "import os, pathlib, time\n" + "def run_case(*args):\n" + f" pathlib.Path({str(pid_path)!r}).write_text(str(os.getpid()))\n" + " time.sleep(60)\n" + ) + with self.assertRaises(subprocess.TimeoutExpired): + run_smoke(path / "runner.py", request, timeout=0.5) + self.assert_stopped(pid_path) + + def test_mock_rejects_wrong_session_route(self): + path, request = self.create() + adapter = path / "adapter.py" + adapter.write_text(adapter.read_text().replace( + 'f"{session_url.rstrip(\'/\')}/chat/completions"', + 'f"{request[\'gateway_base_url\']}/wrong-session/chat/completions"' + )) + with self.assertRaisesRegex(ValueError, "unexpected model path"): + run_smoke(path / "runner.py", request, expected_status="failed") + + def test_optional_evaluator_hook_rejects_missing_and_invalid_scores(self): + path, _ = self.create(evaluation=True) + spec = importlib.util.spec_from_file_location("fixture_evaluator", path / "rule_evaluator.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + request = SimpleNamespace(session_id="one", env_params={"dataset": {}}, + start_result=SimpleNamespace(metrics={"score": 0.7})) + kwargs = dict(request=request, spec=SimpleNamespace(eval_id="rule"), trajectory=None) + self.assertEqual(module.evaluate_rule(**kwargs)["status"], "failed") + module.score_metrics = lambda metrics, dataset, trajectory: (metrics["score"], metrics["score"] * 10, "fixture scale") + for raw in (0.0, 0.7, 1.0): + request.start_result.metrics = {"score": raw} + result = module.evaluate_rule(**kwargs) + self.assertEqual(result["status"], "succeeded") + self.assertEqual(result["normalized_score_10"], raw * 10) + for metrics in ({}, {"score": float("nan")}, {"score": float("inf")}, {"score": 2}): + request.start_result.metrics = metrics + self.assertEqual(module.evaluate_rule(**kwargs)["status"], "failed") + + def live_fixture(self): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + pid_path = self.root / "gateway.pid" + code = ( + "from http.server import BaseHTTPRequestHandler, HTTPServer\n" + "import os, pathlib\n" + f"pathlib.Path({str(pid_path)!r}).write_text(str(os.getpid()))\n" + "class Handler(BaseHTTPRequestHandler):\n" + " def do_GET(self):\n" + " self.send_response(200)\n" + " self.end_headers()\n" + " self.wfile.write(b'{\"status\": \"ready\"}')\n" + f"HTTPServer(('127.0.0.1', {port}), Handler).serve_forever()\n" + ) + return [sys.executable, "-c", code], f"http://127.0.0.1:{port}/readyz", pid_path + + def assert_stopped(self, pid_path): + with self.assertRaises(ProcessLookupError): + os.kill(int(pid_path.read_text()), 0) + + def test_live_helper_propagates_exit_and_stops_gateway(self): + gateway, url, pid_path = self.live_fixture() + result = run_live(gateway, [sys.executable, "-c", "raise SystemExit(7)"], + url, self.root / "gateway.log", startup_timeout=3, run_timeout=3) + self.assertEqual(result, 7) + self.assert_stopped(pid_path) + + def test_live_timeout_stops_both_processes(self): + gateway, url, pid_path = self.live_fixture() + launcher_pid = self.root / "launcher.pid" + launcher = [sys.executable, "-c", "import os, pathlib, time; " + f"pathlib.Path({str(launcher_pid)!r}).write_text(str(os.getpid())); time.sleep(60)"] + with self.assertRaisesRegex(TimeoutError, "Launcher"): + run_live(gateway, launcher, url, self.root / "gateway.log", startup_timeout=3, run_timeout=0.5) + self.assert_stopped(pid_path) + self.assert_stopped(launcher_pid) + + def test_gateway_startup_failure_does_not_launch(self): + _, url, _ = self.live_fixture() + marker = self.root / "launched" + launcher = [sys.executable, "-c", f"open({str(marker)!r}, 'w').close()"] + with self.assertRaisesRegex(RuntimeError, "before readiness"): + run_live([sys.executable, "-c", "raise SystemExit(9)"], launcher, + url, self.root / "gateway.log", startup_timeout=1) + self.assertFalse(marker.exists()) + + def test_live_helper_does_not_take_over_an_existing_port(self): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + sock.listen() + url = f"http://127.0.0.1:{sock.getsockname()[1]}/readyz" + with self.assertRaisesRegex(ValueError, "already occupied"): + run_live([], [], url, self.root / "gateway.log") + self.assertGreaterEqual(sock.fileno(), 0) + + @unittest.skipUnless(importlib.util.find_spec("yaml"), "requires the live runtime's PyYAML dependency") + def test_live_cli_uses_gateway_config_defaults_and_evaluation_is_opt_in(self): + config = self.root / "gateway.json" + config.write_text(json.dumps({ + "listen_port": 8123, "storage_type": "sqlite", + "storage_config": {"db_url": "sqlite://fixture.db"}, + "llm_routes": {"test": {"base_url": "http://unused.invalid/v1"}}, + })) + base = ["live_smoke.py", "--gateway-config", str(config), "--", "--llm-model", "test"] + for evaluation in (False, True): + argv = base + (["--enable-evaluation"] if evaluation else []) + with patch.object(sys, "argv", argv), patch.object(live_smoke, "run_live", return_value=0) as run: + self.assertEqual(live_smoke.main(), 0) + command = run.call_args.args[1] + self.assertEqual("--enable-evaluation" in command, evaluation) + self.assertEqual(command[command.index("--db-path") + 1], "sqlite://fixture.db") + self.assertEqual(command[command.index("--gateway-base-url") + 1], + "http://127.0.0.1:8123/v1/sessions") + for flags in (["--db-path", "sqlite://wrong.db"], ["--storage-type", "cloud"], + ["--mode", "rjob"], ["--gateway-base-url=http://127.0.0.1:9999/v1/sessions"]): + with patch.object(sys, "argv", base + flags), patch.object(live_smoke, "run_live") as run: + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + live_smoke.main() + run.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From c16bd27fc3b25bfe4aff9aea7901f727c60776e7 Mon Sep 17 00:00:00 2001 From: chenxinquan Date: Wed, 16 Sep 2026 15:57:41 +0800 Subject: [PATCH 3/4] =?UTF-8?q?[skills]=20=E7=A7=BB=E9=99=A4=20tests/env/p?= =?UTF-8?q?rmeval=20=E5=86=92=E7=83=9F=E6=B5=8B=E8=AF=95=EF=BC=8C=E4=B8=8E?= =?UTF-8?q?=E4=B8=8A=E6=B8=B8=20v2=20=E4=BF=9D=E6=8C=81=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上游合入 #92 时有意删除了 test_prmeval_config.py 与 test_prmeval_runner.py，本次 feature 同步去掉这两个文件。 --- tests/env/prmeval/test_prmeval_config.py | 36 -------- tests/env/prmeval/test_prmeval_runner.py | 104 ----------------------- 2 files changed, 140 deletions(-) delete mode 100644 tests/env/prmeval/test_prmeval_config.py delete mode 100644 tests/env/prmeval/test_prmeval_runner.py diff --git a/tests/env/prmeval/test_prmeval_config.py b/tests/env/prmeval/test_prmeval_config.py deleted file mode 100644 index 243b1985..00000000 --- a/tests/env/prmeval/test_prmeval_config.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - - -ROOT = Path(__file__).parents[3] -ENV_ROOT = ROOT / "env/prmeval" - - -def _read_yaml_text(path: Path) -> str: - return path.read_text(encoding="utf-8") - - -def test_standard_file_set_and_matching_names() -> None: - expected = { - "runner.py", - "adapter.py", - "rule_evaluator.py", - "prmeval_config.yaml", - "prmeval_start.yaml", - "prmeval_config.rjob.yaml", - "prmeval_start.rjob.yaml", - "request.smoke.json", - } - assert expected.issubset({path.name for path in ENV_ROOT.iterdir()}) - for path in expected - {"request.smoke.json", "runner.py", "adapter.py", "rule_evaluator.py"}: - text = _read_yaml_text(ENV_ROOT / path) - assert "env_name: prmeval" in text or "agent_name: prmeval" in text - - -def test_smoke_request_has_one_dataset_row_and_native_config() -> None: - request = json.loads((ENV_ROOT / "request.smoke.json").read_text(encoding="utf-8")) - assert request["env_params"]["dataset"]["id"] == "prmeval-contract-001" - assert request["env_params"]["prmeval"]["infer"]["name"] == "openai_compatible" - assert len(request["env_params"]["dataset"]["frames"]) == 3 diff --git a/tests/env/prmeval/test_prmeval_runner.py b/tests/env/prmeval/test_prmeval_runner.py deleted file mode 100644 index 7348bae5..00000000 --- a/tests/env/prmeval/test_prmeval_runner.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations - -import importlib.util -import json -from pathlib import Path -import subprocess -import sys - - -ROOT = Path(__file__).parents[3] - - -def _load(name: str, path: Path): - spec = importlib.util.spec_from_file_location(name, path) - assert spec and spec.loader - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - spec.loader.exec_module(module) - return module - - -runner = _load("prmeval_runner_under_test", ROOT / "env/prmeval/runner.py") -adapter = _load("prmeval_adapter_under_test", ROOT / "env/prmeval/adapter.py") -evaluator = _load("prmeval_rule_evaluator_under_test", ROOT / "env/prmeval/rule_evaluator.py") - - -def _request() -> dict: - return { - "job_id": "prmeval-smoke", - "session_id": "session-1", - "model": "route", - "gateway_base_url": "http://127.0.0.1:8000/v1/sessions", - "env_params": { - "dataset": { - "id": "trajectory-1", - "task": "Move the object.", - "frames": "episode.npz", - }, - "media_root": "/tmp/media", - "results_root": "/tmp/results", - "prmeval": {"sampling": {"base_frames": 3}, "infer": {"name": "openai_compatible"}}, - }, - } - - -def test_runner_module_import_does_not_require_prmeval() -> None: - assert callable(runner.read_request) - assert callable(runner.run_episode) - - -def test_relative_frame_path_requires_an_explicit_media_root() -> None: - request = _request() - request["env_params"].pop("media_root") - try: - adapter._prepare_trajectory(request["env_params"]["dataset"], request["env_params"]) - except ValueError as exc: - assert "media_root" in str(exc) - else: - raise AssertionError("relative frames path should require media_root") - - -def test_relative_frame_path_is_resolved_from_media_root() -> None: - task = adapter._prepare_trajectory( - {"id": "one", "task": "Move", "frames": "episode.npz"}, - {"media_root": "/tmp/media"}, - ) - assert task["frames"] == str((Path("/tmp/media") / "episode.npz").resolve()) - - -def test_summary_is_flattened_for_rule_evaluator() -> None: - summary = { - "coverage": {"successful": 1}, - "metrics": {"progress": {"mse": 0.125, "pearson": 0.75, "num_samples": 1}}, - } - metrics = adapter._flatten_summary(summary) - assert metrics["mse"] == 0.125 - assert metrics["pearson"] == 0.75 - assert evaluator.mse_to_reward(0.0) == 10.0 - assert 0.0 < evaluator.mse_to_reward(0.125) < 10.0 - - -def test_malformed_request_emits_one_controlled_failure() -> None: - completed = subprocess.run( - [sys.executable, str(ROOT / "env/prmeval/runner.py")], - input="[1]", - capture_output=True, - text=True, - env={"SAFACTORY_SESSION_ID": "fallback"}, - check=False, - ) - assert completed.returncode == 0 - result = json.loads(completed.stdout) - assert result["session_id"] == "fallback" - assert result["status"] == "failed" - - -def test_rule_evaluator_score_mapping_is_available_without_runtime_dependencies() -> None: - assert evaluator.mse_to_reward(0.125) > 0 - try: - evaluator.mse_to_reward(float("nan")) - except ValueError as exc: - assert "finite" in str(exc) - else: - raise AssertionError("invalid MSE must fail closed") From 8871d8cef0af49956b7b61b0d62d2f056b0cdb97 Mon Sep 17 00:00:00 2001 From: chenxinquan Date: Wed, 16 Sep 2026 17:01:00 +0800 Subject: [PATCH 4/4] =?UTF-8?q?[skills]=20=E4=BB=A5=20env/prmeval=20?= =?UTF-8?q?=E4=B8=BA=E6=A0=87=E5=87=86=E5=9B=BA=E5=8C=96=E6=8E=A5=E5=85=A5?= =?UTF-8?q?=E6=A8=A1=E6=9D=BF=EF=BC=8C=E6=96=B0=E5=A2=9E=E4=B8=80=E9=94=AE?= =?UTF-8?q?=E9=9D=99=E6=80=81=E6=A0=A1=E9=AA=8C=E4=B8=8E=E6=95=85=E9=9A=9C?= =?UTF-8?q?=E4=BE=A7=E5=88=AB=E6=A0=87=E7=AD=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 模板对齐 prmeval 加固版协议壳（artifact 诊断语义、session URL 回退、env_params 校验）； config 模板统一 results_root，新增原生配置块示范；rjob 模板补 private_machine、 代理置空与资源规格；datasets 挂载目标统一为 /tmp/safactory-/datasets； rule_evaluator 收敛为 EvalResult 风格并标注 --enable-evaluation 缺失即失败的语义 - 新增 scripts/validate_environment.py：静态一致性校验（文件集/命名/results_root 与挂载 目标/数据集行内绝对路径/docker-rjob adapter 路径一致），每条 finding 标注 [config]/[env]/[safactory] 归属侧与修复提示 - 新增 scripts/check_environment.py：静态→契约冒烟→可选 live 一键编排，静态失败短路， 原生依赖缺失时提示 --fixture-adapter 只验协议 - scaffold 移除死参数 --mode，生成 README 与 results/.gitkeep，输出下一步命令 - SKILL.md/reference 以 prmeval 为唯一标准，新增故障定位表（症状→归属侧→首查动作） - prmeval adapter 移除框架不注入的 SAFACTORY_RESULTS_ROOT 死代码；evaluation 文档 修正为与代码一致（缺 evaluator 时 episode 失败）；custom-environment 补环境变量与 挂载不变量说明 - 测试：更新 scaffold 断言与 async evaluator，新增校验器/编排器 11 个用例（26 passed） --- .gitignore | 1 + README.md | 2 +- README_CN.md | 2 +- docs/guides/custom-environment.md | 21 +- docs/guides/custom-environment_CN.md | 16 +- docs/guides/evaluation.md | 2 +- docs/guides/evaluation_CN.md | 2 +- env/prmeval/README.md | 14 + env/prmeval/adapter.py | 1 - skills/safactory-workflows/SKILL.md | 41 +- .../assets/environment/README.md.tmpl | 37 ++ .../assets/environment/adapter.py | 23 +- .../assets/environment/config.rjob.yaml.tmpl | 17 +- .../assets/environment/config.yaml.tmpl | 15 +- .../environment/request.smoke.json.tmpl | 2 +- .../assets/environment/rule_evaluator.py | 67 ++- .../assets/environment/runner.py | 134 +++-- .../assets/environment/start.docker.yaml.tmpl | 10 +- .../assets/environment/start.rjob.yaml.tmpl | 27 +- .../references/docker-evaluation.md | 13 +- .../references/environment-integration.md | 126 ++-- .../scripts/check_environment.py | 177 ++++++ .../scripts/scaffold_environment.py | 23 +- .../scripts/validate_environment.py | 548 ++++++++++++++++++ tests/test_environment_skill.py | 45 +- tests/test_environment_validation.py | 139 +++++ 26 files changed, 1302 insertions(+), 203 deletions(-) create mode 100644 skills/safactory-workflows/assets/environment/README.md.tmpl create mode 100644 skills/safactory-workflows/scripts/check_environment.py create mode 100644 skills/safactory-workflows/scripts/validate_environment.py create mode 100644 tests/test_environment_validation.py diff --git a/.gitignore b/.gitignore index 782478e9..a1ae6099 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ AGENTS.md # Local Harbor dataset tooling /env/harbor/generate_vulhub_dataset.py +.zcode/ diff --git a/README.md b/README.md index ba7bba7b..d8260517 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Ask only for missing information needed by the next dependent step. -See [Custom Environments](docs/guides/custom-environment.md) and the skill's [integration reference](skills/safactory-workflows/references/environment-integration.md) for the templates, single-command test helpers, and Docker/RJob deployment settings. If your Agent cannot discover local skills automatically, include `skills/safactory-workflows/` explicitly in the prompt. +See [Custom Environments](docs/guides/custom-environment.md) and the skill's [integration reference](skills/safactory-workflows/references/environment-integration.md) for the templates and Docker/RJob deployment settings. Verify any environment with one command — `python skills/safactory-workflows/scripts/check_environment.py --env env/` — which runs static consistency checks, a contract smoke test against an owned mock endpoint, and an optional live stage, labeling each failure with the side that owns it. If your Agent cannot discover local skills automatically, include `skills/safactory-workflows/` explicitly in the prompt. ## 🚀 Quick Start diff --git a/README_CN.md b/README_CN.md index 31d1e4b2..f515fed8 100644 --- a/README_CN.md +++ b/README_CN.md @@ -97,7 +97,7 @@ runner.py 保留协议处理，adapter.py 只填写单 case 的数据映射、Ga -固定模板、单命令测试 helper 和 Docker/RJob 配置见[自定义环境指南](docs/guides/custom-environment_CN.md)和 skill 的[接入参考](skills/safactory-workflows/references/environment-integration.md)。如果 Agent 不支持自动发现本地 skill，请在 prompt 中显式写出 `skills/safactory-workflows/` 路径。 +固定模板和 Docker/RJob 配置见[自定义环境指南](docs/guides/custom-environment_CN.md)和 skill 的[接入参考](skills/safactory-workflows/references/environment-integration.md)。任意环境可用一条命令完成验证：`python skills/safactory-workflows/scripts/check_environment.py --env env/`，它会依次执行静态一致性检查、基于自有 mock 端点的契约冒烟以及可选的 live 阶段，并为每条失败标注归属方（配置侧 / 环境侧 / 框架侧）。如果 Agent 不支持自动发现本地 skill，请在 prompt 中显式写出 `skills/safactory-workflows/` 路径。 ## 🚀 快速开始 diff --git a/docs/guides/custom-environment.md b/docs/guides/custom-environment.md index ef2466c1..e6768668 100644 --- a/docs/guides/custom-environment.md +++ b/docs/guides/custom-environment.md @@ -32,10 +32,9 @@ Agents and benchmarks mostly differ in the runner and evaluator: Create the guide-derived files with: ```bash -python skills/safactory-workflows/scripts/scaffold_environment.py myagent --mode docker -# Use --mode rjob to mark RJob as the first deployment target. -# Both Docker and RJob config pairs are generated; add --enable-evaluation only -# when evaluation is requested. +python skills/safactory-workflows/scripts/scaffold_environment.py myagent +# Both Docker and RJob config pairs are always generated; add --enable-evaluation +# only when evaluation is requested. ``` The [templates](../../skills/safactory-workflows/assets/environment/) keep protocol handling in `runner.py` and environment logic in `adapter.py:run_case`. Fill that hook and the YAML values; keep the protocol shell unchanged. For benchmarks, replace the generated greeting with the existing native single-case command and output mapping. The scaffolder refuses to overwrite existing directories. Its sample request/dataset verifies the scaffold only; replace those examples with representative cases before claiming integration success. @@ -123,6 +122,13 @@ The runner should print a failed result and exit `0` when the task fails in a co For predictable parsing, keep stdout reserved for the result JSON. Send diagnostic logs to stderr. For long remote runs, the runner may also write the same result object to the path in `SAFACTORY_RESULT_PATH`; Safactory parses stdout first and falls back to that artifact path when stdout does not contain parseable JSON. +Two invariants are checked automatically by +`skills/safactory-workflows/scripts/validate_environment.py` and +`check_environment.py`: `env_params.results_root` must equal the results mount +target declared in the matching start file, and absolute file paths stored +inside dataset rows (for example PRMEval's `frames` field) must point under a +container mount target so they resolve identically in Docker and RJob runs. + ## 2. Read The Request Safactory passes `SimulationStartRequest` both on stdin and in `SAFACTORY_START_REQUEST_JSON`. @@ -158,6 +164,8 @@ Important fields: | `SAFACTORY_GATEWAY_SESSION_URL_CONTAINER` | Container-friendly session URL. Local `localhost` addresses are rewritten to `host.docker.internal`. | | `SAFACTORY_ROUTE_MODEL` | Route model inferred from dataset, `env_params`, or request. | | `SAFACTORY_MODEL_REF` | Provider-style model reference, for example `safactory/`. | +| `SAFACTORY_NATIVE_PARALLEL` | Whether the runtime may run native cases in parallel. | +| `SAFACTORY_OUTPUT_SUBDIR` | Per-episode output subdirectory hint under the results root. | | `OPENROUTER_BASE_URL` | Alias for the container-friendly gateway session URL. | ## 3. Return The Result @@ -219,7 +227,8 @@ environments: dataset_load_mode: eager env_params: task_family: myagent - output_root: /workspace/Safactory/results/myagent + # Must equal the results mount target in myagent_start.yaml. + results_root: /workspace/Safactory/results ``` Create `env/myagent/datasets/tasks.jsonl`: @@ -242,7 +251,7 @@ environments: env_params: task_family: mybench bench_root: /workspace/MyBench - output_root: /workspace/Safactory/results/mybench + results_root: /workspace/Safactory/results ``` ```jsonl diff --git a/docs/guides/custom-environment_CN.md b/docs/guides/custom-environment_CN.md index 18a3e253..4dfb956f 100644 --- a/docs/guides/custom-environment_CN.md +++ b/docs/guides/custom-environment_CN.md @@ -30,8 +30,8 @@ Agent 和 benchmark 的差别主要体现在 runner 和 evaluator： ## 1. 从固定模板开始 ```bash -python skills/safactory-workflows/scripts/scaffold_environment.py myagent --mode docker -# 如果首个目标是 RJob，可改用 --mode rjob；两种模式的 config/start 文件都会生成。 +python skills/safactory-workflows/scripts/scaffold_environment.py myagent +# Docker 与 RJob 两套 config/start 文件固定都会生成。 # 只有需要评测时才追加 --enable-evaluation。 ``` @@ -120,6 +120,11 @@ if __name__ == "__main__": 为了让解析稳定，stdout 最好只输出结果 JSON，诊断日志写到 stderr。对于较长的远程运行，runner 也可以把同一份结果对象写到 `SAFACTORY_RESULT_PATH` 指向的文件中。Safactory 会先解析 stdout，如果 stdout 中没有可解析的 JSON，再从该 artifact 路径读取结果。 +以下两条不变量由 `skills/safactory-workflows/scripts/validate_environment.py` 和 +`check_environment.py` 自动检查：`env_params.results_root` 必须与对应 start 文件中 +results 挂载目标一致；数据集行内存放绝对路径的字段（例如 PRMEval 的 `frames`）必须 +落在某个容器挂载目标之下，以保证在 Docker 与 RJob 两种模式下解析一致。 + ## 2. 读取 Request Safactory 会通过 stdin 和 `SAFACTORY_START_REQUEST_JSON` 同时传入 `SimulationStartRequest`。 @@ -155,6 +160,8 @@ Safactory 会通过 stdin 和 `SAFACTORY_START_REQUEST_JSON` 同时传入 `Simul | `SAFACTORY_GATEWAY_SESSION_URL_CONTAINER` | 容器可访问的 session URL。本地 `localhost` 地址会改写为 `host.docker.internal`。 | | `SAFACTORY_ROUTE_MODEL` | 从 dataset、`env_params` 或 request 推断出的 route model。 | | `SAFACTORY_MODEL_REF` | Provider 风格的模型引用，例如 `safactory/`。 | +| `SAFACTORY_NATIVE_PARALLEL` | 运行时是否可以并行执行原生 case。 | +| `SAFACTORY_OUTPUT_SUBDIR` | 结果根目录下的按 episode 输出子目录提示。 | | `OPENROUTER_BASE_URL` | 容器可访问 gateway session URL 的别名。 | ## 3. 返回 Result @@ -216,7 +223,8 @@ environments: dataset_load_mode: eager env_params: task_family: myagent - output_root: /workspace/Safactory/results/myagent + # 必须与 myagent_start.yaml 中的 results 挂载目标一致。 + results_root: /workspace/Safactory/results ``` 创建 `env/myagent/datasets/tasks.jsonl`： @@ -239,7 +247,7 @@ environments: env_params: task_family: mybench bench_root: /workspace/MyBench - output_root: /workspace/Safactory/results/mybench + results_root: /workspace/Safactory/results ``` ```jsonl diff --git a/docs/guides/evaluation.md b/docs/guides/evaluation.md index 0a36e606..77e53c10 100644 --- a/docs/guides/evaluation.md +++ b/docs/guides/evaluation.md @@ -6,7 +6,7 @@ Safactory only supports environment-local Python rule evaluation. When evaluatio //rule_evaluator.py ``` -The default agent root is `env`. For an environment named `mybench`, the file must be `env/mybench/rule_evaluator.py`. If the file does not exist, evaluation is skipped for that environment. No YAML registration, `env_params` evaluator setting, or separate evaluation config is required. +The default agent root is `env`. For an environment named `mybench`, the file must be `env/mybench/rule_evaluator.py`. If the file does not exist while `--enable-evaluation` is set, the episode is marked as failed with `rule evaluator not found` — omit the flag for integration-only runs. No YAML registration, `env_params` evaluator setting, or separate evaluation config is required. ## Runtime Flow diff --git a/docs/guides/evaluation_CN.md b/docs/guides/evaluation_CN.md index d802e258..487689b6 100644 --- a/docs/guides/evaluation_CN.md +++ b/docs/guides/evaluation_CN.md @@ -6,7 +6,7 @@ Safactory 只保留环境内的 Python 规则评测。开启评测后，系统 //rule_evaluator.py ``` -默认 `agent-root` 是 `env`。例如环境名为 `mybench` 时，评测文件必须是 `env/mybench/rule_evaluator.py`。文件不存在时，该环境跳过评测；不需要在 YAML、`env_params` 或单独的 evaluation config 中注册路径。 +默认 `agent-root` 是 `env`。例如环境名为 `mybench` 时，评测文件必须是 `env/mybench/rule_evaluator.py`。注意：在启用 `--enable-evaluation` 的情况下文件不存在时，该 episode 会被直接判为失败（`rule evaluator not found`）；仅做接入验证时请勿加该开关。不需要在 YAML、`env_params` 或单独的 evaluation config 中注册路径。 ## 运行链路 diff --git a/env/prmeval/README.md b/env/prmeval/README.md index e3662c8e..33f34340 100644 --- a/env/prmeval/README.md +++ b/env/prmeval/README.md @@ -20,6 +20,20 @@ repository root (as `live_smoke.py` requires). Its bind sources therefore use `./env/prmeval/...`; the RJob config instead resolves embedded-file sources relative to its own config file and uses cluster-visible storage. +## Verification + +One command runs the static consistency checks plus the contract smoke +(see `skills/safactory-workflows/references/environment-integration.md`): + +```bash +# Static + contract, with the fixture adapter (PRMEval wheel not required): +python skills/safactory-workflows/scripts/check_environment.py --env env/prmeval \ + --fixture-adapter skills/safactory-workflows/assets/environment/adapter.py + +# Static checks only: +python skills/safactory-workflows/scripts/validate_environment.py env/prmeval +``` + ## Fast local contract check The full PRMEval wheel and model are not needed to validate the SAfactory diff --git a/env/prmeval/adapter.py b/env/prmeval/adapter.py index 593b6673..796efb7b 100644 --- a/env/prmeval/adapter.py +++ b/env/prmeval/adapter.py @@ -40,7 +40,6 @@ def run_case( job_id = _required_text(request.get("job_id"), "job_id") results_root = _first_text( env_params.get("results_root"), - os.environ.get("SAFACTORY_RESULTS_ROOT"), "/tmp/safactory-prmeval-results", ) run_name = _safe_path_part(f"{job_id}-{session_id}") diff --git a/skills/safactory-workflows/SKILL.md b/skills/safactory-workflows/SKILL.md index ba255c17..b3fb0fbd 100644 --- a/skills/safactory-workflows/SKILL.md +++ b/skills/safactory-workflows/SKILL.md @@ -5,7 +5,7 @@ description: Integrate a benchmark or custom environment into SAfactory using fi # SAfactory Workflows -For environment onboarding, read [references/environment-integration.md](references/environment-integration.md) and `docs/guides/custom-environment.md` (or its Chinese translation). The guide defines the runtime contract; the [assets/environment/](assets/environment/) templates implement it. Use `env/geo3k/` for examples of environment-specific behavior, not as boilerplate to copy wholesale. +For environment onboarding, read [references/environment-integration.md](references/environment-integration.md) and `docs/guides/custom-environment.md` (or its Chinese translation). The guide defines the runtime contract; the [assets/environment/](assets/environment/) templates implement it. `env/geo3k/` is only an example of multi-turn agent interaction, not a layout to copy — the standard reference is `env/prmeval/`. For evaluation requests read [references/docker-evaluation.md](references/docker-evaluation.md). For GRPO/RL requests read [references/grpo-training.md](references/grpo-training.md). Read `docs/internal/rjob-mode.md` or its Chinese translation only when working on RJob deployment. @@ -18,47 +18,48 @@ Infer available values from the request and benchmark source before asking: - target deployment mode (`docker` or `rjob`) and image, when known; - whether evaluation is requested; **default to integration only**. -Only evaluation or training requires the native score field, scale, and pass condition. Do not block integration on missing rewards, evaluator code, a running Gateway, or internal cluster access. If mode is unknown, implement and test the shared adapter first; clarify mode before generating deployment configs. Ask about an unknown native command or output format before implementing the dependent hook, while continuing the protocol scaffold and tests. +Only evaluation or training requires the native score field, scale, and pass condition. Do not block integration on missing rewards, evaluator code, a running Gateway, or internal cluster access. If mode is unknown, implement and test the shared adapter first; clarify mode before filling deployment values. Ask about an unknown native command or output format before implementing the dependent hook, while continuing the protocol scaffold and tests. The adapter runs one dataset row through the existing native harness. It does not reimplement the benchmark's case-solving/scoring logic or repair image internals unless that work is requested. Preserve native scores in metrics when available; their presence does not enable evaluation. ## Template-based implementation -1. Scaffold new files with `scripts/scaffold_environment.py` as documented in the integration reference. For an existing directory, adapt the relevant templates without overwriting user files. -2. Keep the protocol shell in `runner.py` fixed. Fill `adapter.py:run_case` with row mapping, model/harness configuration, native execution, and result collection. The generated greeting example only verifies the scaffold; replace it for a real integration. -3. Fill the selected-mode task/start YAML templates with the image, dataset, workdir, mounts, and dependencies. `agent_name` must equal `env_name`; one row is one episode. -4. Create `rule_evaluator.py` only when evaluation is requested. Fill its `score_metrics` hook with the agreed normalization; keep the evaluator interface and failure handling fixed. Do not rerun cases in the evaluator or register its path in YAML. +1. Scaffold the fixed file set with `scripts/scaffold_environment.py ` (both Docker and RJob config pairs are always emitted; append `--enable-evaluation` only when scoring is requested). For an existing directory, adapt the relevant templates without overwriting user files. +2. Keep the protocol shell in `runner.py` fixed — it is a standard part shared with `env/prmeval`; benchmark logic goes only in `adapter.py:run_case`. The generated greeting example only verifies the scaffold; replace it for a real integration. +3. Fill the task/start YAML templates with the image, dataset, workdir, mounts, and dependencies. `agent_name` must equal `env_name`; one row is one episode; `env_params.results_root` must equal the results mount target; absolute paths in dataset rows must point under a mount target. +4. Create `rule_evaluator.py` only when evaluation is requested. Fill its `score_metrics` hook with the agreed normalization; keep the evaluator interface and failure handling fixed. Do not rerun cases in the evaluator or register its path in YAML. **With `--enable-evaluation` set, a missing evaluator marks every episode as failed** — omit the flag for integration-only runs. 5. Python is the default template language. If the native runtime needs Node/shell, retain a thin Python wrapper when possible. If another runner language is required, port the same protocol shell and document why; keep native logic separate and cover the same contract tests. Change shared shell behavior only for an actual runtime-contract requirement, and update the template and its tests together. New files should come from templates, not a fresh implementation of the protocol. ## Standard reference: `env/prmeval` -When an existing benchmark is supplied, compare its integration against -`env/prmeval/` before inventing a new layout. The reference keeps the fixed -`runner.py` protocol shell separate from `adapter.py`, uses one dataset row per -episode, and keeps the optional score mapping in `rule_evaluator.py`. +When an existing benchmark is supplied, compare its integration against `env/prmeval/` before inventing a new layout. The reference keeps the fixed `runner.py` protocol shell separate from `adapter.py`, uses one dataset row per episode, and keeps the optional score mapping in `rule_evaluator.py`. -The Docker pair (`*_config.yaml`, `*_start.yaml`) is the base contract. The -RJob pair (`*_config.rjob.yaml`, `*_start.rjob.yaml`) repeats task parameters -but adds only cluster image, resources, embedded files, and cluster-accessible -mounts. `env_params` is delivered in the request JSON; static `container.env` -or `rjob.env` values are not a replacement for it. +The Docker pair (`*_config.yaml`, `*_start.yaml`) is the base contract. The RJob pair (`*_config.rjob.yaml`, `*_start.rjob.yaml`) repeats task parameters but adds only cluster image, resources, embedded files, and cluster-accessible mounts. `env_params` is delivered in the request JSON; static `container.env` or `rjob.env` values are not a replacement for it. ## Validation and completion -Use the staged checks in the integration reference: +Verify with one command and iterate until green, then add stages as access permits: -1. **Local contract check (default for both target modes):** use `scripts/contract_smoke.py` or equivalent environment-specific fixtures to exercise request input, session URL routing, native output mapping, exact result JSON, and controlled failures. The helper owns its mock endpoint and needs no Gateway, Docker daemon, model credentials, or RJob SDK. Native dependencies still require local availability or explicit test fixtures. -2. **Live deployment check:** when requested or the needed runtime is available, run 1–2 cases with one worker. `scripts/live_smoke.py` starts Gateway, waits for readiness, runs Launcher, and cleans up its processes. Do not require a user to keep a separate terminal open. If an existing Gateway is used, verify it and invoke Launcher directly. -3. **Evaluation check (opt-in):** validate native score conversion and final reward only when evaluation/training is in scope. Pass `--enable-evaluation` explicitly; omit it for integration-only runs. `total_reward: 0.0` satisfies the ungraded runtime result contract and is not an evaluation outcome. +```bash +# Static + contract stages (default; no Gateway, Docker, model credentials, or cluster): +python skills/safactory-workflows/scripts/check_environment.py --env env/ +``` + +1. **Static checks:** file set, naming, and config ↔ start ↔ dataset invariants. Every finding is labeled `[config]`, `[env]`, or `[safactory]` — fix by that ownership, starting with `[config]` errors. +2. **Local contract check:** the real `adapter.py` runs against an owned mock model endpoint. Native dependencies must be available locally; otherwise pass `--fixture-adapter` to verify the protocol shell only and state that native behavior was not validated. +3. **Live deployment check (opt-in via `--live`):** starts Gateway, waits for readiness, runs Launcher, cleans up. Do not require a user to keep a separate terminal open. If an existing Gateway is used, verify it and invoke Launcher directly. +4. **Evaluation check (opt-in):** validate native score conversion and final reward only when evaluation/training is in scope. Pass `--enable-evaluation` explicitly; omit it for integration-only runs. `total_reward: 0.0` satisfies the ungraded runtime result contract and is not an evaluation outcome. + +When something fails, classify it first with the triage table in the integration reference (symptom → owning side → first action) instead of debugging blind; a geo3k live baseline helps isolate shared-infrastructure (safactory-side) problems. Report the adapter/config changes, the cases and fixtures used, which validation level passed, and remaining deployment prerequisites. Local tests can complete adapter validation without an internal cluster; do not claim they validate image pulls, mounts, real Gateway persistence, or RJob scheduling. If a requested live check is blocked, complete independent local checks and report the specific blocker and next command. ## Repository rules - Preserve the user's selected deployment mode. Local contract testing does not substitute Docker deployment for RJob deployment. -- RJob needs the same runner, plus `_config.rjob.yaml` and `_start.rjob.yaml`. The evaluator remains optional. Use cluster-accessible images/storage and `rjob.mount_config`/`rjob.mount`; include all runner dependencies in `rjob.embedded_files`. Do not use loopback Gateway addresses for live cluster runs. +- RJob needs the same runner, plus `_config.rjob.yaml` and `_start.rjob.yaml`. The evaluator remains optional. Use cluster-accessible images/storage and `rjob.mount_config`/`rjob.mount`; include all runner dependencies in `rjob.embedded_files`; replace `CLUSTER_STORAGE` placeholders before live runs. Do not use loopback Gateway addresses for live cluster runs. - Keep Launcher, Gateway, and Buffer Server on the same storage backend/SQLite URI. - Preserve local Gateway configuration and never commit private endpoints or credentials. - Establish a working reward path before starting RL; this requirement does not apply to integration-only work. diff --git a/skills/safactory-workflows/assets/environment/README.md.tmpl b/skills/safactory-workflows/assets/environment/README.md.tmpl new file mode 100644 index 00000000..b13cdcc8 --- /dev/null +++ b/skills/safactory-workflows/assets/environment/README.md.tmpl @@ -0,0 +1,37 @@ +# env/__ENV_NAME__ + +Benchmark integration for SAfactory. Layout follows the standard reference +`env/prmeval/`; the file responsibilities are fixed: + +| File | Responsibility | +| --- | --- | +| `runner.py` | Fixed SAfactory protocol shell. Standard part — do not edit per benchmark; replace wholesale when the protocol evolves. | +| `adapter.py` | The only file with benchmark logic. Fill `run_case(request, task, session_url)` for one dataset row. | +| `__ENV_NAME___config.yaml` | Docker task config: image, dataset, `env_params` passed through to the container per row. | +| `__ENV_NAME___start.yaml` | Docker container startup: entrypoint, mounts, env, network. | +| `__ENV_NAME___config.rjob.yaml` | RJob task config (same schema as Docker, cluster-pullable image). | +| `__ENV_NAME___start.rjob.yaml` | RJob startup: container block plus cluster resources, `mount_config`, `embedded_files`. | +| `rule_evaluator.py` | Optional score mapping. Required only with `--enable-evaluation` (its absence then fails the episode). | +| `request.smoke.json` | One-case request fixture for local contract checks. | +| `datasets/` | One row = one episode. Absolute paths inside rows must point under a mount target. | + +## Verify the integration + +From the repository root: + +```bash +# Static consistency checks (file set, names, config <-> start <-> dataset invariants) +python skills/safactory-workflows/scripts/validate_environment.py env/__ENV_NAME__ + +# Static + contract smoke (owned mock model endpoint; no Gateway/Docker/cluster needed) +python skills/safactory-workflows/scripts/check_environment.py --env env/__ENV_NAME__ + +# Live deployment check (starts and stops Gateway, runs launcher.py); RJob also +# needs cluster access and a cluster-reachable --gateway-base-url. +python skills/safactory-workflows/scripts/check_environment.py --env env/__ENV_NAME__ --live '-- --agent-config env/__ENV_NAME__/__ENV_NAME___config.yaml --agent-start-config env/__ENV_NAME__/__ENV_NAME___start.yaml --llm-model ...' +``` + +Native dependencies must be installed locally for the contract check, or pass +`--fixture-adapter ` to verify the protocol shell only. See +`skills/safactory-workflows/references/environment-integration.md` for the +full workflow and the failure-triage table. diff --git a/skills/safactory-workflows/assets/environment/adapter.py b/skills/safactory-workflows/assets/environment/adapter.py index 236a14c2..1c9e7cff 100644 --- a/skills/safactory-workflows/assets/environment/adapter.py +++ b/skills/safactory-workflows/assets/environment/adapter.py @@ -1,10 +1,19 @@ -"""Environment-specific hook. Replace the guide's greeting example with one native case. +"""Environment-specific hook. The only file with benchmark logic in it. -This example is runnable for scaffold verification; it is not a benchmark integration. -Use only `task` (the current dataset row), never loop over the dataset here. -For a native subprocess, capture stdout or redirect it to stderr explicitly. -Pass session_url and request['model'] to the harness's model client configuration. -Return JSON metrics (including native output paths when available) and a step count. +The runner owns the SAfactory protocol; this module owns only the native +invocation for the current dataset row. Replace the greeting example below +with one native case — it exists only so the scaffold is runnable. + +Rules (see env/prmeval/adapter.py for a full integration): +- Evaluate exactly the one row supplied as ``task``; never loop over the + dataset here. SAfactory schedules one episode per row. +- Do not reimplement the benchmark's case-solving or scoring logic; wrap it. +- Route model calls through ``session_url`` and use ``request['model']`` so + telemetry lands in the Gateway session; never embed provider credentials. +- Capture or redirect native stdout (the runner guards its own stdout, but + subprocess output should still go to a file or stderr explicitly). +- Return JSON-serializable metrics (include native output paths when + available) and a nonnegative step count. """ import json from urllib.request import Request, urlopen @@ -15,7 +24,7 @@ def run_case(request, task, session_url): payload = { "model": request["model"], "messages": [{"role": "user", "content": prompt}], - "temperature": request.get("temperature", 0.3), + "temperature": request.get("temperature", 0.0), } call = Request( f"{session_url.rstrip('/')}/chat/completions", diff --git a/skills/safactory-workflows/assets/environment/config.rjob.yaml.tmpl b/skills/safactory-workflows/assets/environment/config.rjob.yaml.tmpl index 6db954ad..cb572aeb 100644 --- a/skills/safactory-workflows/assets/environment/config.rjob.yaml.tmpl +++ b/skills/safactory-workflows/assets/environment/config.rjob.yaml.tmpl @@ -1,12 +1,19 @@ environments: - env_name: __ENV_NAME__ - # Replace with an image tag that the RJob cluster can pull. - env_image: __ENV_NAME__-image:latest + # Replace with a registry image tag the RJob cluster can pull. + env_image: registry.example.com/__ENV_NAME__-image:latest env_num: 1 dataset: ./datasets/smoke.jsonl dataset_load_mode: eager - # env_params is serialized into SimulationStartRequest and passed to the - # runner for every dataset row. Keep runtime configuration here. + # env_params is serialized into the SimulationStartRequest and delivered to + # the runner inside the container for every dataset row; static container + # env vars are not a replacement for it. env_params: task_family: __ENV_NAME__ - output_root: /app/results/__ENV_NAME__ + # Must equal the results mount target in __ENV_NAME___start.rjob.yaml so + # SAFACTORY_RESULT_PATH artifacts land on the shared results volume. + results_root: /app/results + # Native benchmark configuration goes here and is passed through to + # adapter.py verbatim; keep it identical to the Docker config. + # __ENV_NAME__: + # ...native options... diff --git a/skills/safactory-workflows/assets/environment/config.yaml.tmpl b/skills/safactory-workflows/assets/environment/config.yaml.tmpl index fb88f61f..4197aace 100644 --- a/skills/safactory-workflows/assets/environment/config.yaml.tmpl +++ b/skills/safactory-workflows/assets/environment/config.yaml.tmpl @@ -4,8 +4,17 @@ environments: env_num: 1 dataset: ./datasets/smoke.jsonl dataset_load_mode: eager - # env_params is serialized into SimulationStartRequest and passed to the - # runner for every dataset row. Keep runtime configuration here. + # env_params is serialized into the SimulationStartRequest and delivered to + # the runner inside the container for every dataset row; static container + # env vars are not a replacement for it. env_params: task_family: __ENV_NAME__ - output_root: /workspace/Safactory/results/__ENV_NAME__ + # Must equal the results mount target in __ENV_NAME___start.yaml so + # SAFACTORY_RESULT_PATH artifacts land on the shared results volume. + results_root: /workspace/Safactory/results + # Native benchmark configuration goes here and is passed through to + # adapter.py verbatim (see env/prmeval's prmeval: block). The model + # base_url/api key are injected per episode via session_url; keep only + # benchmark knobs here. + # __ENV_NAME__: + # ...native options... diff --git a/skills/safactory-workflows/assets/environment/request.smoke.json.tmpl b/skills/safactory-workflows/assets/environment/request.smoke.json.tmpl index edd73de9..da07185d 100644 --- a/skills/safactory-workflows/assets/environment/request.smoke.json.tmpl +++ b/skills/safactory-workflows/assets/environment/request.smoke.json.tmpl @@ -6,7 +6,7 @@ "group_id": "", "gateway_base_url": "http://unused.invalid/v1/sessions", "model": "contract-model", - "temperature": 0.3, + "temperature": 0.0, "max_steps": 10, "storage_type": "sqlite", "storage_config": {}, diff --git a/skills/safactory-workflows/assets/environment/rule_evaluator.py b/skills/safactory-workflows/assets/environment/rule_evaluator.py index 10046ada..d74c30d9 100644 --- a/skills/safactory-workflows/assets/environment/rule_evaluator.py +++ b/skills/safactory-workflows/assets/environment/rule_evaluator.py @@ -1,35 +1,62 @@ -"""Optional evaluator shell; fill only score_metrics after agreeing on scoring. +"""Optional evaluation shell; fill only score_metrics after agreeing on scoring. -Discovered by SAfactory only when --enable-evaluation is enabled. -No benchmark execution or runner/image dependencies belong in this file. +Discovered by SAfactory at ``//rule_evaluator.py`` only +when launcher.py runs with ``--enable-evaluation``. WARNING: with that flag +set, a missing rule_evaluator.py marks the whole episode as failed — omit the +flag for integration-only runs. This file must never rerun the benchmark or +call the model; it only maps stored metrics to SAfactory's 0-10 reward. """ +from __future__ import annotations + import math +from typing import Any -def score_metrics(metrics, dataset, trajectory): +def score_metrics(metrics: dict[str, Any], dataset: dict[str, Any], trajectory: Any): """Return (raw_score, score_0_to_10, reason) using the native scoring contract. - This is the environment-specific hook. Reject missing/invalid native scores; - never treat successful adapter execution as a perfect benchmark score. + This is the environment-specific hook. Reject missing or invalid native + scores; never treat successful adapter execution as a perfect score. """ raise NotImplementedError("fill score_metrics using the agreed native score and scale") -def evaluate_rule(*, request, spec, trajectory): - metrics = getattr(request.start_result, "metrics", {}) or {} +async def evaluate_rule(*, request: Any, spec: Any, trajectory: Any) -> Any: + # Keep score helpers importable in lightweight local tooling. The full + # evaluator package is only needed when Launcher actually enables eval. + from evaluator.eval_types import EvalResult, EvalStatus + + metrics = _start_metrics(request) try: - raw, score, reason = score_metrics(metrics, request.env_params.get("dataset", {}), trajectory) + raw, score, reason = score_metrics(metrics, _dataset(request), trajectory) raw, score = float(raw), float(score) if not math.isfinite(raw) or not math.isfinite(score) or not 0 <= score <= 10: - raise ValueError("scores must be finite and normalized score must be in 0–10") - return { - "session_id": request.session_id, "eval_id": spec.eval_id, - "status": "succeeded", "raw_score": raw, - "normalized_score_10": score, "reason": str(reason), "artifacts": {"metrics": metrics}, - } + raise ValueError("scores must be finite and normalized score must be in 0-10") except Exception as exc: - return { - "session_id": request.session_id, "eval_id": spec.eval_id, - "status": "failed", "normalized_score_10": 0.0, - "reason": str(exc), "error_text": str(exc), "artifacts": {"metrics": metrics}, - } + return EvalResult.failed( + session_id=request.session_id, + eval_id=spec.eval_id, + reason=str(exc), + artifacts={"metrics": metrics}, + ) + return EvalResult( + session_id=request.session_id, + eval_id=spec.eval_id, + status=EvalStatus.SUCCEEDED.value, + normalized_score_10=score, + raw_score=raw, + reason=str(reason), + artifacts={"metrics": metrics}, + ) + + +def _start_metrics(request: Any) -> dict[str, Any]: + start_result = getattr(request, "start_result", None) + metrics = getattr(start_result, "metrics", None) + return dict(metrics) if isinstance(metrics, dict) else {} + + +def _dataset(request: Any) -> dict[str, Any]: + env_params = getattr(request, "env_params", None) + dataset = env_params.get("dataset") if isinstance(env_params, dict) else None + return dataset if isinstance(dataset, dict) else {} diff --git a/skills/safactory-workflows/assets/environment/runner.py b/skills/safactory-workflows/assets/environment/runner.py index 126eed0d..01e25438 100644 --- a/skills/safactory-workflows/assets/environment/runner.py +++ b/skills/safactory-workflows/assets/environment/runner.py @@ -1,8 +1,15 @@ #!/usr/bin/env python3 -"""Fixed SAfactory protocol shell, based on docs/guides/custom-environment.md. +"""Fixed SAfactory protocol shell. Standard part: do not edit per benchmark. -Put environment-specific behavior in adapter.py; keep this shell unchanged. +This file is copied unchanged into every environment (see ``env/prmeval``). +It owns only the SAfactory runtime contract: reading the +SimulationStartRequest, resolving the Gateway session URL, invoking the +environment hook, and emitting exactly one result JSON on stdout plus the +optional ``SAFACTORY_RESULT_PATH`` artifact. Benchmark-specific behavior +lives in ``adapter.py:run_case``. When the protocol evolves, replace this +whole file mechanically instead of editing around benchmark logic. """ + from __future__ import annotations import contextlib @@ -10,10 +17,10 @@ import os from pathlib import Path import sys -from urllib.parse import urlsplit, urlunsplit +from typing import Any -def read_request(): +def read_request() -> dict[str, Any]: raw = sys.stdin.read().strip() or os.environ.get("SAFACTORY_START_REQUEST_JSON", "") if not raw: raise ValueError("missing SimulationStartRequest JSON") @@ -23,50 +30,94 @@ def read_request(): return request -def main(): - result = { - "session_id": os.environ.get("SAFACTORY_SESSION_ID", ""), - "status": "failed", "total_reward": 0.0, "step_count": 0, - "terminated": True, "truncated": False, "error_text": None, "metrics": {}, +def run_episode(request: dict[str, Any]) -> dict[str, Any]: + session_id = _required_text(request.get("session_id"), "session_id") + env_params = request.get("env_params") + if not isinstance(env_params, dict): + raise TypeError("env_params must be a JSON object") + task = env_params.get("dataset") + if not isinstance(task, dict): + raise TypeError("env_params.dataset must be a JSON object") + + session_url = _first_text( + os.environ.get("SAFACTORY_GATEWAY_SESSION_URL_CONTAINER"), + _gateway_session_url(request, session_id), + os.environ.get("OPENROUTER_BASE_URL"), + os.environ.get("OPENAI_BASE_URL"), + ) + if not session_url: + raise ValueError("cannot resolve Gateway session URL") + + # Import the environment hook lazily and with stdout guarded: native + # libraries may print, and stdout must contain exactly one result JSON. + # Protocol checks stay runnable without the adapter's native dependencies + # by swapping in a fixture adapter (contract_smoke --adapter). + with contextlib.redirect_stdout(sys.stderr): + from adapter import run_case + + metrics, step_count = run_case(request, task, session_url) + if not isinstance(metrics, dict): + raise TypeError("adapter.run_case metrics must be a JSON object") + if type(step_count) is not int or step_count < 0: + raise ValueError("adapter.run_case step_count must be a nonnegative integer") + json.dumps(metrics, ensure_ascii=False, allow_nan=False) + return { + "session_id": session_id, + "status": "succeeded", + "total_reward": 0.0, + "step_count": step_count, + "terminated": True, + "truncated": False, + "error_text": None, + "metrics": metrics, } + + +def main() -> int: + session_id = os.environ.get("SAFACTORY_SESSION_ID", "") try: request = read_request() - result["session_id"] = str(request["session_id"]) - session_url = os.environ.get("SAFACTORY_GATEWAY_SESSION_URL_CONTAINER") or _gateway_session_url( - request, result["session_id"] - ) - if not session_url: - raise ValueError("cannot resolve Gateway session URL") - task = (request.get("env_params") or {}).get("dataset") or {} - with contextlib.redirect_stdout(sys.stderr): - from adapter import run_case - - metrics, step_count = run_case(request, task, session_url) - if not isinstance(metrics, dict): - raise ValueError("run_case metrics must be a JSON object") - if type(step_count) is not int or step_count < 0: - raise ValueError("run_case step_count must be a nonnegative integer") - json.dumps(metrics, allow_nan=False) - result.update(status="succeeded", metrics=metrics, step_count=step_count) - except Exception as exc: - result["error_text"] = str(exc) - - artifact = os.environ.get("SAFACTORY_RESULT_PATH") + session_id = str(request.get("session_id") or session_id) + result = run_episode(request) + except Exception as exc: # Runtime failures are represented in the result JSON. + result = { + "session_id": session_id, + "status": "failed", + "total_reward": 0.0, + "step_count": 0, + "terminated": True, + "truncated": False, + "error_text": str(exc), + "metrics": {}, + } + _write_result(result) + return 0 + + +def _write_result(result: dict[str, Any]) -> None: + artifact = str(os.environ.get("SAFACTORY_RESULT_PATH") or "").strip() if artifact: try: path = Path(artifact) path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(result, ensure_ascii=False, allow_nan=False), encoding="utf-8") + path.write_text( + json.dumps(result, ensure_ascii=False, allow_nan=False) + "\n", + encoding="utf-8", + ) except Exception as exc: - result.update(status="failed", error_text=f"cannot write result artifact: {exc}") + # Keep the stdout result intact; the framework parses stdout first. + print(f"SAFACTORY_RUNNER_DIAGNOSTIC result_artifact_write_failed: {exc}", file=sys.stderr) print(json.dumps(result, ensure_ascii=False, allow_nan=False), flush=True) - return 0 # Controlled failures are result JSON, not process failures. -def _gateway_session_url(request, session_id): +def _gateway_session_url(request: dict[str, Any], session_id: str) -> str: base = str(request.get("gateway_base_url") or "").rstrip("/") if not base: return "" + # request_env normally performs this rewrite. Keep the fallback useful + # when the runner is invoked by hand or by a local contract test. + from urllib.parse import urlsplit, urlunsplit + parts = urlsplit(base) if parts.hostname in {"127.0.0.1", "localhost", "::1"}: netloc = "host.docker.internal" @@ -76,5 +127,20 @@ def _gateway_session_url(request, session_id): return f"{base}/{session_id}" +def _first_text(*values: Any) -> str: + for value in values: + text = str(value or "").strip() + if text: + return text + return "" + + +def _required_text(value: Any, name: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError(f"SimulationStartRequest missing {name}") + return text + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/skills/safactory-workflows/assets/environment/start.docker.yaml.tmpl b/skills/safactory-workflows/assets/environment/start.docker.yaml.tmpl index bdad79ba..4370ce17 100644 --- a/skills/safactory-workflows/assets/environment/start.docker.yaml.tmpl +++ b/skills/safactory-workflows/assets/environment/start.docker.yaml.tmpl @@ -7,14 +7,16 @@ container: command: "python /tmp/safactory-__ENV_NAME__/runner.py" mounts: # Docker bind sources are resolved from the repository root when the - # launcher/live_smoke helper is run there. + # launcher/live_smoke helper is run there; runner_entrypoint.source is + # resolved from this config file's directory. - source: __ENV_ROOT__/__ENV_NAME__/adapter.py target: /tmp/safactory-__ENV_NAME__/adapter.py mode: ro - # Mount native benchmark assets read by the current case. If the dataset - # is remote, replace this with the image's own path or remove the mount. + # Mount the dataset directory read-only. Absolute file paths stored in + # dataset rows (see env/prmeval's frames field) must point inside a mount + # target — keep them in sync with this path. - source: __ENV_ROOT__/__ENV_NAME__/datasets - target: /workspace/Safactory/datasets/__ENV_NAME__ + target: /tmp/safactory-__ENV_NAME__/datasets mode: ro - source: ./results target: /workspace/Safactory/results diff --git a/skills/safactory-workflows/assets/environment/start.rjob.yaml.tmpl b/skills/safactory-workflows/assets/environment/start.rjob.yaml.tmpl index 5ebb15ea..8fbbe2d4 100644 --- a/skills/safactory-workflows/assets/environment/start.rjob.yaml.tmpl +++ b/skills/safactory-workflows/assets/environment/start.rjob.yaml.tmpl @@ -7,21 +7,34 @@ container: command: "python /tmp/safactory-__ENV_NAME__/runner.py" env: PYTHONDONTWRITEBYTECODE: "1" + # The cluster injects corporate proxies that break loopback Gateway + # access; blank them unless the environment needs egress. + HTTP_PROXY: "" + HTTPS_PROXY: "" + ALL_PROXY: "" + http_proxy: "" + https_proxy: "" + all_proxy: "" + NO_PROXY: "" + no_proxy: "" rjob: name_prefix: __ENV_NAME__ + private_machine: Group image_pull_policy: IfNotPresent no_packaging: true cleanup_on_finish: true keep_failed_jobs: true resources: - cpu: 1 + cpu: 2 gpu: 0 - memory_in_mb: 1024 - embedded_files: - - source: ./adapter.py - target: /tmp/safactory-__ENV_NAME__/adapter.py - # Add cluster-accessible result storage via mount_config/mount for live runs. + memory_in_mb: 4096 mount_config: - # Replace CLUSTER_STORAGE with a path visible to the RJob cluster. + # Replace CLUSTER_STORAGE with a location visible to the RJob cluster. - "gpfs://CLUSTER_STORAGE/SAfactory/results:/app/results" + # Mount the dataset directory at the same container path as the Docker + # start config so dataset-row absolute paths resolve identically. - "gpfs://CLUSTER_STORAGE/SAfactory/env/__ENV_NAME__/datasets:/tmp/safactory-__ENV_NAME__/datasets" + embedded_files: + # runner.py is embedded automatically from runner_entrypoint.source. + - source: ./adapter.py + target: /tmp/safactory-__ENV_NAME__/adapter.py diff --git a/skills/safactory-workflows/references/docker-evaluation.md b/skills/safactory-workflows/references/docker-evaluation.md index 9d5c2c9f..8a7689e8 100644 --- a/skills/safactory-workflows/references/docker-evaluation.md +++ b/skills/safactory-workflows/references/docker-evaluation.md @@ -84,8 +84,11 @@ After a run, inspect: Common failures: -- `--llm-model` does not match a Gateway route key. -- Gateway base URL points to an LLM proxy instead of `/v1/sessions`. -- Docker image or mounted runner path is missing. -- Gateway and Launcher write to different storage backends. -- Runner exits non-zero for benchmark-level failure instead of returning a failed result. +- `--llm-model` does not match a Gateway route key. *(safactory-side: Gateway/launcher configuration)* +- Gateway base URL points to an LLM proxy instead of `/v1/sessions`. *(safactory-side)* +- Docker image or mounted runner path is missing. *(config-side: environment wiring)* +- Gateway and Launcher write to different storage backends. *(safactory-side)* +- Runner exits non-zero for benchmark-level failure instead of returning a failed result. *(env-side: protocol violation)* +- `rule evaluator not found for environment` — `--enable-evaluation` was set but `rule_evaluator.py` is absent; the episode is then marked failed. *(config-side)* + +For a fuller symptom → owning-side → first-action table, see the "Failure triage" section of [environment-integration.md](environment-integration.md); `check_environment.py --env env/` performs the static subset automatically. diff --git a/skills/safactory-workflows/references/environment-integration.md b/skills/safactory-workflows/references/environment-integration.md index c88b0c3b..e983cb3b 100644 --- a/skills/safactory-workflows/references/environment-integration.md +++ b/skills/safactory-workflows/references/environment-integration.md @@ -1,6 +1,6 @@ # Environment integration -Read `docs/guides/custom-environment.md` for the request/result and deployment contracts. This workflow separates adapter validation, live deployment, and optional evaluation. +Read `docs/guides/custom-environment.md` for the request/result and deployment contracts. The standard reference layout is `env/prmeval/`; every new environment comes from the same fixed template set. This workflow separates adapter validation, live deployment, and optional evaluation, and labels every failure with the side that owns it. ## Intake and scope @@ -8,95 +8,99 @@ Infer the environment name, source, one-row schema, 1–2 cases, native single-c Keep native execution and scoring in the existing harness. The adapter maps one `env_params.dataset` row, routes model calls through the current session, invokes one native case, collects output, and emits the runtime result. Do not run a full benchmark inside a single episode. -## Copy the fixed templates +## The fixed file set -Run from the repository root. The scaffolder always emits the shared Docker -pair and the additional RJob pair so the same adapter can be promoted later: +Run from the repository root: ```bash -python skills/safactory-workflows/scripts/scaffold_environment.py mybench --mode docker -# `--mode rjob` records that the first deployment target is RJob; both pairs are still emitted. +python skills/safactory-workflows/scripts/scaffold_environment.py mybench # Append --enable-evaluation only when scoring is requested. ``` -The generated Docker mount sources are rooted at the launcher working -directory (the repository root in the commands below). Runner sources and -RJob embedded-file sources remain relative to their start-config file. Keep -these two path bases distinct; using `./adapter.py` as a Docker mount from the -repository root would point at `SAfactory/adapter.py`, not `env/mybench/adapter.py`. +The scaffolder refuses to overwrite an existing environment and always emits the full fixed file set — the Docker pair is the base contract and the RJob pair adds cluster settings, keeping the environment portable between modes: -The scaffolder refuses to overwrite an existing environment. For existing integrations, copy/adapt individual templates after inspecting the current files. +| File | Responsibility | Filled per benchmark? | +|---|---|---| +| `runner.py` | Fixed SAfactory protocol shell: request parsing, session URL resolution, one result JSON, controlled failure, artifact write. | No — standard part; replace wholesale when the protocol evolves. | +| `adapter.py` | The only benchmark-logic file: `run_case(request, task, session_url)` for exactly one dataset row. | Yes — the main integration work. | +| `_config.yaml` | Docker task config: image, dataset, `env_params` (serialized into every request and transparently passed into the container). | Yes — image, dataset path, native config block. | +| `_start.yaml` | Docker container startup: entrypoint, mounts, env vars, network. | Yes — mounts for datasets/results and any extra deps. | +| `_config.rjob.yaml` | RJob task config, same schema; image must be pullable by the cluster. | Mirror the Docker config. | +| `_start.rjob.yaml` | RJob startup: the same container block plus `rjob:` resources, `mount_config`, `embedded_files`. | Yes — cluster storage paths. | +| `rule_evaluator.py` | Optional score mapping (`score_metrics` hook); discovered only with `--enable-evaluation`. | Only when evaluation is requested. | +| `request.smoke.json` | One-case request fixture for local contract checks. | Yes — use a real row shape. | -| Output | Template / customization | -|---|---| -| `runner.py` | `assets/environment/runner.py`: fixed request parsing, session URL selection, result serialization, artifact output, and controlled-failure handling. | -| `adapter.py` | `assets/environment/adapter.py`: fill `run_case(request, task, session_url)`, returning `(metrics, step_count)`. Replace the guide's greeting example with the native single-case invocation. | -| `_config.yaml` | `assets/environment/config.yaml.tmpl`: fill the Docker image, dataset, and env parameters. The example dataset has one row. | -| `_config.rjob.yaml` | `assets/environment/config.rjob.yaml.tmpl`: repeat task rows with an image tag pullable by the RJob cluster. Keep the shared `env_params` schema. | -| `_start.yaml` | `assets/environment/start.docker.yaml.tmpl`: fill workdir and Docker mounts. Mount runner dependencies beside the runner. | -| `_start.rjob.yaml` | `assets/environment/start.rjob.yaml.tmpl`: fill resources, cluster-accessible result storage, and embedded dependencies. The runner source is staged by the runtime; `adapter.py` is explicitly embedded. | -| `request.smoke.json` | `assets/environment/request.smoke.json.tmpl`: fill one real case and its environment parameters for local tests. | -| `rule_evaluator.py` | Optional `assets/environment/rule_evaluator.py`: fill only `score_metrics`; keep discovery/interface/failed-result handling. | +Key invariants (checked automatically, see below): `agent_name` must equal `env_name`; `env_params.results_root` must equal the results mount target in the matching start file; absolute file paths inside dataset rows must point under a mount target (both modes); the adapter must sit beside the runner in the container in both modes. -All asset paths above are relative to the skill directory. If extra adapter modules are needed, include them in Docker mounts and RJob embedded files. A Dockerfile is optional when no suitable image exists; derive it from the native harness's dependencies without moving case logic into it. +The generated Docker mount sources are rooted at the launcher working directory (the repository root in the commands below). Runner sources and RJob embedded-file sources remain relative to their start-config file. Keep these two path bases distinct; using `./adapter.py` as a Docker mount from the repository root would point at `SAfactory/adapter.py`, not `env/mybench/adapter.py`. + +If extra adapter modules are needed, include them in Docker mounts and RJob embedded files. A Dockerfile is optional when no suitable image exists; derive it from the native harness's dependencies without moving case logic into it. The fixed runner has no benchmark imports until it invokes the hook. Its stdout contains exactly one result. Python diagnostic output is redirected to stderr; native subprocesses must use `capture_output=True` or explicitly send stdout to stderr. Controlled exceptions yield `status: failed`, an `error_text`, and process exit 0. Successful execution is `status: succeeded` even if an optional native score is low. Integration-only results keep `total_reward: 0.0`; there is no required score/pass field in metrics. -## Local validation (no cluster required) +## Verify with one command -Fill `request.smoke.json` with a representative row and configure `adapter.py` to invoke the native harness. Run the same tests for Docker and RJob targets: +`check_environment.py` chains the stages and short-circuits: static errors skip the contract run, and the live stage never starts on top of failures. ```bash -python skills/safactory-workflows/scripts/contract_smoke.py \ - --runner env/mybench/runner.py \ - --request env/mybench/request.smoke.json \ - --require-model-call - -# If native dependencies are not installed locally, copy an explicit fixture -# adapter for protocol-only validation; this does not validate native behavior. -python skills/safactory-workflows/scripts/contract_smoke.py \ - --runner env/mybench/runner.py --adapter path/to/adapter_fixture.py \ - --request env/mybench/request.smoke.json --require-model-call - -# Independently test environment-variable input with empty stdin: -python skills/safactory-workflows/scripts/contract_smoke.py \ - --runner env/mybench/runner.py \ - --request env/mybench/request.smoke.json \ - --input-mode env --require-model-call +python skills/safactory-workflows/scripts/check_environment.py --env env/mybench + +# Static checks only (file set, naming, config <-> start <-> dataset invariants): +python skills/safactory-workflows/scripts/validate_environment.py env/mybench + +# Protocol shell only, when native dependencies are not installed locally: +python skills/safactory-workflows/scripts/check_environment.py --env env/mybench \ + --fixture-adapter path/to/adapter_fixture.py + +# Live deployment: everything after --live is forwarded to live_smoke.py. +python skills/safactory-workflows/scripts/check_environment.py --env env/mybench --live \ + --gateway-config gateway/config.local.yaml --run-timeout 600 -- \ + --mode docker \ + --agent-config env/mybench/mybench_config.yaml \ + --agent-start-config env/mybench/mybench_start.yaml \ + --llm-model YOUR_ROUTE_KEY \ + --job-id mybench-docker-smoke --pool-size 1 --max-workers 1 --max-steps 10 ``` -The helper starts a mock HTTP endpoint on an ephemeral loopback port, routes this one request to it, runs the Python runner with a timeout, checks session identity and result types, checks the result artifact if written, and shuts down the endpoint. It emits a `local-contract-only` summary, not a Gateway trajectory or evaluation reward. It uses only Python's standard library; it does not import Launcher or the RJob SDK. +Static findings are labeled `[config]` (integration wiring in the environment directory), `[env]` (adapter code, dataset rows, request fixture), or `[safactory]` (framework side — the integration is fine). Fix all errors; warnings name the deployment mode they block. -`--response path/to/response.json` supplies a native-compatible non-streaming chat response. The built-in response is a fixture greeting. Streaming, tool protocols beyond chat JSON, other runner languages, or image-only harness dependencies need environment-specific tests/fixtures. Explicitly stub the native command in those tests and state what was mocked; do not add a production runner switch that fabricates a passing benchmark result. A scaffold greeting passing is not evidence that the benchmark integration works. +The contract stage runs the real `adapter.py` against an owned mock model endpoint on an ephemeral loopback port: no Gateway, Docker daemon, model credentials, or RJob SDK. It checks session identity, result schema, artifact equality, model routing through the session, and controlled-failure shape. `--fixture-adapter` substitutes a stub to verify the protocol shell only; a fixture passing is not evidence that the benchmark integration works. `--response` (via `contract_smoke.py`) supplies a native-shaped non-streaming chat response; streaming, tool protocols beyond chat JSON, other runner languages, or image-only harness dependencies need environment-specific fixtures. -Use 1–2 real-row-shaped fixtures and check that each maps to the intended native command and output. Include a controlled native failure (`--expect-status failed`, without requiring a model call if failure precedes it), malformed requests, and stdout isolation. Inspect mapped metrics/output paths as well as the helper summary. When evaluation is requested, add fixture tests for the scoring hook covering success, low/zero score, and missing/invalid metrics. The evaluator must not execute the native case again. +Use 1–2 real-row-shaped fixtures and check that each maps to the intended native command and output. Include a controlled native failure (`--expect-status failed` via `contract_smoke.py`), malformed requests, and stdout isolation. When evaluation is requested, add fixture tests for the scoring hook covering success, low/zero score, and missing/invalid metrics; the evaluator must never rerun the case. -Validate selected-mode YAML and relative source paths with the repository config loaders when dependencies are available. For RJob, inspect the rendered runtime command/embedded files without submitting; installing an internal cluster is not a prerequisite for local adapter validation. Document any unverified config checks. +## Failure triage -## Live validation with an owned Gateway +Classify before debugging; each row names the side that owns the fix: -Only this stage needs the actual image, dataset, model route/credentials, and Docker or an existing configured RJob cluster. Do not require a Geo3K live baseline before local work. A baseline can help diagnose shared infrastructure if a live run fails. +| Symptom | Side | First action | +|---|---|---| +| Static `[config]` finding (name/mount/results_root/dataset-path mismatch) | config | Fix the named file per the hint; rerun `check_environment.py`. | +| Contract: `runner process failed: exit N` | env | The runner must always exit 0; convert the failure into a `status: failed` result JSON. | +| Contract: result `status: failed` with `error_text` | env | Controlled native/adapter failure — debug `adapter.py` or the native command. | +| Contract: `unexpected model path` / `model route differs` | env | The adapter must call `{session_url}/chat/completions` with `request["model"]`. | +| Contract: `ModuleNotFoundError` for native deps | env | Install deps locally, or use `--fixture-adapter` to isolate protocol from native logic. | +| Contract: stdout parse / "extra stdout" errors | env | Native prints leaked onto stdout — capture or redirect them to stderr. | +| live_smoke preflight error (route/storage/URL mismatch) | safactory | Fix Gateway config vs launcher args — configuration of shared infrastructure. | +| `Gateway exited before readiness` / readiness timeout | safactory | Gateway-side startup failure; check the Gateway log, then a geo3k baseline to isolate. | +| `Gateway exited during Launcher run` | safactory | Shared infrastructure died mid-run; Gateway logs first, then rerun. | +| Launcher exit non-zero with no result JSON | safactory/env | Launcher/Gateway logs first; if the container ran, it is an environment protocol violation. | +| Episode result `status: failed`, exit 0 | env | In-band environment failure — read `error_text`, then native output under `results_root`. | +| Everything green but no files under `results/` | config | `results_root` not mounted (the static stage flags this) or wrong artifact path. | +| `rule evaluator not found for environment` | config | `--enable-evaluation` was set without `rule_evaluator.py`; add it or drop the flag. | +| Reward always 0.0 with `--enable-evaluation` off | — | Expected: integration-only runs keep `total_reward: 0.0`; it is not an evaluation outcome. | -Use a task config containing only 1–2 cases; `env_num`, `--pool-size`, and `--max-workers` do not limit dataset length. Preserve the user's Gateway config and verify the route and storage. This single command owns Gateway startup/readiness/shutdown and propagates Launcher failures: +A geo3k live baseline can help decide safactory-side vs environment-side when shared infrastructure is suspected. Do not require it before local work. -```bash -python skills/safactory-workflows/scripts/live_smoke.py \ - --gateway-config gateway/config.local.yaml \ - --run-timeout 600 -- \ - --mode docker \ - --agent-config env/mybench/mybench_config.yaml \ - --agent-start-config env/mybench/mybench_start.yaml \ - --llm-model YOUR_ROUTE_KEY \ - --job-id mybench-docker-smoke \ - --pool-size 1 --max-workers 1 --max-steps 10 -``` +## Live validation with an owned Gateway + +Only this stage needs the actual image, dataset, model route/credentials, and Docker or an existing configured RJob cluster. Use a task config containing only 1–2 cases; `env_num`, `--pool-size`, and `--max-workers` do not limit dataset length. Preserve the user's Gateway config and verify the route and storage. The helper owns Gateway startup/readiness/shutdown and propagates Launcher failures; see the `--live` command above or call `live_smoke.py` directly. The helper passes the Gateway's SQLite URI to Launcher if `--db-path` is omitted. Explicit storage and route mismatches fail before starting processes. It requires a free Gateway port; if a Gateway is already running, verify its readiness, routes, and storage, then use `launcher.py` directly. It never stops a Gateway it did not start. Local helper process cleanup does not replace Launcher/Docker/RJob resource cleanup; inspect runtime resources if a timeout interrupts a job. -For a live RJob run, use `--mode rjob`, `--rjob-config` and both `.rjob.yaml` files, set matching `--storage-type`, and pass `--gateway-base-url http://GATEWAY_HOST:8000/v1/sessions` reachable from the cluster. The helper can start a Gateway on the current host; that host must already be reachable from the cluster. Check that any `gateway_base_url` in the global RJob config agrees, as it can override the CLI value. See `docs/internal/rjob-mode.md` for cluster settings. Local tests do not validate cluster networking, mounts, image pulls, or submission. +For a live RJob run, use `--mode rjob`, `--rjob-config` and both `.rjob.yaml` files, replace the `CLUSTER_STORAGE` placeholders, set matching `--storage-type`, and pass `--gateway-base-url http://GATEWAY_HOST:8000/v1/sessions` reachable from the cluster. The helper can start a Gateway on the current host; that host must already be reachable from the cluster. Check that any `gateway_base_url` in the global RJob config agrees, as it can override the CLI value. See `docs/internal/rjob-mode.md` for cluster settings. Local tests do not validate cluster networking, mounts, image pulls, or submission. Integration-only live checks inspect runner JSON, native outputs, Gateway request/trajectory records, and completed runtime rows. Omit `--enable-evaluation`; do not demand an evaluator or final normalized reward. For evaluation requests, implement/test the optional evaluator and append `--enable-evaluation` to the Launcher arguments, then inspect the final `0–10` reward as well. ## Report the evidence -State which cases ran, which dependencies/model responses were fixtures, and which level passed: local contract, live deployment, and (if requested) evaluation. Local adapter validation can finish while cluster verification is pending. If the user requested live deployment, list its precise blocker and ready-to-run command without presenting local success as live success. +State which cases ran, which dependencies/model responses were fixtures, and which level passed: static, local contract, live deployment, and (if requested) evaluation. Local adapter validation can finish while cluster verification is pending. If the user requested live deployment, list its precise blocker and ready-to-run command without presenting local success as live success. diff --git a/skills/safactory-workflows/scripts/check_environment.py b/skills/safactory-workflows/scripts/check_environment.py new file mode 100644 index 00000000..7e5ed552 --- /dev/null +++ b/skills/safactory-workflows/scripts/check_environment.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""One-command environment verification: static checks -> contract smoke -> optional live run. + +Usage (from the repository root): + + python skills/safactory-workflows/scripts/check_environment.py --env env/ [options] + +Stages: + +1. Static validation (validate_environment.py): file set, naming, and + config <-> start <-> dataset invariants, each finding labeled with the + owning side (config / env). +2. Contract smoke (contract_smoke.py): runs the real runner against an owned + mock model endpoint. Native dependencies must be installed locally; pass + ``--fixture-adapter`` to verify the protocol shell only. Skipped when + static validation reports errors. +3. Live deployment (live_smoke.py), only with ``--live``: starts a Gateway, + runs launcher.py, and cleans up. ``--live`` consumes all remaining + arguments and forwards them to live_smoke.py verbatim, e.g.: + + check_environment.py --env env/prmeval --live \\ + --gateway-config gateway.yaml -- \\ + --agent-config env/prmeval/prmeval_config.yaml \\ + --agent-start-config env/prmeval/prmeval_start.yaml --llm-model + +Exit code 0 only when every executed stage passes. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import subprocess +import sys +from dataclasses import asdict + +SCRIPTS = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPTS)) + +from contract_smoke import run_smoke # noqa: E402 +from validate_environment import ( # noqa: E402 + SEV_INFO, + validate_environment, +) + +LIVE_SMOKE = SCRIPTS / "live_smoke.py" + +DEPS_MARKERS = ("ModuleNotFoundError", "ImportError", "No module named") + + +def check(env_dir: Path, *, expect_evaluation: bool = False, request_path: Path | None = None, + fixture_adapter: Path | None = None, timeout: float = 30, + require_model_call: bool = True, live_args: list[str] | None = None) -> dict: + stages: dict[str, dict] = {} + + # Stage 1: static validation. + report = validate_environment(env_dir, expect_evaluation=expect_evaluation) + stages["static"] = { + "ok": report.ok(), + "findings": [asdict(f) for f in report.findings if f.severity != SEV_INFO], + } + + # Stage 2: contract smoke (skipped when static errors exist). + if not report.ok(): + stages["contract"] = {"ok": False, "skipped": True, + "reason": "static validation failed; fix [config] findings first"} + else: + stages["contract"] = _contract_stage( + env_dir, request_path=request_path, fixture_adapter=fixture_adapter, + timeout=timeout, require_model_call=require_model_call, + ) + + # Stage 3: live deployment (opt-in). + if live_args is None: + stages["live"] = {"ok": True, "skipped": True, "reason": "not requested (pass --live)"} + elif not stages["static"]["ok"] or not stages["contract"]["ok"]: + stages["live"] = {"ok": False, "skipped": True, + "reason": "earlier stages failed; live run would only add noise"} + else: + stages["live"] = _live_stage(live_args) + + stages["summary"] = { + "ok": stages["static"]["ok"] and stages["contract"]["ok"] and stages["live"]["ok"] + } + return stages + + +def _contract_stage(env_dir: Path, *, request_path, fixture_adapter, timeout, + require_model_call) -> dict: + runner = env_dir / "runner.py" + request_file = request_path or (env_dir / "request.smoke.json") + try: + request = json.loads(request_file.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + return {"ok": False, "error": f"cannot read smoke request {request_file}: {exc}", + "side": "env"} + try: + outcome = run_smoke( + runner, request, adapter=fixture_adapter, timeout=timeout, + require_model_call=require_model_call, + ) + except ValueError as exc: + message = str(exc) + side = "env" + hint = "" + if any(marker in message for marker in DEPS_MARKERS): + hint = ("native dependencies are missing locally; install them, or pass " + "--fixture-adapter to verify the protocol shell only") + elif "runner process failed" in message: + hint = "the runner must always exit 0 and report failures inside the result JSON" + return {"ok": False, "error": message, "side": side, "hint": hint} + return {"ok": True, "outcome": {"model_calls": outcome.get("model_calls"), + "result": outcome.get("result")}} + + +def _live_stage(live_args: list[str]) -> dict: + completed = subprocess.run([sys.executable, str(LIVE_SMOKE), *live_args], check=False) + return {"ok": completed.returncode == 0, "exit_code": completed.returncode} + + +def _print_stages(stages: dict) -> None: + order = [("static", "1/3 static validation"), ("contract", "2/3 contract smoke"), + ("live", "3/3 live deployment")] + for key, title in order: + stage = stages[key] + status = "SKIPPED" if stage.get("skipped") else ("PASS" if stage["ok"] else "FAIL") + print(f"== {title}: {status}") + if key == "static": + for finding in stage["findings"]: + severity, side = finding["severity"], finding["side"] + print(f" [{severity}] [{side}] {finding['file']}: {finding['message']}") + if finding["hint"]: + print(f" hint: {finding['hint']}") + elif not stage.get("skipped"): + detail = stage.get("error") or stage.get("outcome") or {"exit_code": stage.get("exit_code")} + print(f" {json.dumps(detail, ensure_ascii=False)[:400]}") + elif stage.get("reason"): + print(f" {stage['reason']}") + summary = stages["summary"] + print(f"SUMMARY: {'OK' if summary['ok'] else 'FAILED'}") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--env", type=Path, required=True, help="environment directory, e.g. env/mybench") + parser.add_argument("--expect-evaluation", action="store_true", + help="require rule_evaluator.py to exist (static stage)") + parser.add_argument("--request", type=Path, default=None, + help="smoke request JSON (default: /request.smoke.json)") + parser.add_argument("--fixture-adapter", type=Path, default=None, + help="swap in a fixture adapter to verify the protocol shell without native deps") + parser.add_argument("--timeout", type=float, default=30, help="contract runner timeout in seconds") + parser.add_argument("--no-require-model-call", action="store_true", + help="do not require the adapter to route through the mock session") + parser.add_argument("--live", nargs=argparse.REMAINDER, default=None, metavar="ARGS", + help="run the live stage; all remaining arguments are forwarded to live_smoke.py") + parser.add_argument("--json", action="store_true", help="print the stage report as JSON") + args = parser.parse_args(argv) + + live_args = list(args.live) if args.live is not None else None + if live_args and live_args[0] == "--": + live_args = live_args[1:] + stages = check( + args.env, expect_evaluation=args.expect_evaluation, request_path=args.request, + fixture_adapter=args.fixture_adapter, timeout=args.timeout, + require_model_call=not args.no_require_model_call, live_args=live_args, + ) + if args.json: + print(json.dumps(stages, ensure_ascii=False, indent=2, default=str)) + else: + _print_stages(stages) + return 0 if stages["summary"]["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/safactory-workflows/scripts/scaffold_environment.py b/skills/safactory-workflows/scripts/scaffold_environment.py index ae15059f..9c017924 100644 --- a/skills/safactory-workflows/scripts/scaffold_environment.py +++ b/skills/safactory-workflows/scripts/scaffold_environment.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 -"""Copy the guide-derived templates without overwriting an existing environment.""" +"""Scaffold a new environment from the guide-derived standard templates. + +Generates the fixed file set (both Docker and RJob config pairs) without +overwriting an existing environment. The standard reference is env/prmeval. +""" import argparse from pathlib import Path import re @@ -7,11 +11,9 @@ ASSETS = Path(__file__).resolve().parents[1] / "assets" / "environment" -def scaffold(name, mode, env_root, enable_evaluation=False): +def scaffold(name, env_root, enable_evaluation=False): if not re.fullmatch(r"[a-z][a-z0-9_]*", name): raise ValueError("name must start with a lowercase letter and contain only a-z, 0-9, _") - if mode not in {"docker", "rjob"}: - raise ValueError("mode must be docker or rjob") env_root = Path(env_root) env_root.mkdir(parents=True, exist_ok=True) destination = env_root / name @@ -24,9 +26,10 @@ def scaffold(name, mode, env_root, enable_evaluation=False): env_root_ref = f"./{env_root_ref}" sources = { "runner.py": "runner.py", "adapter.py": "adapter.py", + "README.md": "README.md.tmpl", "request.smoke.json": "request.smoke.json.tmpl", - # Docker is the base contract. RJob adds two files; generating both - # pairs up front keeps an environment portable between deployment modes. + # Docker is the base contract. RJob adds two files; the fixed file + # set always includes both pairs so an environment stays portable. f"{name}_config.yaml": "config.yaml.tmpl", f"{name}_start.yaml": "start.docker.yaml.tmpl", f"{name}_config.rjob.yaml": "config.rjob.yaml.tmpl", @@ -46,23 +49,25 @@ def scaffold(name, mode, env_root, enable_evaluation=False): '{"task_id": "hello-001", "prompt": "Write one short greeting."}\n', encoding="utf-8" ) (destination / "results").mkdir() + (destination / "results" / ".gitkeep").write_text("", encoding="utf-8") return destination def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("name") - parser.add_argument("--mode", required=True, choices=["docker", "rjob"]) parser.add_argument("--env-root", type=Path, default=Path("env")) parser.add_argument("--enable-evaluation", action="store_true") args = parser.parse_args() - path = scaffold(args.name, args.mode, args.env_root, args.enable_evaluation) + path = scaffold(args.name, args.env_root, args.enable_evaluation) print( - f"Created {path} for {args.mode} (Docker and RJob config pairs included). " + f"Created {path} (Docker and RJob config pairs included). " "Fill adapter.py and deployment values; replace the example dataset." ) if args.enable_evaluation: print("Fill score_metrics in rule_evaluator.py before running evaluation.") + print("Next steps, from the repository root:") + print(f" python skills/safactory-workflows/scripts/check_environment.py --env {path}") if __name__ == "__main__": diff --git a/skills/safactory-workflows/scripts/validate_environment.py b/skills/safactory-workflows/scripts/validate_environment.py new file mode 100644 index 00000000..0d0d1542 --- /dev/null +++ b/skills/safactory-workflows/scripts/validate_environment.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python3 +"""Static consistency checks for a SAfactory environment directory. + +Usage (from the repository root, matching launcher path resolution): + + python skills/safactory-workflows/scripts/validate_environment.py env/ [--expect-evaluation] [--json] + +Every finding carries a *side* label that maps the problem to an owner: + +- ``config`` — integration wiring in this directory (config/start YAMLs, file + set, naming). Fix by editing the environment's files. +- ``env`` — environment/benchmark side (adapter code, dataset rows, + request fixture). Fix in adapter.py, the dataset, or the benchmark image. +- ``safactory`` — framework side; if a check fails here the integration is + fine and the issue belongs to SAfactory itself. + +Exit code 0 when there are no error-severity findings (warnings allowed). +""" +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass, asdict +from pathlib import Path +import re +import sys +from typing import Any + +try: + import yaml # type: ignore +except ImportError: # pragma: no cover - degrade like live_smoke does + yaml = None + +SIDE_ENV = "env" +SIDE_CONFIG = "config" +SIDE_SAFATORY = "safactory" +SEV_ERROR = "error" +SEV_WARN = "warn" +SEV_INFO = "info" + +# Loose protocol markers of the standard runner shell. A missing marker means +# someone hand-edited the standard part; restore it from the skill assets. +RUNNER_MARKERS = ( + "def read_request", + "SAFACTORY_START_REQUEST_JSON", + "SAFACTORY_RESULT_PATH", + "from adapter import run_case", + "redirect_stdout", + "SAFACTORY_RUNNER_DIAGNOSTIC", +) + + +@dataclass +class Finding: + code: str + side: str + severity: str + file: str + message: str + hint: str = "" + + def line(self) -> str: + text = f"[{self.severity}] [{self.side}] {self.file}: {self.message}" + return f"{text}\n hint: {self.hint}" if self.hint else text + + +class Report: + def __init__(self) -> None: + self.findings: list[Finding] = [] + + def add(self, code: str, side: str, severity: str, file: str, message: str, hint: str = "") -> None: + self.findings.append(Finding(code, side, severity, file, message, hint)) + + @property + def errors(self) -> list[Finding]: + return [f for f in self.findings if f.severity == SEV_ERROR] + + def ok(self) -> bool: + return not self.errors + + +def validate_environment(env_dir: Path, expect_evaluation: bool = False) -> Report: + report = Report() + env_dir = env_dir.resolve() + name = env_dir.name + + if not env_dir.is_dir(): + report.add("env-dir", SIDE_CONFIG, SEV_ERROR, str(env_dir), "environment directory does not exist") + return report + if not re.fullmatch(r"[a-z][a-z0-9_]*", name): + report.add( + "env-name", SIDE_CONFIG, SEV_ERROR, name, + "environment name must start with a lowercase letter and contain only a-z, 0-9, _", + ) + + _check_file_set(env_dir, name, expect_evaluation, report) + _check_python_files(env_dir, report) + if yaml is None: + report.add( + "yaml-missing", SIDE_SAFATORY, SEV_INFO, "-", + "PyYAML is not installed; YAML consistency checks skipped", + "pip install pyyaml to enable config/start/dataset checks", + ) + return report + + docker_cfg = _load_yaml(env_dir / f"{name}_config.yaml", report) + rjob_cfg = _load_yaml(env_dir / f"{name}_config.rjob.yaml", report) + docker_start = _load_yaml(env_dir / f"{name}_start.yaml", report) + rjob_start = _load_yaml(env_dir / f"{name}_start.rjob.yaml", report) + + docker_env = _check_config(docker_cfg, env_dir, name, report, docker=True) + rjob_env = _check_config(rjob_cfg, env_dir, name, report, docker=False) + docker_mounts = _check_docker_start(docker_start, env_dir, name, docker_env, report) + rjob_targets, rjob_adapter_target = _check_rjob_start(rjob_start, env_dir, name, rjob_env, report) + _check_cross(docker_env, rjob_env, docker_cfg, rjob_cfg, report) + + docker_adapter_target = next( + (m["target"] for m in docker_mounts if m["target"].endswith("adapter.py")), "" + ) + if docker_adapter_target and rjob_adapter_target and docker_adapter_target != rjob_adapter_target: + report.add( + "adapter-path-drift", SIDE_CONFIG, SEV_ERROR, "adapter.py", + f"docker mounts adapter at {docker_adapter_target!r} but rjob embeds it at {rjob_adapter_target!r}", + "keep the container path identical so `from adapter import run_case` works in both modes", + ) + + rows = _check_dataset(docker_env, env_dir, docker_cfg, report) + _check_dataset_paths(rows, docker_mounts, rjob_targets, report) + _check_smoke_request(env_dir, report) + return report + + +def _check_file_set(env_dir: Path, name: str, expect_evaluation: bool, report: Report) -> None: + required = [ + "runner.py", + "adapter.py", + "request.smoke.json", + f"{name}_config.yaml", + f"{name}_start.yaml", + f"{name}_config.rjob.yaml", + f"{name}_start.rjob.yaml", + ] + for rel in required: + if not (env_dir / rel).is_file(): + report.add( + "file-set", SIDE_CONFIG, SEV_ERROR, rel, f"required file {rel!r} is missing", + "generate it from skills/safactory-workflows/assets/environment/ templates", + ) + if expect_evaluation and not (env_dir / "rule_evaluator.py").is_file(): + report.add( + "evaluator-missing", SIDE_CONFIG, SEV_ERROR, "rule_evaluator.py", + "--expect-evaluation is set but rule_evaluator.py is missing", + "with --enable-evaluation a missing evaluator marks every episode as failed", + ) + if not (env_dir / "datasets").is_dir(): + report.add("datasets-dir", SIDE_CONFIG, SEV_WARN, "datasets/", "datasets/ directory is missing") + if not (env_dir / "results").is_dir(): + report.add("results-dir", SIDE_CONFIG, SEV_WARN, "results/", + "results/ directory is missing", "add results/.gitkeep so the host mount target exists") + + +def _check_python_files(env_dir: Path, report: Report) -> None: + evaluator = env_dir / "rule_evaluator.py" + for path in [env_dir / "runner.py", env_dir / "adapter.py", evaluator]: + if not path.is_file(): + continue + try: + compile(path.read_text(encoding="utf-8"), str(path), "exec") + except SyntaxError as exc: + report.add("py-compile", SIDE_ENV, SEV_ERROR, path.name, f"Python syntax error: {exc}") + runner = env_dir / "runner.py" + if runner.is_file(): + source = runner.read_text(encoding="utf-8") + missing = [m for m in RUNNER_MARKERS if m not in source] + if missing: + report.add( + "runner-drift", SIDE_CONFIG, SEV_WARN, "runner.py", + f"standard protocol shell markers missing: {', '.join(missing)}", + "runner.py is a fixed standard part; restore it from the skill assets or env/prmeval", + ) + if evaluator.is_file(): + source = evaluator.read_text(encoding="utf-8") + if not any(marker in source for marker in ("def evaluate_rule", "class RuleEvaluator", "def evaluate")): + report.add( + "evaluator-contract", SIDE_ENV, SEV_ERROR, "rule_evaluator.py", + "exports none of evaluate_rule / RuleEvaluator / evaluate", + "the framework discovers the evaluator by these entry points", + ) + + +def _load_yaml(path: Path, report: Report) -> dict[str, Any]: + if not path.is_file(): + return {} + try: + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) + except yaml.YAMLError as exc: + report.add("yaml-parse", SIDE_CONFIG, SEV_ERROR, path.name, f"invalid YAML: {exc}") + return {} + return loaded if isinstance(loaded, dict) else {} + + +def _config_entry(config: dict[str, Any]) -> dict[str, Any]: + entries = config.get("environments") + if isinstance(entries, list) and entries and isinstance(entries[0], dict): + return entries[0] + return {} + + +def _check_config( + config: dict[str, Any], env_dir: Path, name: str, report: Report, *, docker: bool +) -> dict[str, Any]: + suffix = "_config.yaml" if docker else "_config.rjob.yaml" + if not config: + report.add("config-empty", SIDE_CONFIG, SEV_ERROR, name + suffix, + "missing `environments:` list or file unparseable") + return {} + entry = _config_entry(config) + rel = name + suffix + + if entry.get("env_name") != name: + report.add( + "env-name", SIDE_CONFIG, SEV_ERROR, rel, + f"env_name {entry.get('env_name')!r} does not match directory name {name!r}", + "the scheduler looks up start configs by env_name", + ) + image = str(entry.get("env_image") or "") + if not image: + report.add("env-image", SIDE_CONFIG, SEV_ERROR, rel, "env_image is empty") + elif docker and (":" not in image or "/" in image and image.rsplit("/", 1)[0].startswith("gpfs")): + report.add("env-image", SIDE_CONFIG, SEV_WARN, rel, + f"docker env_image {image!r} should carry an explicit tag") + env_num = entry.get("env_num", 1) + if not isinstance(env_num, int) or env_num < 1: + report.add("env-num", SIDE_CONFIG, SEV_WARN, rel, "env_num must be an integer >= 1") + + mode = str(entry.get("dataset_load_mode") or "") + if mode and mode not in {"eager", "parquet_row_ref"}: + report.add("dataset-mode", SIDE_CONFIG, SEV_ERROR, rel, + f"dataset_load_mode {mode!r} is not one of eager / parquet_row_ref") + + env_params = entry.get("env_params") + if not isinstance(env_params, dict): + report.add("env-params", SIDE_CONFIG, SEV_ERROR, rel, + "env_params must be a mapping; it is serialized into every request") + env_params = {} + else: + results_root = str(env_params.get("results_root") or "") + if not results_root: + report.add("results-root", SIDE_CONFIG, SEV_WARN, rel, + "env_params.results_root is not set", + "SAFACTORY_RESULT_PATH then defaults to /app/results and may not match any mount") + + dataset = str(entry.get("dataset") or "") + if dataset and not dataset.startswith(("http", "hf://")) and mode != "parquet_row_ref": + dataset_path = Path(dataset) + if not dataset_path.is_absolute(): + dataset_path = env_dir / dataset_path + if not dataset_path.is_file(): + report.add("dataset-file", SIDE_CONFIG, SEV_ERROR, rel, + f"dataset file {dataset!r} not found", + "dataset paths resolve relative to the config file's directory") + return env_params + + +def _mounts_of(start: dict[str, Any]) -> list[dict[str, str]]: + container = start.get("container") or {} + mounts = container.get("mounts") or [] + result = [] + for mount in mounts: + if isinstance(mount, str): + source, _, target = mount.partition(":") + result.append({"source": source, "target": target, "mode": "ro"}) + elif isinstance(mount, dict): + result.append({ + "source": str(mount.get("source") or mount.get("hostPath") or mount.get("host_path") or ""), + "target": str(mount.get("target") or mount.get("containerPath") or mount.get("container_path") or ""), + "mode": str(mount.get("mode") or "ro"), + }) + return result + + +def _check_docker_start( + start: dict[str, Any], env_dir: Path, name: str, env_params: dict[str, Any], report: Report +) -> list[dict[str, str]]: + rel = f"{name}_start.yaml" + if not start: + report.add("start-empty", SIDE_CONFIG, SEV_ERROR, rel, "file missing or unparseable") + return [] + agent_names = {str(start.get("agent_name") or "")} if start.get("agent_name") else set(start.get("agents") or {}) + if name not in agent_names: + report.add( + "agent-name", SIDE_CONFIG, SEV_ERROR, rel, + f"agent_name {sorted(a for a in agent_names if a)} does not match env_name {name!r}", + "agent_name must equal env_name so the scheduler finds this start config", + ) + + container = start.get("container") or {} + entrypoint = container.get("runner_entrypoint") or {} + if isinstance(entrypoint, dict): + source = str(entrypoint.get("source") or "") + if source: + source_path = Path(source) + if not source_path.is_absolute(): + # runner_entrypoint sources resolve from the config file. + source_path = (env_dir / source_path).resolve() + if not source_path.is_file(): + report.add("entrypoint-source", SIDE_CONFIG, SEV_ERROR, rel, + f"runner_entrypoint source {source!r} not found") + if not str(entrypoint.get("command") or ""): + report.add("entrypoint-command", SIDE_CONFIG, SEV_ERROR, rel, + "runner_entrypoint.command is empty") + + mounts = _mounts_of(start) + if not any(m["target"].endswith("adapter.py") for m in mounts): + report.add( + "adapter-mount", SIDE_CONFIG, SEV_WARN, rel, + "adapter.py is not mounted into the container", + "the runner does `from adapter import run_case`; mount it beside the runner target", + ) + for mount in mounts: + source = mount["source"] + if not source or source.startswith(("gpfs://", "nfs://")) or source.startswith("/"): + if source.startswith(("gpfs://", "nfs://")) or (source.startswith("/") and not Path(source).exists()): + report.add("mount-source", SIDE_CONFIG, SEV_WARN, rel, + f"mount source {source!r} cannot be checked from this workstation") + continue + # Plain bind sources resolve from the launcher cwd (repository root). + resolved = Path(source) + if not resolved.exists(): + report.add( + "mount-source", SIDE_CONFIG, SEV_ERROR, rel, + f"mount source {source!r} not found from the current directory", + "docker bind sources resolve from the launcher's cwd; run from the repository root", + ) + if mount["target"]: + _check_mount_mode_rw(rel, mount, report) + + results_root = str(env_params.get("results_root") or "") + if results_root and not any(m["target"].rstrip("/") == results_root.rstrip("/") for m in mounts): + report.add( + "results-mount", SIDE_CONFIG, SEV_ERROR, rel, + f"env_params.results_root {results_root!r} is not a mount target", + "SAFACTORY_RESULT_PATH artifacts would stay inside the container; mount the host results dir there", + ) + return mounts + + +def _check_mount_mode_rw(rel: str, mount: dict[str, str], report: Report) -> None: + if "results" in mount["target"] and mount["mode"] not in {"rw", "w"}: + report.add("results-mount-ro", SIDE_CONFIG, SEV_WARN, rel, + f"results mount {mount['target']!r} is read-only", "artifacts cannot be written") + + +def _check_rjob_start( + start: dict[str, Any], env_dir: Path, name: str, env_params: dict[str, Any], report: Report +) -> tuple[list[str], str]: + rel = f"{name}_start.rjob.yaml" + if not start: + report.add("start-empty", SIDE_CONFIG, SEV_ERROR, rel, "file missing or unparseable") + return [], "" + agent_name = str(start.get("agent_name") or "") + if agent_name and agent_name != name: + report.add("agent-name", SIDE_CONFIG, SEV_ERROR, rel, + f"agent_name {agent_name!r} does not match env_name {name!r}") + rjob = start.get("rjob") + if not isinstance(rjob, dict): + report.add("rjob-section", SIDE_CONFIG, SEV_ERROR, rel, "rjob: section is missing") + return [], "" + + embedded = [e for e in rjob.get("embedded_files") or [] if isinstance(e, dict)] + if not embedded and not str((start.get("container") or {}).get("runner_entrypoint", {}).get("source") or ""): + report.add("embedded-files", SIDE_CONFIG, SEV_WARN, rel, "no embedded_files declared") + adapter_targets = [] + for item in embedded: + source = str(item.get("source") or "") + target = str(item.get("target") or "") + if target.endswith("adapter.py"): + adapter_targets.append(target) + if source: + source_path = Path(source) + if not source_path.is_absolute(): + source_path = (env_dir / source_path).resolve() + if not source_path.is_file(): + report.add("embedded-source", SIDE_CONFIG, SEV_ERROR, rel, + f"embedded file source {source!r} not found") + if not adapter_targets: + report.add("adapter-embed", SIDE_CONFIG, SEV_WARN, rel, + "adapter.py is not in embedded_files", + "the runner does `from adapter import run_case`; embed it at the same path as the docker mount") + + mount_config = rjob.get("mount_config") or rjob.get("mount") or [] + if isinstance(mount_config, str): + mount_config = [mount_config] + targets = [] + results_root = str(env_params.get("results_root") or "") + has_results_mount = False + cluster_storage_reported = False + for spec in mount_config if isinstance(mount_config, list) else []: + text = str(spec) + # mount specs look like "gpfs://SRC:/container/target"; keep the part after the last ':'. + target = text.rsplit(":", 1)[-1] if ":" in text else "" + targets.append(target) + if "CLUSTER_STORAGE" in text and not cluster_storage_reported: + cluster_storage_reported = True + report.add("cluster-storage", SIDE_CONFIG, SEV_WARN, rel, + "CLUSTER_STORAGE placeholder is still present", + "replace it with a path visible to the RJob cluster before live runs") + if results_root and target.rstrip("/") == results_root.rstrip("/"): + has_results_mount = True + if results_root and mount_config and not has_results_mount: + report.add("results-mount", SIDE_CONFIG, SEV_WARN, rel, + f"env_params.results_root {results_root!r} is not an rjob mount_config target", + "results stay on the node-local filesystem otherwise") + return targets, (adapter_targets[0] if adapter_targets else "") + + +def _check_cross( + docker_env: dict[str, Any], rjob_env: dict[str, Any], + docker_cfg: dict[str, Any], rjob_cfg: dict[str, Any], report: Report +) -> None: + docker_entry, rjob_entry = _config_entry(docker_cfg), _config_entry(rjob_cfg) + if docker_entry and rjob_entry: + for key in ("dataset", "dataset_load_mode"): + if docker_entry.get(key) != rjob_entry.get(key): + report.add( + "config-drift", SIDE_CONFIG, SEV_WARN, key, + f"docker {key} {docker_entry.get(key)!r} != rjob {key} {rjob_entry.get(key)!r}", + "keep task parameters identical across the two config files", + ) + docker_params = {k: v for k, v in docker_env.items() if k != "results_root"} + rjob_params = {k: v for k, v in rjob_env.items() if k != "results_root"} + if docker_params != rjob_params: + report.add("config-drift", SIDE_CONFIG, SEV_WARN, "env_params", + "env_params (excluding results_root) differ between docker and rjob configs", + "native benchmark options must behave identically in both modes") + + +def _check_dataset( + env_params: dict[str, Any], env_dir: Path, config: dict[str, Any], report: Report +) -> list[dict[str, Any]]: + entry = _config_entry(config) + dataset = str(entry.get("dataset") or "") + if not dataset or dataset.startswith(("http", "hf://")): + return [] + path = Path(dataset) + if not path.is_absolute(): + path = env_dir / path + if not path.is_file() or path.suffix not in {".jsonl", ".ndjson", ".json", ".yaml", ".yml"}: + return [] + rows: list[dict[str, Any]] = [] + try: + if path.suffix in {".jsonl", ".ndjson"}: + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line: + rows.append(json.loads(line)) + elif path.suffix == ".json": + loaded = json.loads(path.read_text(encoding="utf-8")) + rows = loaded if isinstance(loaded, list) else [loaded] + else: + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) + rows = loaded if isinstance(loaded, list) else [loaded] + except (ValueError, yaml.YAMLError) as exc: + report.add("dataset-parse", SIDE_ENV, SEV_ERROR, path.name, f"dataset row is not valid JSON/YAML: {exc}") + return [] + bad = [i for i, row in enumerate(rows) if not isinstance(row, dict)] + if bad: + report.add("dataset-row", SIDE_ENV, SEV_ERROR, path.name, + f"rows {bad[:5]} are not JSON objects", "one dataset row = one episode and must be an object") + return [] + if rows: + report.add("dataset-info", SIDE_ENV, SEV_INFO, path.name, + f"{len(rows)} row(s); first-row keys: {', '.join(sorted(map(str, rows[0].keys())))[:160]}") + return rows + + +def _iter_path_values(row: dict[str, Any], prefix: str = ""): + for key, value in row.items(): + field = f"{prefix}{key}" + if isinstance(value, dict): + yield from _iter_path_values(value, f"{field}.") + elif isinstance(value, str) and value.startswith("/"): + yield field, value + + +def _check_dataset_paths( + rows: list[dict[str, Any]], docker_mounts: list[dict[str, str]], rjob_targets: list[str], report: Report +) -> None: + if not rows: + return + targets = [m["target"].rstrip("/") for m in docker_mounts if m["target"]] + targets += [t.rstrip("/") for t in rjob_targets if t] + for row in rows[:50]: + for field, value in _iter_path_values(row): + if not any(value == t or value.startswith(t + "/") for t in targets): + report.add( + "dataset-path", SIDE_ENV, SEV_ERROR, field, + f"row {row.get('id', rows.index(row))} field {field!r} points to {value!r}, " + "which is under no mount target", + "absolute paths in dataset rows must match a container mount target in both start files", + ) + break # one finding per row is enough to act on + + +def _check_smoke_request(env_dir: Path, report: Report) -> None: + path = env_dir / "request.smoke.json" + if not path.is_file(): + return + try: + request = json.loads(path.read_text(encoding="utf-8")) + except ValueError as exc: + report.add("smoke-request", SIDE_ENV, SEV_ERROR, path.name, f"invalid JSON: {exc}") + return + if not isinstance(request, dict): + report.add("smoke-request", SIDE_ENV, SEV_ERROR, path.name, "must be a JSON object") + return + for key in ("session_id", "model"): + if not str(request.get(key) or ""): + report.add("smoke-request", SIDE_ENV, SEV_ERROR, path.name, f"field {key!r} is empty") + dataset = (request.get("env_params") or {}).get("dataset") + if not isinstance(dataset, dict) or not dataset: + report.add("smoke-request", SIDE_ENV, SEV_ERROR, path.name, + "env_params.dataset must be a non-empty object (one case row)") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("env_dir", type=Path, help="environment directory, e.g. env/mybench") + parser.add_argument("--expect-evaluation", action="store_true", + help="require rule_evaluator.py to exist") + parser.add_argument("--json", action="store_true", help="print findings as JSON") + args = parser.parse_args(argv) + + report = validate_environment(args.env_dir, expect_evaluation=args.expect_evaluation) + if args.json: + print(json.dumps({"ok": report.ok(), "findings": [asdict(f) for f in report.findings]}, + ensure_ascii=False, indent=2)) + else: + for finding in report.findings: + if finding.severity != SEV_INFO: + print(finding.line()) + errors, warns = len(report.errors), sum(1 for f in report.findings if f.severity == SEV_WARN) + print(f"validate: {errors} error(s), {warns} warning(s) -> {'FAIL' if not report.ok() else 'PASS'}") + return 0 if report.ok() else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_environment_skill.py b/tests/test_environment_skill.py index 3a42fc2d..78660f90 100644 --- a/tests/test_environment_skill.py +++ b/tests/test_environment_skill.py @@ -1,4 +1,5 @@ """Behavioral checks for the portable onboarding templates and owned test services.""" +import asyncio import contextlib import importlib.util import io @@ -15,6 +16,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] SKILL = REPO_ROOT / "skills" / "safactory-workflows" +sys.path.insert(0, str(REPO_ROOT)) sys.path.insert(0, str(SKILL / "scripts")) from contract_smoke import run_smoke from live_smoke import run_live @@ -28,34 +30,39 @@ def setUp(self): self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) - def create(self, mode="docker", evaluation=False): - path = scaffold("mybench", mode, self.root, evaluation) + def create(self, evaluation=False): + path = scaffold("mybench", self.root, evaluation) return path, json.loads((path / "request.smoke.json").read_text()) def test_scaffold_preserves_existing_files_and_validates_name(self): path, _ = self.create() for name in ( - "runner.py", "adapter.py", "mybench_config.yaml", "mybench_start.yaml", + "runner.py", "adapter.py", "README.md", "mybench_config.yaml", "mybench_start.yaml", "mybench_config.rjob.yaml", "mybench_start.rjob.yaml", "request.smoke.json", ): self.assertTrue((path / name).exists(), name) + self.assertTrue((path / "results" / ".gitkeep").exists()) (path / "adapter.py").write_text("user code") with self.assertRaises(FileExistsError): - scaffold("mybench", "rjob", self.root, True) + scaffold("mybench", self.root, True) self.assertEqual((path / "adapter.py").read_text(), "user code") self.assertFalse((path / "rule_evaluator.py").exists()) rjob_config = (path / "mybench_config.rjob.yaml").read_text() self.assertIn("RJob cluster", rjob_config) - self.assertIn("output_root: /app/results/mybench", rjob_config) + self.assertIn("results_root: /app/results", rjob_config) docker_start = (path / "mybench_start.yaml").read_text() self.assertIn(f"{path / 'adapter.py'}", docker_start) self.assertIn(f"{path / 'datasets'}", docker_start) + self.assertIn("/tmp/safactory-mybench/datasets", docker_start) + rjob_start = (path / "mybench_start.rjob.yaml").read_text() + self.assertIn("private_machine: Group", rjob_start) + self.assertIn('HTTP_PROXY: ""', rjob_start) for name in ("../escape", "bad-name", "bad\nname"): with self.assertRaises(ValueError): - scaffold(name, "docker", self.root) + scaffold(name, self.root) def test_both_input_transports_work_without_evaluation_or_cluster(self): - path, request = self.create(mode="rjob") + path, request = self.create() self.assertFalse((path / "rule_evaluator.py").exists()) # Even a present evaluator must never be imported by integration-only execution. (path / "rule_evaluator.py").write_text("raise RuntimeError('evaluation was invoked')") @@ -164,22 +171,36 @@ def test_mock_rejects_wrong_session_route(self): def test_optional_evaluator_hook_rejects_missing_and_invalid_scores(self): path, _ = self.create(evaluation=True) + # Load evaluator/eval_types.py directly: importing the evaluator package + # pulls heavy runtime dependencies irrelevant to the hook contract. + types_spec = importlib.util.spec_from_file_location( + "evaluator.eval_types", REPO_ROOT / "evaluator" / "eval_types.py") + types_module = importlib.util.module_from_spec(types_spec) + sys.modules.setdefault("evaluator", importlib.util.module_from_spec( + importlib.util.spec_from_file_location("evaluator", REPO_ROOT / "evaluator" / "__init__.py"))) + sys.modules["evaluator"].__path__ = [str(REPO_ROOT / "evaluator")] + sys.modules["evaluator.eval_types"] = types_module + types_spec.loader.exec_module(types_module) spec = importlib.util.spec_from_file_location("fixture_evaluator", path / "rule_evaluator.py") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) request = SimpleNamespace(session_id="one", env_params={"dataset": {}}, start_result=SimpleNamespace(metrics={"score": 0.7})) + + def evaluate(**kwargs): + return asyncio.run(module.evaluate_rule(**kwargs)) + kwargs = dict(request=request, spec=SimpleNamespace(eval_id="rule"), trajectory=None) - self.assertEqual(module.evaluate_rule(**kwargs)["status"], "failed") + self.assertEqual(evaluate(**kwargs).status, "failed") module.score_metrics = lambda metrics, dataset, trajectory: (metrics["score"], metrics["score"] * 10, "fixture scale") for raw in (0.0, 0.7, 1.0): request.start_result.metrics = {"score": raw} - result = module.evaluate_rule(**kwargs) - self.assertEqual(result["status"], "succeeded") - self.assertEqual(result["normalized_score_10"], raw * 10) + result = evaluate(**kwargs) + self.assertEqual(result.status, "succeeded") + self.assertEqual(result.normalized_score_10, raw * 10) for metrics in ({}, {"score": float("nan")}, {"score": float("inf")}, {"score": 2}): request.start_result.metrics = metrics - self.assertEqual(module.evaluate_rule(**kwargs)["status"], "failed") + self.assertEqual(evaluate(**kwargs).status, "failed") def live_fixture(self): with socket.socket() as sock: diff --git a/tests/test_environment_validation.py b/tests/test_environment_validation.py new file mode 100644 index 00000000..669326d5 --- /dev/null +++ b/tests/test_environment_validation.py @@ -0,0 +1,139 @@ +"""Checks for validate_environment.py and the check_environment.py orchestration.""" +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest + +REPO_ROOT = Path(__file__).resolve().parents[1] +SKILL = REPO_ROOT / "skills" / "safactory-workflows" +sys.path.insert(0, str(SKILL / "scripts")) + +from check_environment import check as check_environment +from scaffold_environment import scaffold +from validate_environment import SEV_ERROR, validate_environment + + +def load_yaml(path: Path) -> dict: + import yaml + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def dump_yaml(path: Path, data: dict) -> None: + import yaml + path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + + +@unittest.skipUnless(importlib.util.find_spec("yaml"), "requires PyYAML for consistency checks") +class EnvironmentValidatorTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.path = scaffold("mybench", self.root) + + def codes(self, report): + return {f.code for f in report.findings} + + def test_standard_scaffold_passes_with_expected_warnings_only(self): + report = validate_environment(self.path) + self.assertTrue(report.ok(), [f.line() for f in report.errors]) + self.assertIn("cluster-storage", self.codes(report)) + self.assertNotIn("results-mount", self.codes(report)) + self.assertNotIn("dataset-path", self.codes(report)) + + def test_agent_name_mismatch_is_a_config_error(self): + start = load_yaml(self.path / "mybench_start.yaml") + start["agent_name"] = "other" + dump_yaml(self.path / "mybench_start.yaml", start) + report = validate_environment(self.path) + finding = next(f for f in report.errors if f.code == "agent-name") + self.assertEqual(finding.side, "config") + + def test_results_root_not_mounted_is_a_config_error(self): + config = load_yaml(self.path / "mybench_config.yaml") + config["environments"][0]["env_params"]["results_root"] = "/elsewhere/results" + dump_yaml(self.path / "mybench_config.yaml", config) + report = validate_environment(self.path) + finding = next(f for f in report.errors if f.code == "results-mount") + self.assertEqual(finding.side, "config") + + def test_dataset_row_absolute_path_outside_mounts_is_env_error(self): + dataset = self.path / "datasets" / "smoke.jsonl" + dataset.write_text('{"task_id": "a", "frames": "/nowhere/trajectory.mp4"}\n', encoding="utf-8") + report = validate_environment(self.path) + finding = next(f for f in report.errors if f.code == "dataset-path") + self.assertEqual(finding.side, "env") + + def test_missing_evaluator_with_expect_evaluation_is_config_error(self): + report = validate_environment(self.path, expect_evaluation=True) + finding = next(f for f in report.errors if f.code == "evaluator-missing") + self.assertEqual(finding.side, "config") + + def test_adapter_container_path_drift_between_modes_is_config_error(self): + start = load_yaml(self.path / "mybench_start.rjob.yaml") + start["rjob"]["embedded_files"][0]["target"] = "/tmp/somewhere-else/adapter.py" + dump_yaml(self.path / "mybench_start.rjob.yaml", start) + report = validate_environment(self.path) + finding = next(f for f in report.errors if f.code == "adapter-path-drift") + self.assertEqual(finding.side, "config") + + def test_hand_edited_runner_shell_is_reported_as_drift_warning(self): + runner = self.path / "runner.py" + source = runner.read_text(encoding="utf-8").replace( + 'print(f"SAFACTORY_RUNNER_DIAGNOSTIC result_artifact_write_failed: {exc}", file=sys.stderr)', + "print('custom diagnostics', file=sys.stderr)") + runner.write_text(source, encoding="utf-8") + report = validate_environment(self.path) + self.assertIn("runner-drift", self.codes(report)) + self.assertTrue(report.ok(), "drift is a warning, not an error") + + def test_broken_dataset_row_is_env_error(self): + dataset = self.path / "datasets" / "smoke.jsonl" + dataset.write_text("not json\n", encoding="utf-8") + report = validate_environment(self.path) + finding = next(f for f in report.errors if f.code == "dataset-parse") + self.assertEqual(finding.side, "env") + + +class CheckEnvironmentTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.path = scaffold("mybench", self.root) + + def test_full_pipeline_passes_on_standard_scaffold(self): + stages = check_environment(self.path) + self.assertTrue(stages["static"]["ok"]) + self.assertTrue(stages["contract"]["ok"]) + self.assertTrue(stages["contract"]["outcome"]["model_calls"] >= 1) + self.assertTrue(stages["live"]["skipped"]) + self.assertTrue(stages["summary"]["ok"]) + + def test_static_failure_short_circuits_contract_stage(self): + config = self.path / "mybench_config.yaml" + text = config.read_text(encoding="utf-8").replace( + "results_root: /workspace/Safactory/results", "results_root: /elsewhere") + config.write_text(text, encoding="utf-8") + stages = check_environment(self.path) + self.assertFalse(stages["static"]["ok"]) + self.assertTrue(stages["contract"]["skipped"]) + self.assertFalse(stages["summary"]["ok"]) + + def test_native_dependency_failure_is_labeled_with_fixture_hint(self): + (self.path / "adapter.py").write_text( + "import a_missing_native_module\n" + "def run_case(request, task, session_url):\n" + " return {}, 1\n", + encoding="utf-8", + ) + stages = check_environment(self.path) + self.assertTrue(stages["static"]["ok"]) + self.assertFalse(stages["contract"]["ok"]) + self.assertIn("fixture-adapter", stages["contract"].get("hint", "")) + + +if __name__ == "__main__": + unittest.main()