From 1f485a17763ec6d9638640275166d1f3f06f6950 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Wed, 14 Oct 2026 13:33:09 +0530 Subject: [PATCH 1/3] fix: env and importer rewards give no verdict for trusted setup failures and no credit for wrong fields --- deepgym/CHANGELOG.md | 1 + deepgym/scripts/import_bigcodebench.py | 12 ++++ deepgym/scripts/import_evalplus.py | 59 ++++++++----------- .../cve_2021_44228_log_triage/verifier.py | 44 ++++++++------ .../deepgym/envs/file_organizer/verifier.py | 14 +++-- deepgym/tests/test_cyberbench_envs.py | 34 +++++++++++ deepgym/tests/test_envs_rewards.py | 18 +++++- deepgym/tests/test_import_bigcodebench.py | 11 ++++ deepgym/tests/test_import_evalplus.py | 28 ++++++--- 9 files changed, 154 insertions(+), 67 deletions(-) diff --git a/deepgym/CHANGELOG.md b/deepgym/CHANGELOG.md index e58f1f4..915b75d 100644 --- a/deepgym/CHANGELOG.md +++ b/deepgym/CHANGELOG.md @@ -22,6 +22,7 @@ - **Traces keep every action, every grade, and the code and image that made them (schema 1.1).** A model turn that issued two actions (two commands, or a tool call and a command) dropped the whole trace from `--rows rl` exports, and ATIF split it into one agent turn per action. Step each action and pass the turn's messages with its first one only: `Trace.turns` groups them (a 1.0 trace, which used empty messages for "not recorded", keeps one turn per step), RL rows keep the trace and gain `steps` (every action with its observation text and screenshot, done flag, exit code, error, step reward and the `message_index` of the reply that produced it), and ATIF writes one agent step per turn with a tool call and a result per action, the step reward and step verifier output now in each result's `extra`. Calling `score()` twice kept only the last grade; earlier ones are now in `prior_results`, and `deepgym traces export` refuses two copies of a trace that disagree instead of keeping one. `Trace.to_trajectory()` keeps an action the session applied even when its screenshot then failed (it dropped the action and its reward), and totals step rewards the way the session does (`sum()` rounds differently from Python 3.12 on). Provenance adds `deepgym_commit`, the git commit of the deepgym checkout (null for an installed package), with `deepgym_dirty`, set if the package folder's files differ from that commit or changed after deepgym was imported, and the image the sandbox booted with the digest its provider resolved: Apple container and Docker from the container, Daytona from its snapshot (the default or a named one). Where there is none, as for an image Daytona builds, Modal, E2B or the local backend, `digest_unavailable` says why. Schema 1.0 traces still load and export. - **Core reward path: each failure lands on whoever caused it.** A verifier that prints a score and then crashes no longer keeps that score, in any verifier shape; before, Python's exit 1 for an uncaught exception read as the protocol's "failed" and the printed partial score reached the reward function. An exception hook now decides, while the script keeps running as `__main__` with its own `__file__`: before the policy's output is in play, the crash is the verifier's own failure, with no verdict (`runtime_error`, returned rather than raised); once it is (a solution process started, or the verifier opened the solution's file, or a session verifier opened a file in the agent's work directory, so running the solution with `exec` and parsing malformed policy output count), the crash came from handling it, so it is a judged 0 with `error_type='solution_error'`, a new model fault. So is a result the verifier can't normalize (NaN, not a number) after that point. Whatever the solution does (raise, exit, hang, die, return garbage, print unparseable output) is therefore its own 0 even when the verifier doesn't handle it; before, an uncaught solution error became a verifier error with no verdict, as `import_error` or `timeout` when the solution raised `ImportError` or `TimeoutError` itself, so a solution could dodge its score by breaking the verifier. Every failure from the solution process now reaches the verifier as a `deepgym_solution.SolutionError`, still of the builtin type it raised (`except ValueError` and `except TimeoutError` keep working). A solution that runs past its time limit is a judged 0 with `error_type='solution_timeout'`, a new model fault, on every path; `timeout` stays DeepGym's own timeout, with no verdict. Load test data before the solution: a verifier that fails after starting it scores 0. `LaunchError` is now a `BaseException`, so a verifier's `except Exception` can't score the machine's failure; uncaught, it ends the run with `sandbox_error` (no verdict). Before the first solution start, a preflight checks free disk and inodes and that the isolation launcher (sudo, setpriv, bwrap, sandbox-exec) starts code from a fresh directory, and a judge-side failure while preparing the run (a temp file, the loopback relay) is a `LaunchError` too. Once solution code has run, a failed relaunch is the solution's, a `SolutionError` scored 0, since it may have used up the disk or processes. The judge prints a per-run token on stdout once the policy's output is in play. A single-turn sandbox that dies after that token (its Docker or Apple container stops, or a Modal exec ends with the sandbox finished, which return what was printed) is the policy's doing: a judged 0 with `error_type='policy_sandbox_failure'`, where it was `sandbox_error`; so is a run that outruns its time limit after the token, locally or in a sandbox (`solution_timeout`), such as a verifier regex the solution's output made backtrack. One that dies before the token (creation, setup, uploads, the verifier loading its own fixtures), one whose provider returns nothing printed (Daytona, an E2B or Modal call that raises), or one Daytona evicts or times out in its queue (as its error or its refreshed sandbox record says) keeps no verdict. Wrappers report it as the new `deepgym.exceptions.SandboxDiedError` (also in `deepgym.sandboxes`), which `is_infrastructure_failure` never counts as the provider's failure. Every verdict line the harness prints starts a fresh stdout line, so a partial line the verifier left unterminated can't hide it, and a stderr the solution broke (a full disk under a file-backed stderr) can't change the verdict or the exit code. Session verifiers use the same guard, and the code they load gets the step's time limit as its deadline, as the judge does, with a few seconds' grace for the verifier to report. A session step that fails after its action may have run (the provider failing while it applies the action, takes the screenshot or runs the step verifier, not only a `VerifierError`) now ends the episode: the action stays on the trajectory with no step reward, further steps are refused, and `score()` returns no verdict for a network or provider failure (`is_infrastructure_failure`) or a failed step verifier; anything else follows from the agent's actions (its sandbox died, a write failed, its files, a folder or a planted link, blocked the step-output file) and is a judged 0 (`error_type='policy_sandbox_failure'`). A sandbox that dies under the final verifier is a judged 0 too, once the agent has acted. `run_episode` returns the result a failed step fixed instead of raising. Before, the session stayed runnable, so a retry applied a second action past `max_steps`, scored it, and the first action was never recorded. The new observation is now taken before the step verifier runs, so a failed screenshot no longer leaves a recorded reward and a stale observation behind for a retry to replay, and trace replay (`Trace.to_trajectory()`) keeps a recorded step that failed without an observation; trace capture also keeps a step interrupted after its action ran. A `write_file` blocked by the agent's own files (a folder or link in the way, found before anything is written) raises `ActionError` (rejected, the episode goes on); any other write failure is a `SandboxError`. The judge's setup refuses an existing `dgsolver` account that is root or shares its uid, gid or groups with another account (the check session setup already ran, now shared and also refusing uid or gid 0), and the judge never drops the solution to uid or gid 0; before, an image with a root-alias `dgsolver` ran the solution as root. Nothing a solution starts outlives its verifier any more: with its first solution the verifier starts a watcher process, in its own session, that kills every solution process group (a `load_solution` server, a `run_solution` script and whatever they started) once the verifier ends, however it ends (a host timeout or cancel, a crash), even when the solution hogs the interpreter in a long C call; a `run_solution` script waits to start until the watcher knows it, and a verifier's own exit waits for the watcher, so a sandbox user's sweep never reaches the next grade (on macOS, a child that detaches into a session of its own still escapes, which is why subprocesses need `allow_subprocess` there); before, such a server, or a busy child a solution started, kept running after the run. A cancelled or timed-out local run, sync or async, kills its verifier's whole process group and reaps the verifier, so neither a verifier's child nor one holding its output pipes outlives or stalls it, and a traced session's `close()` retries a sandbox delete that failed instead of returning silently. - **SWE-bench Pro / MiMo patch runner: two cases where a correct patch got no grade.** In a task whose `repo_dir` is a subdirectory of the repository, the hidden test files were saved and restored at the wrong path, so a patch that also edited a hidden test kept that edit and the test patch then failed (`test_patch_failed`). The check that runs before the patch now only looks up programs and modules that run before an earlier command could set them up (only `cd`, `export` and similar builtins may come first); before, a test command that sets up its own environment, such as `python3 -m venv /tmp/env && /tmp/env/bin/python -m unittest`, was refused (`preflight_failed`) because the venv did not exist yet. +- **Imported and built-in env rewards.** BigCodeBench checks the tests' top-level imports before the solution runs, so a module the sandbox lacks gives no verdict instead of a 0. The EvalPlus importer no longer publishes a task whose reference fails on any input; before, those cases left the suite, so a solution that failed them could score 1.0. `file_organizer` checks each README's description line, not only its heading. `cve_2021_44228_log_triage` scores only the answer's own fields (keywords elsewhere in the output earn nothing), needs the exact CVE ID and `log4shell`, counts payload indicators that name the `jndi:ldap` and `jndi:dns` lookups, and ignores remediation steps a negation in the same sentence voids ("do not patch"). ## 0.4.2 (2026-10-13) diff --git a/deepgym/scripts/import_bigcodebench.py b/deepgym/scripts/import_bigcodebench.py index dbb7dc9..ef39cb9 100644 --- a/deepgym/scripts/import_bigcodebench.py +++ b/deepgym/scripts/import_bigcodebench.py @@ -20,6 +20,7 @@ # Runs inside the isolated solution process: loads the solution, runs the # BigCodeBench unittest code against it, prints one JSON result line. _RUNNER_SOURCE = """ +import ast import importlib.util import io import json @@ -37,6 +38,17 @@ def report(score, passed, details, **extra): real_stdout.write(json.dumps({"score": score, "passed": passed, "details": details, **extra}) + "\\n") real_stdout.flush() + # The tests and their imports are trusted: check them before the solution runs, + # so a module the sandbox lacks is no verdict instead of the solution's 0. + # ponytail: only top-level imports are checked; imports inside test methods still + # fail as tests, since running other test code needs names the solution defines. + try: + tree = ast.parse(spec_data["test_code"]) + imports = [node for node in tree.body if isinstance(node, (ast.Import, ast.ImportFrom))] + exec(compile(ast.Module(body=imports, type_ignores=[]), "", "exec"), {}) + except Exception as e: + return report(0.0, False, f"Test setup error: {type(e).__name__}: {e}", error_type="test_setup_error") + try: spec = importlib.util.spec_from_file_location("solution", sys.argv[2]) mod = importlib.util.module_from_spec(spec) diff --git a/deepgym/scripts/import_evalplus.py b/deepgym/scripts/import_evalplus.py index 25a3f55..0b807c1 100644 --- a/deepgym/scripts/import_evalplus.py +++ b/deepgym/scripts/import_evalplus.py @@ -46,7 +46,7 @@ def compute_expected_outputs(prompt, canonical_solution, entry_point, inputs): Returns a list of expected outputs (one per input). If the reference solution itself errors on an input, that entry is set to the sentinel - string '__REFERENCE_ERROR__' so the verifier can skip it. + string '__REFERENCE_ERROR__', and the verifier generator rejects the task. """ ns = {} code = prompt + canonical_solution @@ -82,7 +82,14 @@ def generate_evalplus_verifier(task_id, entry_point, base_inputs, plus_inputs, For tasks with very large data (huge integers), test data is stored as compressed pickle to keep verifier files manageable. + + Raises ValueError when the reference failed on any input: dropping those + cases would publish a smaller suite than the benchmark's. """ + errors = sum(1 for e in expected_outputs if e == "__REFERENCE_ERROR__") + if errors or not expected_outputs: + raise ValueError(f"the reference failed on {errors}/{len(expected_outputs)} inputs") + import pickle as _pickle import zlib as _zlib import base64 as _base64 @@ -141,8 +148,6 @@ def generate_evalplus_verifier(task_id, entry_point, base_inputs, plus_inputs, def _compare(result, expected, atol): """Compare result to expected, handling floats with tolerance.""" - if expected == "__REFERENCE_ERROR__": - return True # skip inputs where reference itself errored if isinstance(expected, float): return isinstance(result, (int, float)) and math.isclose( result, expected, abs_tol=atol if atol else 1e-6 @@ -177,13 +182,9 @@ def verify(solution_path): base_passed = 0 plus_passed = 0 total = len(TEST_INPUTS) - skipped = 0 details = [] for i, (inp, expected) in enumerate(zip(TEST_INPUTS, EXPECTED_OUTPUTS)): - if expected == "__REFERENCE_ERROR__": - skipped += 1 - continue try: result = func(*inp) if _compare(result, expected, ATOL): @@ -202,30 +203,19 @@ def verify(solution_path): if len(details) < 5: details.append(f"Test {{i}}: {{type(e).__name__}}: {{e}}") - effective_total = total - skipped - if effective_total == 0: - raise RuntimeError("the reference errored on every input, so nothing can be scored") - score = passed / effective_total - effective_base = NUM_BASE - sum( - 1 for e in EXPECTED_OUTPUTS[:NUM_BASE] if e == "__REFERENCE_ERROR__" - ) - effective_plus = NUM_PLUS - sum( - 1 for e in EXPECTED_OUTPUTS[NUM_BASE:] if e == "__REFERENCE_ERROR__" - ) - - # The score keeps partial credit; passing means every scored test passed. + # The score keeps partial credit; passing means every test passed. return {{ - "score": score, - "passed": passed == effective_total, + "score": passed / total, + "passed": passed == total, "details": ( - f"{{passed}}/{{effective_total}} passed " - f"(base: {{base_passed}}/{{effective_base}}, " - f"plus: {{plus_passed}}/{{effective_plus}}). " + f"{{passed}}/{{total}} passed " + f"(base: {{base_passed}}/{{NUM_BASE}}, " + f"plus: {{plus_passed}}/{{NUM_PLUS}}). " + "; ".join(details) ), "reward_components": {{ - "base_score": base_passed / effective_base if effective_base > 0 else 0.0, - "plus_score": plus_passed / effective_plus if effective_plus > 0 else 0.0, + "base_score": base_passed / NUM_BASE if NUM_BASE > 0 else 0.0, + "plus_score": plus_passed / NUM_PLUS if NUM_PLUS > 0 else 0.0, }} }} @@ -269,9 +259,14 @@ def import_benchmark(name, data, output_dir): prompt, canonical, entry_point, all_inputs ) - ref_errors = sum(1 for o in expected_outputs if o == "__REFERENCE_ERROR__") - if ref_errors > 0: - errors.append(f" {safe_id}: {ref_errors}/{len(all_inputs)} reference errors") + try: + verifier_src = generate_evalplus_verifier( + task_id, entry_point, base_inputs, plus_inputs, + expected_outputs, atol + ) + except ValueError as e: + errors.append(f" {safe_id}: not imported, {e}") + continue env_dir = output_dir / safe_id env_dir.mkdir(exist_ok=True) @@ -288,10 +283,6 @@ def import_benchmark(name, data, output_dir): (env_dir / "reference_solution.py").write_text(ref_code, encoding="utf-8") # verifier.py - verifier_src = generate_evalplus_verifier( - task_id, entry_point, base_inputs, plus_inputs, - expected_outputs, atol - ) (env_dir / "verifier.py").write_text(verifier_src, encoding="utf-8") # metadata.json @@ -322,7 +313,7 @@ def import_benchmark(name, data, output_dir): ) if errors: - print(f" Reference errors in {len(errors)} tasks:") + print(f" Skipped {len(errors)} tasks with reference errors:") for e in errors[:10]: print(e) if len(errors) > 10: diff --git a/deepgym/src/deepgym/envs/cyber/cve_2021_44228_log_triage/verifier.py b/deepgym/src/deepgym/envs/cyber/cve_2021_44228_log_triage/verifier.py index fb9365e..e9250c5 100644 --- a/deepgym/src/deepgym/envs/cyber/cve_2021_44228_log_triage/verifier.py +++ b/deepgym/src/deepgym/envs/cyber/cve_2021_44228_log_triage/verifier.py @@ -4,10 +4,14 @@ import json import re import sys +from urllib.parse import unquote EXPECTED_IPS = {'203.0.113.42', '198.51.100.77'} -REQUIRED_TERMS = {'cve-2021-44228', 'log4shell'} -REQUIRED_PAYLOAD_TERMS = {'jndi', 'ldap', 'dns'} +EXPECTED_PAYLOADS = {'jndi:ldap', 'jndi:dns'} # the two lookups in the log +ACTION_TERMS = {'patch', 'block', 'search', 'logs'} +# ponytail: keyword negation check, not language understanding. A negation voids +# the action terms after it in the same sentence; use a judge model if this is gamed. +_NEGATION = re.compile(r"\b(?:not|no|never|avoid|without)\b|n['’]t\b") def _read_output(solution_path): @@ -17,19 +21,19 @@ def _read_output(solution_path): text = re.sub(r'\s*```$', '', text) try: data = json.loads(text) - return (data if isinstance(data, dict) else {}), text + return data if isinstance(data, dict) else {} except json.JSONDecodeError: match = re.search(r'\{.*\}', text, re.DOTALL) if match: try: - return json.loads(match.group(0)), text + return json.loads(match.group(0)) except json.JSONDecodeError: pass try: parsed = ast.literal_eval(text) except Exception: parsed = None - return parsed if isinstance(parsed, dict) else {}, text + return parsed if isinstance(parsed, dict) else {} def _string_set(value): @@ -40,12 +44,20 @@ def _string_set(value): return set() +def _affirmed_actions(text): + found = set() + for sentence in re.split(r'[.;!?](?:\s|$)|\n', text): + negation = _NEGATION.search(sentence) + kept = sentence[: negation.start()] if negation else sentence + found |= {term for term in ACTION_TERMS if term in kept} + return found + + def verify(solution_path, test_cases_path=None): - data, raw = _read_output(solution_path) - raw_lower = raw.lower() + # Only the answer's own fields count: keywords elsewhere in the output earn nothing. + data = _read_output(solution_path) ips = _string_set(data.get('source_ips')) - indicators = ' '.join(_string_set(data.get('payload_indicators'))).lower() - cve_text = f'{data.get("cve", "")} {data.get("vulnerability", "")}'.lower() + indicators = unquote(' '.join(_string_set(data.get('payload_indicators')))).lower() action_text = str(data.get('recommended_action', '')).lower() cases = [] @@ -56,23 +68,19 @@ def verify(solution_path, test_cases_path=None): score += 0.4 * ip_score cases.append({'id': 'source_ips', 'passed': ip_score == 1.0, 'score': ip_score}) - cve_score = ( - 1.0 if all(term in cve_text or term in raw_lower for term in REQUIRED_TERMS) else 0.0 - ) + cve_ok = str(data.get('cve', '')).strip().lower() == 'cve-2021-44228' + cve_score = 1.0 if cve_ok and 'log4shell' in str(data.get('vulnerability', '')).lower() else 0.0 score += 0.2 * cve_score cases.append({'id': 'cve_identification', 'passed': cve_score == 1.0, 'score': cve_score}) - payload_hits = sum( - 1 for term in REQUIRED_PAYLOAD_TERMS if term in indicators or term in raw_lower - ) - payload_score = payload_hits / len(REQUIRED_PAYLOAD_TERMS) + payload_hits = sum(1 for payload in EXPECTED_PAYLOADS if payload in indicators) + payload_score = payload_hits / len(EXPECTED_PAYLOADS) score += 0.25 * payload_score cases.append( {'id': 'payload_indicators', 'passed': payload_score == 1.0, 'score': payload_score} ) - action_terms = {'patch', 'block', 'search', 'logs'} - action_score = min(1.0, sum(1 for term in action_terms if term in action_text) / 3) + action_score = min(1.0, len(_affirmed_actions(action_text)) / 3) score += 0.15 * action_score cases.append( {'id': 'recommended_action', 'passed': action_score >= 0.66, 'score': action_score} diff --git a/deepgym/src/deepgym/envs/file_organizer/verifier.py b/deepgym/src/deepgym/envs/file_organizer/verifier.py index e2009f1..d051c20 100644 --- a/deepgym/src/deepgym/envs/file_organizer/verifier.py +++ b/deepgym/src/deepgym/envs/file_organizer/verifier.py @@ -34,18 +34,20 @@ def verify(solution_path, test_cases_path=None): dirpath = os.path.join(project_dir, dirname) checks[f'{dirname}_dir_exists'] = os.path.isdir(dirpath) - # Check README files exist and have correct content. + # Check README files exist and hold the requested lines (blank lines and + # surrounding whitespace aside). expected_content = { - 'src/README.md': '# Source Code', - 'tests/README.md': '# Tests', - 'docs/README.md': '# Documentation', + 'src/README.md': ['# Source Code', 'Main application source files.'], + 'tests/README.md': ['# Tests', 'Unit and integration tests.'], + 'docs/README.md': ['# Documentation', 'Project documentation and guides.'], } - for filepath, expected_heading in expected_content.items(): + for filepath, expected_lines in expected_content.items(): try: content = read_output(os.path.join(project_dir, filepath), work_dir) checks[f'{filepath}_exists'] = True - checks[f'{filepath}_content'] = expected_heading in content + lines = [line.strip() for line in content.splitlines() if line.strip()] + checks[f'{filepath}_content'] = lines == expected_lines except (OSError, ValueError): # missing, a symlink, or not UTF-8 text checks[f'{filepath}_exists'] = False checks[f'{filepath}_content'] = False diff --git a/deepgym/tests/test_cyberbench_envs.py b/deepgym/tests/test_cyberbench_envs.py index e0d1f1b..2d465ae 100644 --- a/deepgym/tests/test_cyberbench_envs.py +++ b/deepgym/tests/test_cyberbench_envs.py @@ -69,3 +69,37 @@ def test_log_triage_penalizes_every_ip_that_did_not_probe(dg: DeepGym, extra_ip: assert correct.score == pytest.approx(1.0) assert wrong.passed is False assert wrong.score < correct.score + + +def test_log_triage_scores_the_answer_fields_not_keywords_elsewhere(dg: DeepGym) -> None: + """Wrong fields padded with the right keywords used to earn 1.0.""" + env = load_environment('cyber/cve_2021_44228_log_triage') + padded = { + 'cve': 'CVE-2021-45046', + 'vulnerability': 'SQL injection', + 'source_ips': _TRIAGE['source_ips'], + 'payload_indicators': [], + 'notes': 'CVE-2021-44228 log4shell jndi ldap dns', + 'recommended_action': 'Do not patch, block or search the logs.', + } + + result = dg.run(env, json.dumps(padded)) + + assert result.passed is False + assert result.score == pytest.approx(0.4) # the IPs, nothing else + + +@pytest.mark.parametrize( + ('action', 'score'), + [ + ('Do not patch, block or search the logs.', 0.85), + ("Don't patch Log4j; there is no need to block or search logs.", 0.85), + ('Patch Log4j, block both sources and search the logs. Do not expose JNDI.', 1.0), + ], +) +def test_log_triage_negated_actions_earn_nothing(dg: DeepGym, action: str, score: float) -> None: + env = load_environment('cyber/cve_2021_44228_log_triage') + + result = dg.run(env, json.dumps({**_TRIAGE, 'recommended_action': action})) + + assert result.score == pytest.approx(score) diff --git a/deepgym/tests/test_envs_rewards.py b/deepgym/tests/test_envs_rewards.py index 00c4030..bf6d6e0 100644 --- a/deepgym/tests/test_envs_rewards.py +++ b/deepgym/tests/test_envs_rewards.py @@ -87,9 +87,13 @@ def test_string_manipulation_scores_split_and_slice(dg: DeepGym, string_env: Env from pathlib import Path root = Path(os.environ['PROJECT_DIR']) -for name, heading in [('src', '# Source Code'), ('tests', '# Tests'), ('docs', '# Documentation')]: +for name, readme in [ + ('src', '# Source Code\\nMain application source files.'), + ('tests', '# Tests\\nUnit and integration tests.'), + ('docs', '# Documentation\\nProject documentation and guides.'), +]: (root / name).mkdir(parents=True) - (root / name / 'README.md').write_text(heading + '\\n') + (root / name / 'README.md').write_text(readme + '\\n') print('done') """ @@ -107,6 +111,16 @@ def test_file_organizer_prompt_names_the_checked_directory(dg: DeepGym) -> None: assert wrong.passed is False +def test_file_organizer_readmes_need_their_descriptions(dg: DeepGym) -> None: + """Heading-only READMEs used to earn 1.0; the task asks for a description line too.""" + headings_only = _ORGANIZE.replace('write_text(readme', 'write_text(readme.splitlines()[0]') + + result = dg.run(load_environment('file_organizer'), headings_only) + + assert result.passed is False + assert result.score < 1.0 + + _FILTER = """ import csv diff --git a/deepgym/tests/test_import_bigcodebench.py b/deepgym/tests/test_import_bigcodebench.py index 665c41e..53293b0 100644 --- a/deepgym/tests/test_import_bigcodebench.py +++ b/deepgym/tests/test_import_bigcodebench.py @@ -121,3 +121,14 @@ def test_hidden_test_cases_score_zero(env: Environment, dg: DeepGym) -> None: assert result.score == 0.0 assert result.passed is False + + +def test_missing_test_dependency_has_no_verdict(dg: DeepGym) -> None: + """A module the tests import but the sandbox lacks used to score a correct solution 0.""" + tests = 'import deepgym_no_such_module\n' + _TESTS + env = Environment(task='t', verifier_code=generate_verifier('t', 'task_func', tests, [])) + + result = dg.run(env, 'def task_func(x):\n return x ** 2\n') + + assert result.exit_code == 2 + assert result.has_verdict is False diff --git a/deepgym/tests/test_import_evalplus.py b/deepgym/tests/test_import_evalplus.py index 3485403..37cde89 100644 --- a/deepgym/tests/test_import_evalplus.py +++ b/deepgym/tests/test_import_evalplus.py @@ -17,6 +17,7 @@ _has_large_int, compute_expected_outputs, generate_evalplus_verifier, + import_benchmark, ) @@ -63,13 +64,26 @@ def test_unsendable_entry_point_is_a_scored_failure(self) -> None: assert result.score == 0.0 assert result.has_verdict is True - def test_reference_errors_on_every_input_is_a_verifier_error(self) -> None: - env = _verifier_env('f', [[1], [2]], ['__REFERENCE_ERROR__'] * 2) - - result = DeepGym(mode='local').run(env, 'def f(x):\n return x\n') - - assert result.exit_code == 2 - assert result.has_verdict is False + @pytest.mark.parametrize('outputs', [[1, '__REFERENCE_ERROR__'], ['__REFERENCE_ERROR__'] * 2]) + def test_incomplete_reference_outputs_are_rejected(self, outputs: list) -> None: + with pytest.raises(ValueError, match='reference'): + _verifier_env('f', [[1], [2]], outputs) + + def test_task_with_a_reference_error_is_not_published(self, tmp_path: Path) -> None: + """A reference failing on input 1 used to drop that case, so answering 0 scored 1.0.""" + canonical = 'def f(x):\n if x == 1:\n import no_such_module\n return x\n' + problem = { + 'entry_point': 'f', + 'prompt': '', + 'canonical_solution': canonical, + 'base_input': [[0], [1]], + 'plus_input': [], + } + + registry = import_benchmark('humaneval_plus', {'T/0': problem}, tmp_path) + + assert registry == [] + assert not (tmp_path / 'T_0').exists() class TestEvalPlusLargeIntHandling: From 54335837b19490e6350bcca6a1d81eb7a666fc92 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Wed, 14 Oct 2026 13:37:45 +0530 Subject: [PATCH 2/3] fix: void negated remediation clauses in log triage and remove stale rejected evalplus tasks --- deepgym/CHANGELOG.md | 2 +- deepgym/scripts/import_evalplus.py | 5 ++++- .../envs/cyber/cve_2021_44228_log_triage/verifier.py | 12 ++++++------ deepgym/tests/test_cyberbench_envs.py | 2 ++ deepgym/tests/test_import_evalplus.py | 3 +++ 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/deepgym/CHANGELOG.md b/deepgym/CHANGELOG.md index 915b75d..0843e53 100644 --- a/deepgym/CHANGELOG.md +++ b/deepgym/CHANGELOG.md @@ -22,7 +22,7 @@ - **Traces keep every action, every grade, and the code and image that made them (schema 1.1).** A model turn that issued two actions (two commands, or a tool call and a command) dropped the whole trace from `--rows rl` exports, and ATIF split it into one agent turn per action. Step each action and pass the turn's messages with its first one only: `Trace.turns` groups them (a 1.0 trace, which used empty messages for "not recorded", keeps one turn per step), RL rows keep the trace and gain `steps` (every action with its observation text and screenshot, done flag, exit code, error, step reward and the `message_index` of the reply that produced it), and ATIF writes one agent step per turn with a tool call and a result per action, the step reward and step verifier output now in each result's `extra`. Calling `score()` twice kept only the last grade; earlier ones are now in `prior_results`, and `deepgym traces export` refuses two copies of a trace that disagree instead of keeping one. `Trace.to_trajectory()` keeps an action the session applied even when its screenshot then failed (it dropped the action and its reward), and totals step rewards the way the session does (`sum()` rounds differently from Python 3.12 on). Provenance adds `deepgym_commit`, the git commit of the deepgym checkout (null for an installed package), with `deepgym_dirty`, set if the package folder's files differ from that commit or changed after deepgym was imported, and the image the sandbox booted with the digest its provider resolved: Apple container and Docker from the container, Daytona from its snapshot (the default or a named one). Where there is none, as for an image Daytona builds, Modal, E2B or the local backend, `digest_unavailable` says why. Schema 1.0 traces still load and export. - **Core reward path: each failure lands on whoever caused it.** A verifier that prints a score and then crashes no longer keeps that score, in any verifier shape; before, Python's exit 1 for an uncaught exception read as the protocol's "failed" and the printed partial score reached the reward function. An exception hook now decides, while the script keeps running as `__main__` with its own `__file__`: before the policy's output is in play, the crash is the verifier's own failure, with no verdict (`runtime_error`, returned rather than raised); once it is (a solution process started, or the verifier opened the solution's file, or a session verifier opened a file in the agent's work directory, so running the solution with `exec` and parsing malformed policy output count), the crash came from handling it, so it is a judged 0 with `error_type='solution_error'`, a new model fault. So is a result the verifier can't normalize (NaN, not a number) after that point. Whatever the solution does (raise, exit, hang, die, return garbage, print unparseable output) is therefore its own 0 even when the verifier doesn't handle it; before, an uncaught solution error became a verifier error with no verdict, as `import_error` or `timeout` when the solution raised `ImportError` or `TimeoutError` itself, so a solution could dodge its score by breaking the verifier. Every failure from the solution process now reaches the verifier as a `deepgym_solution.SolutionError`, still of the builtin type it raised (`except ValueError` and `except TimeoutError` keep working). A solution that runs past its time limit is a judged 0 with `error_type='solution_timeout'`, a new model fault, on every path; `timeout` stays DeepGym's own timeout, with no verdict. Load test data before the solution: a verifier that fails after starting it scores 0. `LaunchError` is now a `BaseException`, so a verifier's `except Exception` can't score the machine's failure; uncaught, it ends the run with `sandbox_error` (no verdict). Before the first solution start, a preflight checks free disk and inodes and that the isolation launcher (sudo, setpriv, bwrap, sandbox-exec) starts code from a fresh directory, and a judge-side failure while preparing the run (a temp file, the loopback relay) is a `LaunchError` too. Once solution code has run, a failed relaunch is the solution's, a `SolutionError` scored 0, since it may have used up the disk or processes. The judge prints a per-run token on stdout once the policy's output is in play. A single-turn sandbox that dies after that token (its Docker or Apple container stops, or a Modal exec ends with the sandbox finished, which return what was printed) is the policy's doing: a judged 0 with `error_type='policy_sandbox_failure'`, where it was `sandbox_error`; so is a run that outruns its time limit after the token, locally or in a sandbox (`solution_timeout`), such as a verifier regex the solution's output made backtrack. One that dies before the token (creation, setup, uploads, the verifier loading its own fixtures), one whose provider returns nothing printed (Daytona, an E2B or Modal call that raises), or one Daytona evicts or times out in its queue (as its error or its refreshed sandbox record says) keeps no verdict. Wrappers report it as the new `deepgym.exceptions.SandboxDiedError` (also in `deepgym.sandboxes`), which `is_infrastructure_failure` never counts as the provider's failure. Every verdict line the harness prints starts a fresh stdout line, so a partial line the verifier left unterminated can't hide it, and a stderr the solution broke (a full disk under a file-backed stderr) can't change the verdict or the exit code. Session verifiers use the same guard, and the code they load gets the step's time limit as its deadline, as the judge does, with a few seconds' grace for the verifier to report. A session step that fails after its action may have run (the provider failing while it applies the action, takes the screenshot or runs the step verifier, not only a `VerifierError`) now ends the episode: the action stays on the trajectory with no step reward, further steps are refused, and `score()` returns no verdict for a network or provider failure (`is_infrastructure_failure`) or a failed step verifier; anything else follows from the agent's actions (its sandbox died, a write failed, its files, a folder or a planted link, blocked the step-output file) and is a judged 0 (`error_type='policy_sandbox_failure'`). A sandbox that dies under the final verifier is a judged 0 too, once the agent has acted. `run_episode` returns the result a failed step fixed instead of raising. Before, the session stayed runnable, so a retry applied a second action past `max_steps`, scored it, and the first action was never recorded. The new observation is now taken before the step verifier runs, so a failed screenshot no longer leaves a recorded reward and a stale observation behind for a retry to replay, and trace replay (`Trace.to_trajectory()`) keeps a recorded step that failed without an observation; trace capture also keeps a step interrupted after its action ran. A `write_file` blocked by the agent's own files (a folder or link in the way, found before anything is written) raises `ActionError` (rejected, the episode goes on); any other write failure is a `SandboxError`. The judge's setup refuses an existing `dgsolver` account that is root or shares its uid, gid or groups with another account (the check session setup already ran, now shared and also refusing uid or gid 0), and the judge never drops the solution to uid or gid 0; before, an image with a root-alias `dgsolver` ran the solution as root. Nothing a solution starts outlives its verifier any more: with its first solution the verifier starts a watcher process, in its own session, that kills every solution process group (a `load_solution` server, a `run_solution` script and whatever they started) once the verifier ends, however it ends (a host timeout or cancel, a crash), even when the solution hogs the interpreter in a long C call; a `run_solution` script waits to start until the watcher knows it, and a verifier's own exit waits for the watcher, so a sandbox user's sweep never reaches the next grade (on macOS, a child that detaches into a session of its own still escapes, which is why subprocesses need `allow_subprocess` there); before, such a server, or a busy child a solution started, kept running after the run. A cancelled or timed-out local run, sync or async, kills its verifier's whole process group and reaps the verifier, so neither a verifier's child nor one holding its output pipes outlives or stalls it, and a traced session's `close()` retries a sandbox delete that failed instead of returning silently. - **SWE-bench Pro / MiMo patch runner: two cases where a correct patch got no grade.** In a task whose `repo_dir` is a subdirectory of the repository, the hidden test files were saved and restored at the wrong path, so a patch that also edited a hidden test kept that edit and the test patch then failed (`test_patch_failed`). The check that runs before the patch now only looks up programs and modules that run before an earlier command could set them up (only `cd`, `export` and similar builtins may come first); before, a test command that sets up its own environment, such as `python3 -m venv /tmp/env && /tmp/env/bin/python -m unittest`, was refused (`preflight_failed`) because the venv did not exist yet. -- **Imported and built-in env rewards.** BigCodeBench checks the tests' top-level imports before the solution runs, so a module the sandbox lacks gives no verdict instead of a 0. The EvalPlus importer no longer publishes a task whose reference fails on any input; before, those cases left the suite, so a solution that failed them could score 1.0. `file_organizer` checks each README's description line, not only its heading. `cve_2021_44228_log_triage` scores only the answer's own fields (keywords elsewhere in the output earn nothing), needs the exact CVE ID and `log4shell`, counts payload indicators that name the `jndi:ldap` and `jndi:dns` lookups, and ignores remediation steps a negation in the same sentence voids ("do not patch"). +- **Imported and built-in env rewards.** BigCodeBench checks the tests' top-level imports before the solution runs, so a module the sandbox lacks gives no verdict instead of a 0. The EvalPlus importer no longer publishes a task whose reference fails on any input, and removes an earlier import's copy of it; before, those cases left the suite, so a solution that failed them could score 1.0. `file_organizer` checks each README's description line, not only its heading. `cve_2021_44228_log_triage` scores only the answer's own fields (keywords elsewhere in the output earn nothing), needs the exact CVE ID and `log4shell`, counts payload indicators that name the `jndi:ldap` and `jndi:dns` lookups, and gives no credit for remediation steps in a clause that holds a negation ("do not patch", "patching is not needed"). ## 0.4.2 (2026-10-13) diff --git a/deepgym/scripts/import_evalplus.py b/deepgym/scripts/import_evalplus.py index 0b807c1..fad5802 100644 --- a/deepgym/scripts/import_evalplus.py +++ b/deepgym/scripts/import_evalplus.py @@ -10,6 +10,7 @@ import copy import json +import shutil import sys import traceback from pathlib import Path @@ -259,6 +260,7 @@ def import_benchmark(name, data, output_dir): prompt, canonical, entry_point, all_inputs ) + env_dir = output_dir / safe_id try: verifier_src = generate_evalplus_verifier( task_id, entry_point, base_inputs, plus_inputs, @@ -266,9 +268,10 @@ def import_benchmark(name, data, output_dir): ) except ValueError as e: errors.append(f" {safe_id}: not imported, {e}") + # An earlier import's copy would still load by name. + shutil.rmtree(env_dir, ignore_errors=True) continue - env_dir = output_dir / safe_id env_dir.mkdir(exist_ok=True) # task.md diff --git a/deepgym/src/deepgym/envs/cyber/cve_2021_44228_log_triage/verifier.py b/deepgym/src/deepgym/envs/cyber/cve_2021_44228_log_triage/verifier.py index e9250c5..d4ebf68 100644 --- a/deepgym/src/deepgym/envs/cyber/cve_2021_44228_log_triage/verifier.py +++ b/deepgym/src/deepgym/envs/cyber/cve_2021_44228_log_triage/verifier.py @@ -9,8 +9,9 @@ EXPECTED_IPS = {'203.0.113.42', '198.51.100.77'} EXPECTED_PAYLOADS = {'jndi:ldap', 'jndi:dns'} # the two lookups in the log ACTION_TERMS = {'patch', 'block', 'search', 'logs'} -# ponytail: keyword negation check, not language understanding. A negation voids -# the action terms after it in the same sentence; use a judge model if this is gamed. +# ponytail: keyword check, not language understanding. A clause (split at . ; : ! ?) +# with any negation earns no action terms, so "patch, but don't wait" loses credit +# rather than "patching is not needed" gaining it; use a judge model to do better. _NEGATION = re.compile(r"\b(?:not|no|never|avoid|without)\b|n['’]t\b") @@ -46,10 +47,9 @@ def _string_set(value): def _affirmed_actions(text): found = set() - for sentence in re.split(r'[.;!?](?:\s|$)|\n', text): - negation = _NEGATION.search(sentence) - kept = sentence[: negation.start()] if negation else sentence - found |= {term for term in ACTION_TERMS if term in kept} + for clause in re.split(r'[.;:!?](?:\s|$)|\n', text): + if not _NEGATION.search(clause): + found |= {term for term in ACTION_TERMS if term in clause} return found diff --git a/deepgym/tests/test_cyberbench_envs.py b/deepgym/tests/test_cyberbench_envs.py index 2d465ae..799b3c3 100644 --- a/deepgym/tests/test_cyberbench_envs.py +++ b/deepgym/tests/test_cyberbench_envs.py @@ -94,7 +94,9 @@ def test_log_triage_scores_the_answer_fields_not_keywords_elsewhere(dg: DeepGym) [ ('Do not patch, block or search the logs.', 0.85), ("Don't patch Log4j; there is no need to block or search logs.", 0.85), + ('Patching Log4j, blocking the sources and searching logs are not needed.', 0.85), ('Patch Log4j, block both sources and search the logs. Do not expose JNDI.', 1.0), + ('Do not delay: patch Log4j, block both sources and search the logs.', 1.0), ], ) def test_log_triage_negated_actions_earn_nothing(dg: DeepGym, action: str, score: float) -> None: diff --git a/deepgym/tests/test_import_evalplus.py b/deepgym/tests/test_import_evalplus.py index 37cde89..b1bf878 100644 --- a/deepgym/tests/test_import_evalplus.py +++ b/deepgym/tests/test_import_evalplus.py @@ -80,6 +80,9 @@ def test_task_with_a_reference_error_is_not_published(self, tmp_path: Path) -> N 'plus_input': [], } + (tmp_path / 'T_0').mkdir() # an earlier import's copy + (tmp_path / 'T_0' / 'verifier.py').write_text('') + registry = import_benchmark('humaneval_plus', {'T/0': problem}, tmp_path) assert registry == [] From fcaffc622371e959bce08c391014c9713cc2b945 Mon Sep 17 00:00:00 2001 From: Abhishek Gahlot Date: Wed, 14 Oct 2026 13:42:15 +0530 Subject: [PATCH 3/3] fix: unlink symlinked stale evalplus tasks and surface removal failures --- deepgym/scripts/import_evalplus.py | 5 ++++- deepgym/tests/test_import_evalplus.py | 20 ++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/deepgym/scripts/import_evalplus.py b/deepgym/scripts/import_evalplus.py index fad5802..2bdfaad 100644 --- a/deepgym/scripts/import_evalplus.py +++ b/deepgym/scripts/import_evalplus.py @@ -269,7 +269,10 @@ def import_benchmark(name, data, output_dir): except ValueError as e: errors.append(f" {safe_id}: not imported, {e}") # An earlier import's copy would still load by name. - shutil.rmtree(env_dir, ignore_errors=True) + if env_dir.is_symlink(): + env_dir.unlink() + elif env_dir.exists(): + shutil.rmtree(env_dir) continue env_dir.mkdir(exist_ok=True) diff --git a/deepgym/tests/test_import_evalplus.py b/deepgym/tests/test_import_evalplus.py index b1bf878..c74e7c7 100644 --- a/deepgym/tests/test_import_evalplus.py +++ b/deepgym/tests/test_import_evalplus.py @@ -69,7 +69,10 @@ def test_incomplete_reference_outputs_are_rejected(self, outputs: list) -> None: with pytest.raises(ValueError, match='reference'): _verifier_env('f', [[1], [2]], outputs) - def test_task_with_a_reference_error_is_not_published(self, tmp_path: Path) -> None: + @pytest.mark.parametrize('linked', [False, True]) + def test_task_with_a_reference_error_is_not_published( + self, tmp_path: Path, linked: bool + ) -> None: """A reference failing on input 1 used to drop that case, so answering 0 scored 1.0.""" canonical = 'def f(x):\n if x == 1:\n import no_such_module\n return x\n' problem = { @@ -79,14 +82,19 @@ def test_task_with_a_reference_error_is_not_published(self, tmp_path: Path) -> N 'base_input': [[0], [1]], 'plus_input': [], } + out = tmp_path / 'out' + old = tmp_path / 'old' if linked else out / 'T_0' # an earlier import's copy + old.mkdir(parents=True) + (old / 'verifier.py').write_text('') + if linked: + out.mkdir() + (out / 'T_0').symlink_to(old) - (tmp_path / 'T_0').mkdir() # an earlier import's copy - (tmp_path / 'T_0' / 'verifier.py').write_text('') - - registry = import_benchmark('humaneval_plus', {'T/0': problem}, tmp_path) + registry = import_benchmark('humaneval_plus', {'T/0': problem}, out) assert registry == [] - assert not (tmp_path / 'T_0').exists() + assert not (out / 'T_0').exists() + assert not (out / 'T_0').is_symlink() class TestEvalPlusLargeIntHandling: