From 51e9b22f379a8c0590174b721c7adf70574d0d70 Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 7 Aug 2026 09:31:50 +0800 Subject: [PATCH 1/6] feat: add wave 2 code benchmark adapters --- benchmark/wave2-code/bigcodebench.md | 112 +++++++++++++ benchmark/wave2-code/contract.md | 56 +++++++ benchmark/wave2-code/matrix.tsv | 14 ++ benchmark/wave2-code/mega.md | 75 +++++++++ benchmark/wave2-code/ojbench.md | 64 ++++++++ benchmark/wave2-code/spider.md | 64 ++++++++ .../wave2-code/validate_bigcodebench_arm64.py | 77 +++++++++ benchmark/wave2-code/vita_bench.md | 82 ++++++++++ pyproject.toml | 1 + src/openbench/_registry.py | 2 + src/openbench/config.py | 16 ++ src/openbench/datasets/bigcodebench.py | 128 +++++++++++++++ src/openbench/datasets/livebench.py | 146 +++++++++++++++++ .../evals/bigcodebench/Dockerfile.arm64 | 77 +++++++++ src/openbench/evals/bigcodebench/__init__.py | 5 + .../evals/bigcodebench/bigcodebench.py | 74 +++++++++ .../evals/bigcodebench/compose.arm64.yaml | 24 +++ src/openbench/evals/bigcodebench/compose.yaml | 19 +++ src/openbench/evals/livebench/__init__.py | 5 + src/openbench/evals/livebench/livebench.py | 32 ++++ src/openbench/scorers/bigcodebench.py | 140 ++++++++++++++++ src/openbench/scorers/bigcodebench_runner.py | 57 +++++++ src/openbench/scorers/livecodebench.py | 95 +++++++++++ tests/test_bigcodebench.py | 151 ++++++++++++++++++ tests/test_livebench.py | 111 +++++++++++++ tests/test_registry.py | 4 + 26 files changed, 1631 insertions(+) create mode 100644 benchmark/wave2-code/bigcodebench.md create mode 100644 benchmark/wave2-code/contract.md create mode 100644 benchmark/wave2-code/matrix.tsv create mode 100644 benchmark/wave2-code/mega.md create mode 100644 benchmark/wave2-code/ojbench.md create mode 100644 benchmark/wave2-code/spider.md create mode 100644 benchmark/wave2-code/validate_bigcodebench_arm64.py create mode 100644 benchmark/wave2-code/vita_bench.md create mode 100644 src/openbench/datasets/bigcodebench.py create mode 100644 src/openbench/datasets/livebench.py create mode 100644 src/openbench/evals/bigcodebench/Dockerfile.arm64 create mode 100644 src/openbench/evals/bigcodebench/__init__.py create mode 100644 src/openbench/evals/bigcodebench/bigcodebench.py create mode 100644 src/openbench/evals/bigcodebench/compose.arm64.yaml create mode 100644 src/openbench/evals/bigcodebench/compose.yaml create mode 100644 src/openbench/evals/livebench/__init__.py create mode 100644 src/openbench/evals/livebench/livebench.py create mode 100644 src/openbench/scorers/bigcodebench.py create mode 100644 src/openbench/scorers/bigcodebench_runner.py create mode 100644 tests/test_bigcodebench.py create mode 100644 tests/test_livebench.py diff --git a/benchmark/wave2-code/bigcodebench.md b/benchmark/wave2-code/bigcodebench.md new file mode 100644 index 00000000..e6674748 --- /dev/null +++ b/benchmark/wave2-code/bigcodebench.md @@ -0,0 +1,112 @@ +# BigCodeBench audit and integration note + +## Decision + +BigCodeBench now has an OpenBench adapter and an arm64 source-equivalent scorer +image that passes a local Docker runner smoke on Apple Silicon. The pinned +official evaluator image is still `linux/amd64` only, so official-image parity +remains blocked until it is smoke-tested on a usable native `linux/amd64` Docker +host. + +## Canonical sources + +- Repository: `https://github.com/bigcode-project/bigcodebench.git` +- Repository commit inspected: `09dd993f46c3fbf3a799465bb96d524edcb0b199` +- Official package/data version used by upstream loader: `v0.1.4` +- Full dataset: `bigcode/bigcodebench`, revision + `b74c0d0bf70d2c0bc459be537895cca163007f1a`, 1,140 tasks. +- Hard dataset: `bigcode/bigcodebench-hard`, revision + `298d2cc7b96612e15e47313c3603ee124cee0c1f`, 148 tasks. +- Official evaluator image: `bigcodebench/bigcodebench-evaluate` manifest + `sha256:1327bddf60be9bc241648c59e6060cac4ca50248a0588ab735cd0200b17cc8c2` + for `linux/amd64`. +- License: MIT. + +## Implemented OpenBench surface + +- Registry ID: `bigcodebench`. +- Parameters: `split="complete" | "instruct"`, `subset="full" | "hard"`, + `runtime="auto" | "official" | "arm64"`, optional `limit`, `epochs`, and + `total_timeout`. +- Dataset loader: immutable Hugging Face revisions, with hidden execution fields + resolved only at score time. +- Prompting: follows BigCodeBench's OpenAI/API chat backend wrapper by applying + the official instruction prefix and using the official complete-vs-instruct + prompt field. +- Scoring: sandbox runner uses BigCodeBench's own `sanitize`, `trusted_check`, + and `untrusted_check`. Canonical solution timing is computed per task and used + to calibrate the generated solution timeout, matching the upstream evaluator. +- Docker: `runtime="official"` pins the official `linux/amd64` evaluator image; + `runtime="arm64"` builds OpenBench's source-equivalent scorer image from the + pinned upstream source commit for Apple Silicon; `runtime="auto"` selects + arm64 on arm64/aarch64 hosts. Both compose files disable network, drop + capabilities, set no-new-privileges, and use tmpfs work directories. + +## Local validation + +Passed locally: + +```text +source .venv/bin/activate && ruff check src/openbench/datasets/bigcodebench.py src/openbench/scorers/bigcodebench.py src/openbench/scorers/bigcodebench_runner.py src/openbench/evals/bigcodebench tests/test_bigcodebench.py tests/test_registry.py +All checks passed! + +source .venv/bin/activate && pytest tests/test_bigcodebench.py tests/test_registry.py +21 passed + +source .venv/bin/activate && mypy src/openbench/datasets/bigcodebench.py src/openbench/scorers/bigcodebench.py src/openbench/scorers/bigcodebench_runner.py src/openbench/evals/bigcodebench tests/test_bigcodebench.py +Success: no issues found in 6 source files +``` + +Also passed dataset construction smoke for `bigcodebench(limit=1)` and the hard +subset loader without printing benchmark prompts. + +Arm64 source-equivalent smoke passed locally: + +```text +docker build --platform linux/arm64 -f src/openbench/evals/bigcodebench/Dockerfile.arm64 -t openbench-bigcodebench-arm64:dev src/openbench/evals/bigcodebench + +docker run --rm --network none --read-only --tmpfs /tmp:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=4294967296 --cap-drop ALL --security-opt no-new-privileges:true --pids-limit 256 --entrypoint python3 -v "$smoke_dir:/app:rw" openbench-bigcodebench-arm64:dev /app/runner.py /app/payload.json +BigCodeBench/13 +{"passed": true, "status": "pass", ...} + +docker compose -f src/openbench/evals/bigcodebench/compose.arm64.yaml -p openbench-bcb-arm64-smoke up -d --build +docker compose -f src/openbench/evals/bigcodebench/compose.arm64.yaml -p openbench-bcb-arm64-smoke exec -T default python3 -c "from bigcodebench.eval import untrusted_check; from bigcodebench.sanitize import sanitize; print('compose-imports-ok')" +compose-imports-ok +``` + +The arm64 image intentionally does not install the full upstream +`requirements-eval.txt`, because several old scientific pins are amd64-oriented +and brittle on arm64. It installs the pinned BigCodeBench source plus the +minimal dependencies required by OpenBench's official sanitizer/evaluator path. + +Blocked locally: + +```text +docker run --platform linux/amd64 ... bigcodebench/bigcodebench-evaluate@sha256:1327bddf60be9bc241648c59e6060cac4ca50248a0588ab735cd0200b17cc8c2 ... +qemu: uncaught target signal 11 (Segmentation fault) - core dumped +``` + +The host is `arm64` and Docker reports `linux/arm64`; the official evaluator is +only available as `linux/amd64`. Even a synthetic tiny runner payload segfaulted +under QEMU, so repeated local smoke attempts were stopped to avoid freezes. + +An x86_64 GPU instance (`ghub5090`) was also tried after explicit user approval. +Docker was installed and started with bridge/iptables disabled, then retried with +the `vfs` storage driver because the instance appears containerized. The daemon +could start, but pulling/registering the official image failed with: + +```text +failed to register layer: unshare: operation not permitted +``` + +Temporary Docker data and the smoke payload were removed from the GPU instance +after the failed attempt. The `docker.io` package remains installed there because +installation was explicitly approved for this validation attempt. + +## Remaining requirement + +For strict upstream-image parity, run the official Docker scorer smoke on a +native `linux/amd64` host with Docker privileges sufficient for image extraction +and container creation. Until that passes, BigCodeBench should be described as +implemented with an arm64 source-equivalent smoke, not as validated against the +official amd64 image. diff --git a/benchmark/wave2-code/contract.md b/benchmark/wave2-code/contract.md new file mode 100644 index 00000000..e4ef8342 --- /dev/null +++ b/benchmark/wave2-code/contract.md @@ -0,0 +1,56 @@ +# Wave 2 benchmark integration contract + +## Goal + +Integrate OJBench, TIR-Bench, Codeforces ELO, LiveBench, BigCodeBench, Spider, +VITA-Bench, and MEGA into OpenBench without substituting similarly named data or +approximating metrics that require unavailable model/provider capabilities. + +## Candidates + +The candidate list is frozen to the eight benchmark identities above. Each is +treated as a separate integration candidate; no benchmark may borrow another's +score or dataset identity. + +## Task matrix + +For every candidate: + +1. identify the canonical repository, release/dataset revision, license, prompt, + sampling settings, and scorer; +2. record an immutable revision and checksums where practical; +3. implement the complete public evaluation protocol or mark the candidate + unsupported with evidence; +4. add registry metadata, unit tests, and a real dataset/task construction smoke; +5. reuse a hardened Docker boundary whenever generated code, SQL, shell, or other + untrusted actions execute; +6. run global lint, typing, unit tests, package build, and applicable Docker tests. + +## Metrics and fairness + +The primary integration metric is protocol completeness: `1` only when the +canonical public protocol is runnable and tested, otherwise `0`. Secondary +evidence records logical case count, task coverage, resource boundary, and known +historical-score limitations. Every candidate receives the same source audit and +validation gates; failures remain in `matrix.tsv` rather than being dropped. + +## Environment + +- Baseline commit: `7f11867` (`main`, after BFCL live merge) +- Python: project `.venv`, managed by UV +- Execution host: macOS/Docker Desktop; Linux execution images are digest-pinned +- Upstream network artifacts may drift unless an immutable revision and digest + are recorded + +## Baseline + +At contract creation, none of the eight benchmark IDs exists in `src/` or the +registry. The repository baseline passes 446 unit tests with two documented +environment skips and four Docker sandbox integration tests. + +## Stopping condition + +Stop only when every candidate is either integrated and validated or explicitly +blocked with source-backed evidence, then commit, open a pull request, and follow +CI to completion. Model runs requiring unavailable paid credentials are logged as +blocked and never represented as benchmark scores. diff --git a/benchmark/wave2-code/matrix.tsv b/benchmark/wave2-code/matrix.tsv new file mode 100644 index 00000000..8eff8eca --- /dev/null +++ b/benchmark/wave2-code/matrix.tsv @@ -0,0 +1,14 @@ +candidate case rep metric resource status notes +ojbench registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 +ojbench official_protocol_audit 464_prompts_232_problems 0 dmoj+g++17+pypy3+git_lfs_testdata unsupported Official OJBench repo commit 5e94480b1e135b98855cf5bc81213c256aff5b17 and HF testdata HEAD 61cf9986f22c25d08e1657b03742124099c74353 expose 464 prompts with sha256 bcc8c94eb1fefb856355aa8b5a3e20cc0a2112f5436c5d83ab686edb417bce2c, but faithful judging requires DMOJ 4.1.0 at judge-server commit f098cd3a49a60186d1fadde5132329ec5f4f2213 plus g++17/PyPy3 and LFS problem zips; no hardened OpenBench Docker image has been validated for DMOJ under cap-drop/no-network, and OpenCompass only loads prompts without scoring. +tir_bench registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 +codeforces_elo registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 +livebench registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 +bigcodebench registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 +bigcodebench openbench_adapter 1140_full+148_hard 0 arm64_source_equiv+x86_official arm64_smoke_passed_official_amd64_blocked Implemented registry ID bigcodebench for official v0.1.4 complete/instruct and full/hard axes using HF dataset revisions b74c0d0bf70d2c0bc459be537895cca163007f1a and 298d2cc7b96612e15e47313c3603ee124cee0c1f plus pinned official amd64 evaluator image bigcodebench/bigcodebench-evaluate@sha256:1327bddf60be9bc241648c59e6060cac4ca50248a0588ab735cd0200b17cc8c2; OpenBench now also provides a source-equivalent arm64 scorer image built from upstream commit 09dd993f46c3fbf3a799465bb96d524edcb0b199, with Docker runner smoke passing on BigCodeBench/13 under network none/read-only/cap-drop/no-new-privileges. Unit tests, ruff, targeted mypy, and compose arm64 import smoke pass; official amd64 image remains unvalidated locally because it segfaults under QEMU on arm64 and the x86_64 GPU instance disallows Docker layer registration/unshare. +spider registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 +spider official_protocol_audit 1034_dev 0 5.15GB_testsuite+missing_llm_prompt unsupported Official Spider repo commit b7b5b8c890cd30e35427348bb9eb8c6d1350ca7c and official test-suite-sql-eval commit e97acc546ecbee8fa27fa8dbf025ef61493a876c define data and test-suite execution accuracy, but no canonical LLM prompt/sampling protocol is published; the required test-suite DB archive is external Google Drive data sha256 9ec24ea8debc6bd04abfe137b5f1a739b5a8836f32c0464e4dfc94eb7f41da96, 1.2GB compressed and 5.15GB uncompressed, so OpenBench should not report a hand-prompted approximation. +vita_bench registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 +vita_bench official_protocol_audit 400 0 3_llm_roles+external_judge unsupported Official repo commit 973756f4754873474e2931a404f68093df9ef4e2 and HF dataset HEAD 5ca6848c215cdffd5ef9bc704ddcb62ed74696f0 expose 100 cross-domain plus 300 single-domain tasks, but the public protocol requires a target agent LLM, LLM user simulator, and LLM trajectory evaluator configured through models.yaml; no faithful offline Inspect score can be produced without extra model credentials and judge authority. +mega registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 +mega official_protocol_audit 16_datasets_70_languages 0 openai+azure_translate+hf_endpoints unsupported Official repo commit 3e96bab146151942ed6a7bbe59c0364a78ebf94f is public MIT code, but the released framework is a script/notebook collection with provider-specific OpenAI/Azure/HF endpoint dependencies, hard-coded key/env requirements, PromptSource setup, and no immutable all-task main-results manifest suitable for a faithful provider-agnostic Inspect task. diff --git a/benchmark/wave2-code/mega.md b/benchmark/wave2-code/mega.md new file mode 100644 index 00000000..2cc8637f --- /dev/null +++ b/benchmark/wave2-code/mega.md @@ -0,0 +1,75 @@ +# MEGA audit + +## Decision + +MEGA is recorded as unsupported for this OpenBench wave. The public repository is +available, but the released artifact is a collection of experiment scripts and +notebooks for multilingual LLM evaluation rather than a single stable benchmark +protocol that OpenBench can run provider-agnostically. + +## Canonical source + +- Repository: `https://github.com/microsoft/Multilingual-Evaluation-of-Generative-AI-MEGA.git` +- Repository commit: `3e96bab146151942ed6a7bbe59c0364a78ebf94f` +- License: MIT +- Paper scope stated in README: 16 NLP datasets across 70 languages. + +## Official execution shape + +The README describes MEGA as a framework and documents XNLI as the concrete +example. The repository then supplies task-specific shell scripts and notebooks. +The current script inventory contains 21 `python -m mega...` invocations across +these modules: + +- `mega.XLSUM` +- `mega.analysis.contamination` +- `mega.answer_cls` +- `mega.eval_pawsx` +- `mega.eval_qa_gptindex` +- `mega.eval_qa_gptturbo` +- `mega.eval_tag` +- `mega.eval_xcopa` +- `mega.eval_xnli` +- `mega.eval_xstory_cloze` + +Those scripts encode per-task/per-language choices such as prompt names, +few-shot counts, model names, validation-vs-test switches, translation modes, +and metric output paths. There is no single main-results manifest that freezes +all 16 datasets, languages, prompts, splits, few-shot selections, and model +settings in one runnable protocol. + +## External service requirements + +The official setup requires API credentials in `keys/` for OpenAI and Bing +Translator. The code also imports environment variables at module import time, +including OpenAI endpoint settings, Hugging Face endpoint/key settings, and Bing +Translator endpoint/key settings. + +`mega/models/completion_models.py` calls the legacy OpenAI completion/chat APIs +directly, sleeps for rate limiting, and supports a fixed model list including +Azure-style names such as `gpt-35-turbo`, `gpt-35-turbo-16k`, `gpt-4`, and +`gpt-4-32k`, plus BLOOM/BLOOMZ through Hugging Face endpoints. That is not the +same execution path as OpenBench's Inspect model abstraction. + +## Dependencies and artifacts + +The repository declares Python 3.7 compatibility and includes older unpinned or +pinned dependencies such as `transformers==4.30.0`, `langchain==0.0.317`, +`networkx==1.11`, `word2word==1.0.0`, `openai`, `datasets`, `evaluate`, +`torch`, and a vendored PromptSource tree with 331 template files. + +The repository includes prior GPT-4 XLSUM artifacts under `gpt-4-all-lang-eval/`: +35 prediction CSV files and `xlsum_gpt_4_metrics.csv` with SHA-256 +`a235258dc91cd5d219f8fde96376db1ee07f5de4a6dc0bb8ed9011477ef0b429`. These are +historical result artifacts, not an evaluation dataset for arbitrary models. + +## OpenBench compatibility finding + +A faithful OpenBench integration would need a new MEGA manifest first: exact +dataset revisions, language list per dataset, prompt template IDs, split policy, +few-shot selection policy, translation-test policy, model sampling settings, and +metric aggregation rules. It would also need a provider-agnostic rewrite of the +OpenAI/Azure/HF-specific generation layer. Without those pieces, exposing a +single `mega` registry ID would either cover only a hand-picked subset or report +scores that are not comparable to the official MEGA experiments. The candidate +is therefore blocked rather than approximated. diff --git a/benchmark/wave2-code/ojbench.md b/benchmark/wave2-code/ojbench.md new file mode 100644 index 00000000..1d65f8e6 --- /dev/null +++ b/benchmark/wave2-code/ojbench.md @@ -0,0 +1,64 @@ +# OJBench audit + +## Decision + +OJBench is recorded as unsupported for this OpenBench wave. Unlike Spider and +MEGA, OJBench does publish a full prompt file for LLM generation, but faithful +scoring depends on a DMOJ-based online-judge runtime that has not been validated +inside OpenBench's hardened Docker policy. + +## Canonical sources + +- Repository: `https://github.com/He-Ren/OJBench.git` +- Repository commit: `5e94480b1e135b98855cf5bc81213c256aff5b17` +- Test data: `https://huggingface.co/datasets/He-Ren/OJBench_testdata` +- Test data HEAD: `61cf9986f22c25d08e1657b03742124099c74353` +- OJBench license: AGPL-3.0. + +## Public prompt asset + +The official test-data repository contains `prompts/full.jsonl`. It was read +without printing benchmark prompts. + +- Rows: 464 +- Problems: 232, each with Python and C++ variants +- SHA-256: `bcc8c94eb1fefb856355aa8b5a3e20cc0a2112f5436c5d83ab686edb417bce2c` +- Datasets: 318 NOI rows and 146 ICPC rows +- Languages: 232 Python rows and 232 C++ rows +- Difficulty labels: 72 easy, 158 medium, 234 hard +- Fields: `id`, `prompt`, `dataset`, `language`, `difficulty` + +## Official judging requirements + +The README requires: + +- DMOJ judge-server, specifically checked out at + `f098cd3a49a60186d1fadde5132329ec5f4f2213`; +- `dmoj==4.1.0`; +- C++17-compatible `g++`; +- `pypy3`; +- OJBench test data cloned through Git LFS; +- `ojbench.init()` pointed at both NOI and ICPC problem directories before + calling `judge_jsonl`. + +The judge reports full AC/WA/RE-style verdicts plus partial verdicts after 1/8, +1/4, and 1/2 of test cases. The main score is `is_passed`, equivalent to final +verdict `AC`. + +## OpenCompass note + +The OpenCompass adapter at +`opencompass/opencompass/datasets/ojbench.py` only loads `id` and `prompt` from a +JSONL file. It does not implement DMOJ setup, code extraction, test execution, +partial verdicts, or scoring, so it is not sufficient evidence for a faithful +OpenBench integration. + +## OpenBench compatibility finding + +OJBench should be integrated only once there is a validated Docker execution +boundary for DMOJ that preserves OpenBench's safety policy: network disabled, +capabilities dropped, no new privileges, bounded process/memory limits, and no +host compiler/runtime escape. That image also needs the large LFS problem zips +or a reproducible cache step. Until that exists, adding a registry ID would risk +either weakening the sandbox or reporting scores from an unvalidated judge. The +candidate is therefore blocked rather than approximated. diff --git a/benchmark/wave2-code/spider.md b/benchmark/wave2-code/spider.md new file mode 100644 index 00000000..a16bc741 --- /dev/null +++ b/benchmark/wave2-code/spider.md @@ -0,0 +1,64 @@ +# Spider audit + +## Decision + +Spider is recorded as unsupported for this OpenBench wave. The public data and +official scorer are available, but the canonical sources do not define a full +LLM evaluation protocol: there is no official prompt template, schema rendering, +or sampling configuration for modern generative models. Adding one in OpenBench +would create a new Spider-like evaluation, not a faithful Spider benchmark. + +## Canonical sources + +- Spider repository: `https://github.com/taoyds/spider.git` +- Spider repository commit: `b7b5b8c890cd30e35427348bb9eb8c6d1350ca7c` +- Test-suite repository: `https://github.com/taoyds/test-suite-sql-eval.git` +- Test-suite repository commit: `e97acc546ecbee8fa27fa8dbf025ef61493a876c` +- Licenses: Apache-2.0 for both repositories. + +## Public data and scorer facts + +- Hugging Face `spider` and `xlangai/spider` expose the development split with + 1,034 examples and fields such as `db_id`, `question`, and `query`. +- The Spider repository states that since November 2020 Spider uses test-suite + accuracy as the official metric and points to `test-suite-sql-eval`. +- The test-suite README instructs users to download the database/test-suite + archive from Google Drive and place it under `database/`. +- Local archive inspected: `/tmp/spider-drive.body` + - SHA-256: `9ec24ea8debc6bd04abfe137b5f1a739b5a8836f32c0464e4dfc94eb7f41da96` + - Compressed size: about 1.2 GB + - Zip entries: 3,999 + - SQLite files: 3,889 + - Uncompressed bytes: 5,155,457,198 +- The checked-out `test-suite-sql-eval/database/` directory contains only a + README; it does not include the SQLite suites. + +## Official metric + +The official command shape is: + +```text +python3 evaluation.py --gold [gold file] --pred [predicted file] --etype exec --db [database dir] --table [table file] --plug_value --keep_distinct --progress_bar_for_each_datapoint +``` + +The `exec` metric compares predicted SQL and gold SQL denotations over all +SQLite databases in each test-suite database directory. This is materially +different from ordinary single-database execution accuracy and cannot be +approximated by evaluating only the original Spider database. + +## OpenBench compatibility finding + +A faithful OpenBench task would need all of the following frozen by the +canonical benchmark: + +- how to render each database schema to the model; +- the exact natural-language prompt around the user question; +- whether to include table contents or sampled rows; +- whether models predict SQL values directly or rely on `--plug_value`; +- generation temperature, stop sequences, and max tokens; +- how to package or require the 5.15 GB test-suite databases. + +The official repositories define the data and scorer but not those generation +choices. OpenBench therefore records Spider as blocked rather than shipping a +hand-prompted variant whose scores would not be comparable to the official +leaderboard protocol. diff --git a/benchmark/wave2-code/validate_bigcodebench_arm64.py b/benchmark/wave2-code/validate_bigcodebench_arm64.py new file mode 100644 index 00000000..7d7e0e5e --- /dev/null +++ b/benchmark/wave2-code/validate_bigcodebench_arm64.py @@ -0,0 +1,77 @@ +"""Preflight canonical BigCodeBench records inside the arm64 scorer image. + +This script intentionally consumes a generated payload file rather than loading +the dataset in the sandbox. It emits only task identifiers, status, and timing; +prompts, tests, and canonical solutions remain in the local payload file. +""" + +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path +from typing import Any + +from bigcodebench.gen.util import trusted_check + +RESULT_PREFIX = "OPENBENCH_RESULT\t" +SUMMARY_PREFIX = "OPENBENCH_SUMMARY\t" + + +def validate_record(record: dict[str, Any]) -> dict[str, Any]: + """Execute one canonical solution with the pinned upstream checker.""" + + started = time.monotonic() + try: + result = trusted_check( + record["complete_prompt"] + "\n" + record["canonical_solution"], + record["test"], + record["task_id"], + record["max_as_limit"], + record["max_data_limit"], + record["max_stack_limit"], + record["min_time_limit"], + ) + canonical_time = result["time"] + return { + "task_id": record["task_id"], + "passed": canonical_time is not None, + "canonical_time": canonical_time, + "wall_seconds": round(time.monotonic() - started, 6), + "error_type": None, + } + except Exception as exc: # noqa: BLE001 - diagnostic boundary + return { + "task_id": record["task_id"], + "passed": False, + "canonical_time": None, + "wall_seconds": round(time.monotonic() - started, 6), + "error_type": type(exc).__name__, + } + + +def main() -> None: + """Validate every payload row and emit a machine-readable summary.""" + + payload_path = Path(sys.argv[1]) + records = json.loads(payload_path.read_text()) + started = time.monotonic() + passed = 0 + + for record in records: + result = validate_record(record) + passed += int(result["passed"]) + print(f"{RESULT_PREFIX}{json.dumps(result, sort_keys=True)}", flush=True) + + summary = { + "passed": passed, + "total": len(records), + "pass_rate": passed / len(records) if records else 0.0, + "wall_seconds": round(time.monotonic() - started, 6), + } + print(f"{SUMMARY_PREFIX}{json.dumps(summary, sort_keys=True)}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmark/wave2-code/vita_bench.md b/benchmark/wave2-code/vita_bench.md new file mode 100644 index 00000000..fcaa9270 --- /dev/null +++ b/benchmark/wave2-code/vita_bench.md @@ -0,0 +1,82 @@ +# VITA-Bench audit + +## Decision + +VITA-Bench is recorded as unsupported for this OpenBench wave. The public +benchmark is available, but the official protocol is not a single-model offline +task: it requires a model under test, a separate LLM user simulator, and a +separate LLM trajectory evaluator. OpenBench should not report a VITA score by +substituting a deterministic scorer or by silently choosing auxiliary LLMs. + +## Canonical sources + +- Repository: `https://github.com/meituan/vitabench.git` +- Repository commit: `973756f4754873474e2931a404f68093df9ef4e2` +- Dataset: `https://huggingface.co/datasets/meituan-longcat/VitaBench` +- Dataset HEAD: `5ca6848c215cdffd5ef9bc704ddcb62ed74696f0` +- License: MIT + +## Public task assets + +The official repository includes four task files under `data/vita/domains/`: + +| Domain | Tasks | SHA-256 | Bytes | +| --- | ---: | --- | ---: | +| `cross_domain` | 100 | `3d662cd36efae511e256842e81d54b97f399152be9fdf26dc36ffe87cf0765bd` | 10,323,128 | +| `delivery` | 100 | `5a122f783d2b501f063c718b8dc9a637de573b90c9a87a7f9f4336b1ea9c9404` | 2,536,391 | +| `instore` | 100 | `f92b9313e5476499d51b73a929cc901b7d87cd7c3315df56942d62c83aa34407` | 5,290,647 | +| `ota` | 100 | `874e9117f94758a33ec565ab43ed53361bf09253e157b6f3174e0e5dbf32cbde` | 8,732,896 | + +The audit intentionally does not inline task instructions, user scenarios, or +judge prompts. They are benchmark inputs in the canonical files above, and +printing them into OpenBench documentation is unnecessary for reproducibility. + +## Official run settings + +Observed from `src/vita/config.py`, `src/vita/cli.py`, and `src/vita/run.py`: + +- Default domain: `delivery,instore,ota` for cross-domain evaluation. +- Default agent implementation: `llm_agent`. +- Default user implementation: `user_simulator`. +- Default agent LLM: `gpt-4.1`. +- Default user-simulator LLM: `gpt-4.1`. +- Default evaluator LLM: `anthropic.claude-3.7-sonnet`. +- Default evaluation type: `trajectory`. +- Default maximum steps: 300. +- Default maximum consecutive errors: 10. +- Default trials: 1. +- Default seed: 300. +- Default language: `chinese`. + +`models.yaml` supplies provider base URLs, headers, token settings, and model +costs. The official runner reads that configuration and calls external model +APIs directly. + +## Scoring protocol + +All public evaluation modes route through `TrajectoryEvaluator` and call +`generate()` for the evaluator model: + +- `trajectory` +- `trajectory_full_traj_rubric` +- `trajectory_sliding_wo_rubric` +- `trajectory_full_traj_wo_rubric` + +The default `trajectory` evaluator uses a sliding window over the complete +conversation, keeps rubric state across windows, and returns a binary reward of +1 only when every natural-language rubric is met. That makes the judge model a +required part of the metric, not an implementation detail. + +The simulator also calls `generate()` for the user role. Even the registered +`dummy_user` path is implemented as an LLM-backed user class, so it is not a +deterministic replacement for the official user simulator. + +## OpenBench compatibility finding + +OpenBench can support agentic tasks when the environment and scorer are +deterministic or explicitly parameterized, as with AgentDojo and tau-bench. A +faithful VITA-Bench integration would need new support for auxiliary LLM roles +and a configured judge model in addition to the evaluated model. Without that, +any OpenBench task would either omit the official user simulator, omit the +official judge, or hard-code extra paid model dependencies. The candidate is +therefore blocked rather than approximated. diff --git a/pyproject.toml b/pyproject.toml index 3bb7a640..33f1203d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ include = ["openbench*"] [tool.setuptools.package-data] "openbench.evals.livecodebench" = ["Dockerfile", "compose.yaml"] +"openbench.evals.bigcodebench" = ["Dockerfile.arm64", "compose.yaml", "compose.arm64.yaml"] "openbench.evals.evalplus" = ["Dockerfile", "compose.yaml"] "openbench.evals.bfcl" = [ "Dockerfile", diff --git a/src/openbench/_registry.py b/src/openbench/_registry.py index 3a12f9c7..8143711a 100644 --- a/src/openbench/_registry.py +++ b/src/openbench/_registry.py @@ -265,7 +265,9 @@ def openbench_vllm_override(): from .evals.hellaswag import hellaswag # noqa: F401, E402 from .evals.hle import hle, hle_text # noqa: F401, E402 from .evals.humaneval import humaneval # noqa: F401, E402 +from .evals.bigcodebench import bigcodebench # noqa: F401, E402 from .evals.livecodebench import livecodebench_v6 # noqa: F401, E402 +from .evals.livebench import livebench_coding_2024_11_25 # noqa: F401, E402 from .evals.ifeval import ifeval # noqa: F401, E402 from .evals.ifbench import ifbench # noqa: F401, E402 from .evals.exercism.exercism import ( # noqa: F401, E402 diff --git a/src/openbench/config.py b/src/openbench/config.py index 6cd31b3d..26f30704 100644 --- a/src/openbench/config.py +++ b/src/openbench/config.py @@ -433,6 +433,14 @@ class EvalGroup: module_path="openbench.evals.humaneval", function_name="humaneval", ), + "bigcodebench": BenchmarkMetadata( + name="BigCodeBench", + description="Practical Python code generation with diverse function calls and complex instructions", + category="core", + tags=["coding", "generation", "execution", "docker", "python"], + module_path="openbench.evals.bigcodebench", + function_name="bigcodebench", + ), "livecodebench_v6": BenchmarkMetadata( name="LiveCodeBench v6", description="Contamination-aware Python code generation through the April 2025 release", @@ -441,6 +449,14 @@ class EvalGroup: module_path="openbench.evals.livecodebench", function_name="livecodebench_v6", ), + "livebench_coding_2024_11_25": BenchmarkMetadata( + name="LiveBench Coding 2024-11-25", + description="Latest fully public LiveBench contamination-free coding release", + category="core", + tags=["coding", "generation", "execution", "docker", "live"], + module_path="openbench.evals.livebench", + function_name="livebench_coding_2024_11_25", + ), # Exercism benchmarks "exercism": BenchmarkMetadata( name="Exercism", diff --git a/src/openbench/datasets/bigcodebench.py b/src/openbench/datasets/bigcodebench.py new file mode 100644 index 00000000..3114fc11 --- /dev/null +++ b/src/openbench/datasets/bigcodebench.py @@ -0,0 +1,128 @@ +"""Dataset loader for BigCodeBench v0.1.4.""" + +from __future__ import annotations + +from functools import lru_cache +from typing import Any, Literal + +from datasets import load_dataset # type: ignore[import-untyped] +from inspect_ai.dataset import Dataset, MemoryDataset, Sample +from inspect_ai.model import ChatMessageUser + +Split = Literal["complete", "instruct"] +Subset = Literal["full", "hard"] + +DATASET_VERSION = "v0.1.4" +DATASET_REPOSITORIES: dict[Subset, str] = { + "full": "bigcode/bigcodebench", + "hard": "bigcode/bigcodebench-hard", +} +DATASET_REVISIONS: dict[Subset, str] = { + "full": "b74c0d0bf70d2c0bc459be537895cca163007f1a", + "hard": "298d2cc7b96612e15e47313c3603ee124cee0c1f", +} + +INSTRUCTION_PREFIX = ( + "Please provide a self-contained Python script that solves the following " + "problem in a markdown code block:" +) + + +def validate_bigcodebench_options(split: str, subset: str) -> tuple[Split, Subset]: + """Validate BigCodeBench split/subset names.""" + + if split not in {"complete", "instruct"}: + raise ValueError("split must be one of: complete, instruct") + if subset not in {"full", "hard"}: + raise ValueError("subset must be one of: full, hard") + return split, subset # type: ignore[return-value] + + +def format_bigcodebench_prompt(prompt: str, split: Split) -> str: + """Format prompts like BigCodeBench's API/chat backend.""" + + prompt = prompt.strip() + if split == "complete": + return f"{INSTRUCTION_PREFIX}\n```\n{prompt}\n```\n" + return f"{INSTRUCTION_PREFIX}\n{prompt}\n" + + +@lru_cache(maxsize=2) +def _bigcodebench_records(subset: Subset) -> dict[str, dict[str, Any]]: + repository = DATASET_REPOSITORIES[subset] + dataset = load_dataset( + repository, + split=DATASET_VERSION, + revision=DATASET_REVISIONS[subset], + ) + return {str(record["task_id"]): dict(record) for record in dataset} + + +def load_bigcodebench_execution_fields( + metadata: dict[str, Any], +) -> tuple[str, str, str, str, str]: + """Resolve test and calibration fields from the immutable HF dataset.""" + + subset = metadata["subset"] + if subset not in DATASET_REPOSITORIES: + raise ValueError(f"Unknown BigCodeBench subset: {subset}") + if metadata.get("dataset_version") != DATASET_VERSION: + raise ValueError("BigCodeBench dataset_version mismatch") + if metadata.get("dataset_revision") != DATASET_REVISIONS[subset]: + raise ValueError("BigCodeBench dataset_revision mismatch") + + task_id = metadata["task_id"] + record = _bigcodebench_records(subset)[task_id] + return ( + record["code_prompt"], + record["test"], + record["entry_point"], + record["complete_prompt"], + record["canonical_solution"], + ) + + +def record_to_sample(record: dict[str, Any], split: Split, subset: Subset) -> Sample: + """Convert one official BigCodeBench record into an Inspect sample.""" + + prompt_key = f"{split}_prompt" + task_id = str(record["task_id"]) + return Sample( + id=task_id, + input=[ + ChatMessageUser( + content=format_bigcodebench_prompt(record[prompt_key], split) + ) + ], + target="", + metadata={ + "task_id": task_id, + "split": split, + "subset": subset, + "dataset_version": DATASET_VERSION, + "dataset_revision": DATASET_REVISIONS[subset], + }, + ) + + +def get_bigcodebench_dataset( + split: str = "instruct", + subset: str = "full", + limit: int | None = None, +) -> Dataset: + """Load the official BigCodeBench dataset as Inspect samples.""" + + resolved_split, resolved_subset = validate_bigcodebench_options(split, subset) + if limit is not None and limit <= 0: + raise ValueError("limit must be positive when provided") + + records = list(_bigcodebench_records(resolved_subset).values()) + samples = [ + record_to_sample(record, split=resolved_split, subset=resolved_subset) + for record in records[:limit] + ] + return MemoryDataset( + samples=samples, + name=f"bigcodebench_{resolved_split}_{resolved_subset}", + location=DATASET_REPOSITORIES[resolved_subset], + ) diff --git a/src/openbench/datasets/livebench.py b/src/openbench/datasets/livebench.py new file mode 100644 index 00000000..0eb00c4e --- /dev/null +++ b/src/openbench/datasets/livebench.py @@ -0,0 +1,146 @@ +"""Dataset adapter for the public LiveBench coding release 2024-11-25.""" + +from __future__ import annotations + +import json +from functools import cache +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pyarrow.parquet as pq +from huggingface_hub import hf_hub_download +from inspect_ai.dataset import Dataset, MemoryDataset, Sample + +DATASET_REPOSITORY = "livebench/coding" +DATASET_REVISION = "a958549fdd8aa57be0a3fafe7b205ffc160ed5f4" +DATASET_FILE = "data/test-00000-of-00001.parquet" +RELEASE_DATE = "2024-11-25" +SUPPORTED_TASKS = frozenset({"LCB_generation", "coding_completion"}) + +_INDEX_COLUMNS = ( + "question_id", + "turns", + "question_title", + "task", + "livebench_release_date", + "livebench_removal_date", + "partial_solution", +) +_TEST_COLUMNS = ( + "question_id", + "public_test_cases", + "private_test_cases", + "original_json", + "partial_solution", +) + + +def _dataset_path(*, local_files_only: bool = False) -> Path: + return Path( + hf_hub_download( + repo_id=DATASET_REPOSITORY, + filename=DATASET_FILE, + repo_type="dataset", + revision=DATASET_REVISION, + local_files_only=local_files_only, + ) + ) + + +def _read_table(path: Path, columns: tuple[str, ...]) -> pa.Table: + return pq.read_table(path, columns=list(columns)) + + +@cache +def _cached_test_rows(path: str) -> list[dict[str, Any]]: + """Keep hidden tests in process memory, outside Inspect sample metadata.""" + + return _read_table(Path(path), _TEST_COLUMNS).to_pylist() + + +def _iso_date(value: Any) -> str: + if hasattr(value, "date"): + return value.date().isoformat() + return str(value)[:10] + + +def _is_release_member(record: dict[str, Any]) -> bool: + released = _iso_date(record["livebench_release_date"]) + removal_value = record.get("livebench_removal_date") + removed = _iso_date(removal_value) if removal_value else "" + return released <= RELEASE_DATE and (not removed or removed > RELEASE_DATE) + + +def record_to_sample(record: dict[str, Any], row_index: int) -> Sample | None: + """Convert one official row while applying LiveBench release semantics.""" + + task_name = record["task"] + if task_name not in SUPPORTED_TASKS or not _is_release_member(record): + return None + turns = record["turns"] + if not isinstance(turns, list) or len(turns) != 1: + raise ValueError("LiveBench coding 2024-11-25 expects one user turn") + question_id = record["question_id"] + return Sample( + id=question_id, + input=turns[0], + target="", + metadata={ + "question_title": record["question_title"], + "task": task_name, + "release_date": RELEASE_DATE, + "dataset_revision": DATASET_REVISION, + "source_file": DATASET_FILE, + "source_row": row_index, + "source_question_id": question_id, + }, + ) + + +def get_livebench_coding_dataset() -> Dataset: + """Load the 128 public coding questions active in release 2024-11-25.""" + + path = _dataset_path() + records = _read_table(path, _INDEX_COLUMNS).to_pylist() + samples = [ + sample + for index, record in enumerate(records) + if (sample := record_to_sample(record, index)) is not None + ] + samples.sort(key=lambda sample: str(sample.id)) + return MemoryDataset( + samples=samples, + name="livebench_coding_2024_11_25", + location=DATASET_REPOSITORY, + ) + + +def load_livebench_test_fields( + metadata: dict[str, Any], +) -> tuple[str, str, str, str]: + """Resolve hidden tests from the immutable parquet source at score time.""" + + row_index = metadata.get("source_row") + if not isinstance(row_index, int): + raise TypeError("LiveBench source_row must be an integer") + path = _dataset_path(local_files_only=True) + rows = _cached_test_rows(str(path)) + try: + record = rows[row_index] + except IndexError as error: + raise ValueError( + "LiveBench source_row is outside the pinned dataset" + ) from error + if record["question_id"] != metadata.get("source_question_id"): + raise ValueError("LiveBench source reference resolved to the wrong record") + original = record["original_json"] + test_metadata = original.get("metadata") if isinstance(original, dict) else None + if not isinstance(test_metadata, str): + test_metadata = json.dumps(test_metadata or {}) + return ( + record["public_test_cases"], + record["private_test_cases"], + test_metadata, + record.get("partial_solution") or "", + ) diff --git a/src/openbench/evals/bigcodebench/Dockerfile.arm64 b/src/openbench/evals/bigcodebench/Dockerfile.arm64 new file mode 100644 index 00000000..dfc3e16c --- /dev/null +++ b/src/openbench/evals/bigcodebench/Dockerfile.arm64 @@ -0,0 +1,77 @@ +FROM python:3.10-slim + +ARG BIGCODEBENCH_COMMIT=09dd993f46c3fbf3a799465bb96d524edcb0b199 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + git \ + libfreetype6-dev \ + libgdal-dev \ + libpng-dev \ + libsndfile1 \ + pkg-config \ + procps \ + python3-dev \ + python3-tk \ + r-base \ + tesseract-ocr \ + unzip \ + zip \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --upgrade pip setuptools wheel + +RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench \ + && cd /bigcodebench \ + && git checkout "${BIGCODEBENCH_COMMIT}" \ + && python -m pip install . --no-deps + +# Source-equivalent scorer dependencies exercised by the canonical Hard suite. +# Upstream pins that conflict on Python 3.10/arm64 are adjusted together: +# numpy 1.23.5 satisfies TensorFlow 2.11, while scipy 1.10.1 provides an arm64 +# wheel compatible with scikit-learn, statsmodels, and the selected NumPy. +RUN python -m pip install --no-cache-dir \ + appdirs==1.4.4 \ + beautifulsoup4==4.8.2 \ + chardet==5.2.0 \ + datasets==2.17.0 \ + Faker==20.1.0 \ + Flask==3.0.3 \ + geopandas==0.13.2 \ + Levenshtein==0.25.0 \ + lxml==4.9.3 \ + matplotlib==3.7.0 \ + multipledispatch==1.0.0 \ + nltk==3.8 \ + numpy==1.23.5 \ + opencv-python-headless==4.9.0.80 \ + pqdm==0.2.0 \ + psutil==5.9.5 \ + pyarrow==14.0.1 \ + pyquery==1.4.3 \ + pytesseract==0.3.10 \ + python-docx==1.1.0 \ + rsa==4.9 \ + scikit-learn==1.3.1 \ + scipy==1.10.1 \ + seaborn==0.13.2 \ + soundfile==0.12.1 \ + statsmodels==0.14.0 \ + tempdir==0.7.1 \ + tensorflow==2.11.0 \ + termcolor==3.3.0 \ + tqdm==4.70.0 \ + tree-sitter==0.23.0 \ + tree-sitter-python==0.23.2 \ + wget==3.2 \ + wordcloud==1.9.3 \ + xlwt==1.3.0 + +RUN useradd --create-home --uid 1000 bigcodebenchuser \ + && mkdir --parents /app \ + && chown bigcodebenchuser:bigcodebenchuser /app + +USER bigcodebenchuser +WORKDIR /app + +CMD ["tail", "-f", "/dev/null"] diff --git a/src/openbench/evals/bigcodebench/__init__.py b/src/openbench/evals/bigcodebench/__init__.py new file mode 100644 index 00000000..4b86f8a1 --- /dev/null +++ b/src/openbench/evals/bigcodebench/__init__.py @@ -0,0 +1,5 @@ +"""BigCodeBench evaluation package.""" + +from .bigcodebench import bigcodebench + +__all__ = ["bigcodebench"] diff --git a/src/openbench/evals/bigcodebench/bigcodebench.py b/src/openbench/evals/bigcodebench/bigcodebench.py new file mode 100644 index 00000000..58f6cc2c --- /dev/null +++ b/src/openbench/evals/bigcodebench/bigcodebench.py @@ -0,0 +1,74 @@ +"""BigCodeBench code-generation evaluation.""" + +from pathlib import Path +from platform import machine as platform_machine +from typing import Literal + +from inspect_ai import Epochs, Task, task +from inspect_ai.model import GenerateConfig +from inspect_ai.solver import generate + +from openbench.datasets.bigcodebench import get_bigcodebench_dataset +from openbench.scorers.bigcodebench import bigcodebench_scorer + +TASK_DIR = Path(__file__).parent +OFFICIAL_COMPOSE_PATH = (TASK_DIR / "compose.yaml").resolve() +ARM64_COMPOSE_PATH = (TASK_DIR / "compose.arm64.yaml").resolve() +Runtime = Literal["auto", "official", "arm64"] + + +def compose_path_for_runtime(runtime: Runtime) -> Path: + """Select the BigCodeBench sandbox compose file for the local runtime.""" + + if runtime == "official": + return OFFICIAL_COMPOSE_PATH + if runtime == "arm64": + return ARM64_COMPOSE_PATH + if runtime == "auto": + host_machine = platform_machine().lower() + if host_machine in {"arm64", "aarch64"}: + return ARM64_COMPOSE_PATH + return OFFICIAL_COMPOSE_PATH + raise ValueError("runtime must be one of: auto, official, arm64") + + +@task +def bigcodebench( + split: Literal["complete", "instruct"] = "instruct", + subset: Literal["full", "hard"] = "full", + runtime: Runtime = "auto", + epochs: int = 1, + limit: int | None = None, + total_timeout: int = 900, +) -> Task: + """Run BigCodeBench v0.1.4 with the official Docker evaluator. + + The official protocol supports `complete` and `instruct` splits and `full` + and `hard` subsets. `runtime="official"` uses the pinned upstream amd64 + image; `runtime="arm64"` builds OpenBench's source-equivalent arm64 scorer + image for Apple Silicon machines. `runtime="auto"` selects arm64 only on + arm64/aarch64 hosts. Greedy decoding uses temperature 0 with one sample by + default; increasing `epochs` enables Inspect pass@k reducers. + """ + + if epochs <= 0: + raise ValueError("epochs must be positive") + reducers = ["mean", "pass_at_1"] + if epochs >= 5: + reducers.append("pass_at_5") + if epochs >= 10: + reducers.append("pass_at_10") + + return Task( + name=f"bigcodebench_{split}_{subset}", + dataset=get_bigcodebench_dataset(split=split, subset=subset, limit=limit), + solver=generate(), + scorer=bigcodebench_scorer(total_timeout=total_timeout), + sandbox=("docker", str(compose_path_for_runtime(runtime))), + epochs=Epochs(epochs, reducer=reducers), + config=GenerateConfig( + temperature=0, + top_p=0.95, + max_tokens=1280, + ), + ) diff --git a/src/openbench/evals/bigcodebench/compose.arm64.yaml b/src/openbench/evals/bigcodebench/compose.arm64.yaml new file mode 100644 index 00000000..e6e60117 --- /dev/null +++ b/src/openbench/evals/bigcodebench/compose.arm64.yaml @@ -0,0 +1,24 @@ +services: + default: + build: + context: . + dockerfile: Dockerfile.arm64 + args: + BIGCODEBENCH_COMMIT: 09dd993f46c3fbf3a799465bb96d524edcb0b199 + image: openbench-bigcodebench-arm64:dev + platform: linux/arm64 + entrypoint: ["tail", "-f", "/dev/null"] + init: true + network_mode: none + read_only: true + environment: + MPLCONFIGDIR: /tmp/matplotlib + tmpfs: + - /app:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=2147483648 + - /tmp:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=4294967296 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + pids_limit: 256 + mem_limit: 36g diff --git a/src/openbench/evals/bigcodebench/compose.yaml b/src/openbench/evals/bigcodebench/compose.yaml new file mode 100644 index 00000000..0d721663 --- /dev/null +++ b/src/openbench/evals/bigcodebench/compose.yaml @@ -0,0 +1,19 @@ +services: + default: + image: bigcodebench/bigcodebench-evaluate@sha256:1327bddf60be9bc241648c59e6060cac4ca50248a0588ab735cd0200b17cc8c2 + platform: linux/amd64 + entrypoint: ["tail", "-f", "/dev/null"] + init: true + network_mode: none + read_only: true + environment: + MPLCONFIGDIR: /tmp/matplotlib + tmpfs: + - /app:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=2147483648 + - /tmp:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=4294967296 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + pids_limit: 256 + mem_limit: 36g diff --git a/src/openbench/evals/livebench/__init__.py b/src/openbench/evals/livebench/__init__.py new file mode 100644 index 00000000..ccd72a89 --- /dev/null +++ b/src/openbench/evals/livebench/__init__.py @@ -0,0 +1,5 @@ +"""LiveBench evaluations.""" + +from .livebench import livebench_coding_2024_11_25 + +__all__ = ["livebench_coding_2024_11_25"] diff --git a/src/openbench/evals/livebench/livebench.py b/src/openbench/evals/livebench/livebench.py new file mode 100644 index 00000000..c930dc95 --- /dev/null +++ b/src/openbench/evals/livebench/livebench.py @@ -0,0 +1,32 @@ +"""Public LiveBench coding release 2024-11-25.""" + +from pathlib import Path + +from inspect_ai import Task, task +from inspect_ai.model import GenerateConfig +from inspect_ai.solver import generate + +from openbench.datasets.livebench import get_livebench_coding_dataset +from openbench.scorers.livecodebench import livebench_coding_scorer + +COMPOSE_PATH = (Path(__file__).parents[1] / "livecodebench" / "compose.yaml").resolve() + + +@task +def livebench_coding_2024_11_25( + test_timeout: int = 6, + total_timeout: int = 600, +) -> Task: + """Run the latest fully public LiveBench coding release.""" + + return Task( + name="livebench_coding_2024_11_25", + dataset=get_livebench_coding_dataset(), + solver=generate(), + scorer=livebench_coding_scorer( + test_timeout=test_timeout, + total_timeout=total_timeout, + ), + sandbox=("docker", str(COMPOSE_PATH)), + config=GenerateConfig(temperature=0, max_tokens=4096), + ) diff --git a/src/openbench/scorers/bigcodebench.py b/src/openbench/scorers/bigcodebench.py new file mode 100644 index 00000000..889826e8 --- /dev/null +++ b/src/openbench/scorers/bigcodebench.py @@ -0,0 +1,140 @@ +"""BigCodeBench scorer using the official evaluator inside Docker.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from inspect_ai.scorer import ( + CORRECT, + INCORRECT, + Score, + Scorer, + Target, + accuracy, + scorer, + stderr, +) +from inspect_ai.solver import TaskState +from inspect_ai.util import sandbox + +from openbench.datasets.bigcodebench import load_bigcodebench_execution_fields + + +def build_bigcodebench_payload( + completion: str, + fields: tuple[str, str, str, str, str], + *, + task_id: str, + calibrated: bool, + min_time_limit: int, + max_as_limit: int, + max_data_limit: int, + max_stack_limit: int, +) -> dict[str, Any]: + """Build the JSON payload consumed by the sandbox runner.""" + + code_prompt, test, entry_point, complete_prompt, canonical_solution = fields + return { + "completion": completion, + "task_id": task_id, + "code_prompt": code_prompt, + "test": test, + "entry_point": entry_point, + "complete_prompt": complete_prompt, + "canonical_solution": canonical_solution, + "calibrated": calibrated, + "min_time_limit": min_time_limit, + "max_as_limit": max_as_limit, + "max_data_limit": max_data_limit, + "max_stack_limit": max_stack_limit, + } + + +@scorer(metrics=[accuracy(), stderr()]) +def bigcodebench_scorer( + *, + calibrated: bool = True, + min_time_limit: int = 1, + max_as_limit: int = 30 * 1024, + max_data_limit: int = 30 * 1024, + max_stack_limit: int = 10, + total_timeout: int = 900, +) -> Scorer: + """Score BigCodeBench completions with the official sanitizer/evaluator.""" + + if min_time_limit <= 0: + raise ValueError("min_time_limit must be positive") + if max_as_limit <= 0: + raise ValueError("max_as_limit must be positive") + if max_data_limit <= 0: + raise ValueError("max_data_limit must be positive") + if max_stack_limit <= 0: + raise ValueError("max_stack_limit must be positive") + if total_timeout <= 0: + raise ValueError("total_timeout must be positive") + + async def score(state: TaskState, target: Target) -> Score: + del target + fields = load_bigcodebench_execution_fields(state.metadata) + payload = build_bigcodebench_payload( + state.output.completion, + fields, + task_id=str(state.metadata["task_id"]), + calibrated=calibrated, + min_time_limit=min_time_limit, + max_as_limit=max_as_limit, + max_data_limit=max_data_limit, + max_stack_limit=max_stack_limit, + ) + environment = sandbox() + payload_path = ".openbench_bigcodebench_payload.json" + runner_path = ".openbench_bigcodebench_runner.py" + runner_source = Path(__file__).with_name("bigcodebench_runner.py").read_text() + await environment.write_file(payload_path, json.dumps(payload)) + await environment.write_file(runner_path, runner_source) + + try: + result = await environment.exec( + ["python3", runner_path, payload_path], + timeout=total_timeout, + timeout_retry=False, + ) + except TimeoutError: + return Score( + value=INCORRECT, + explanation="BigCodeBench evaluation exceeded its total timeout.", + ) + + if not result.success: + return Score( + value=INCORRECT, + explanation="BigCodeBench runner failed inside the sandbox.", + ) + + try: + evaluation = json.loads(result.stdout.strip().splitlines()[-1]) + except (IndexError, json.JSONDecodeError): + return Score( + value=INCORRECT, + explanation="BigCodeBench runner returned an invalid result.", + ) + + status = evaluation.get("status", "unknown") + passed = evaluation.get("passed") is True + canonical_time = evaluation.get("canonical_time") + explanation = ( + "Passed official BigCodeBench tests." + if passed + else f"Failed official BigCodeBench tests with status: {status}." + ) + if canonical_time is None: + explanation += " Canonical timing failed; used official fallback timeout." + return Score( + value=CORRECT if passed else INCORRECT, + answer=evaluation.get("solution", ""), + explanation=explanation, + ) + + return score diff --git a/src/openbench/scorers/bigcodebench_runner.py b/src/openbench/scorers/bigcodebench_runner.py new file mode 100644 index 00000000..7bc52a65 --- /dev/null +++ b/src/openbench/scorers/bigcodebench_runner.py @@ -0,0 +1,57 @@ +"""Sandbox-side BigCodeBench runner.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from bigcodebench.eval import PASS, untrusted_check # type: ignore[import-not-found] +from bigcodebench.gen.util import trusted_check # type: ignore[import-not-found] +from bigcodebench.sanitize import sanitize # type: ignore[import-not-found] + + +def main() -> None: + payload_path = Path(sys.argv[1]) + payload = json.loads(payload_path.read_text()) + payload_path.unlink(missing_ok=True) + + solution = sanitize(payload["completion"], payload["entry_point"]) + if payload["calibrated"]: + solution = f"{payload['code_prompt']}\n pass\n{solution}" + + canonical = trusted_check( + payload["complete_prompt"] + "\n" + payload["canonical_solution"], + payload["test"], + payload["task_id"], + payload["max_as_limit"], + payload["max_data_limit"], + payload["max_stack_limit"], + payload["min_time_limit"], + ) + canonical_time = canonical["time"] + status, details = untrusted_check( + solution, + payload["test"], + payload["entry_point"], + payload["max_as_limit"], + payload["max_data_limit"], + payload["max_stack_limit"], + payload["min_time_limit"], + canonical_time if canonical_time is not None else 20, + ) + print( + json.dumps( + { + "passed": status == PASS, + "status": status, + "details": details, + "canonical_time": canonical_time, + "solution": solution, + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/src/openbench/scorers/livecodebench.py b/src/openbench/scorers/livecodebench.py index 427b4f13..44f5456b 100644 --- a/src/openbench/scorers/livecodebench.py +++ b/src/openbench/scorers/livecodebench.py @@ -42,6 +42,21 @@ def extract_code(completion: str) -> str: return "\n".join(lines[fence_lines[-2] + 1 : fence_lines[-1]]) +def extract_livebench_code(completion: str) -> str: + """Extract code using LiveBench's generic-model fallback behavior.""" + + stripped = completion.rstrip() + lines = stripped.splitlines() + fence_lines = [index for index, line in enumerate(lines) if "```" in line] + if len(fence_lines) >= 2: + return "\n".join(lines[fence_lines[-2] + 1 : fence_lines[-1]]) + if len(completion) > 1 and completion[0] == "`" and completion[-1] == "`": + return completion[1:-1] + if len(fence_lines) == 1 and fence_lines[0] == len(lines) - 1: + return "\n".join(lines[:-1]) + return stripped + + def decode_test_cases(value: str | list[dict[str, object]]) -> list[dict[str, object]]: """Decode plain JSON or the pinned dataset's compressed hidden tests.""" @@ -152,3 +167,83 @@ async def score(state: TaskState, target: Target) -> Score: ) return score + + +@scorer(metrics=[accuracy(), stderr()]) +def livebench_coding_scorer( + test_timeout: int = 6, + total_timeout: int = 600, +) -> Scorer: + """Score public LiveBench coding tasks in the hardened LCB sandbox.""" + + if test_timeout <= 0: + raise ValueError("test_timeout must be positive") + if total_timeout <= 0: + raise ValueError("total_timeout must be positive") + + async def score(state: TaskState, target: Target) -> Score: + del target + from openbench.datasets.livebench import load_livebench_test_fields + + code = extract_livebench_code(state.output.completion) + public_tests, private_tests, metadata, partial_solution = ( + load_livebench_test_fields(state.metadata) + ) + if partial_solution and not code.startswith(partial_solution): + code = f"{partial_solution}\n{code}" + tests = decode_test_cases(public_tests) + decode_test_cases(private_tests) + test_metadata = decode_metadata(metadata) + payload = { + "code": code, + "tests": tests, + "function_name": test_metadata.get("func_name"), + "timeout": test_timeout, + } + environment = sandbox() + payload_path = ".openbench_livebench_payload.json" + runner_path = ".openbench_livebench_runner.py" + runner_source = Path(__file__).with_name("livecodebench_runner.py").read_text() + await environment.write_file(payload_path, json.dumps(payload)) + await environment.write_file(runner_path, runner_source) + try: + result = await environment.exec( + ["python", runner_path, payload_path], + timeout=total_timeout, + timeout_retry=False, + ) + except TimeoutError: + return Score( + value=INCORRECT, + answer=code, + explanation="LiveBench evaluation exceeded its total timeout.", + ) + if not result.success: + return Score( + value=INCORRECT, + answer=code, + explanation="LiveBench runner failed inside the sandbox.", + ) + try: + evaluation = json.loads(result.stdout.strip().splitlines()[-1]) + except (IndexError, json.JSONDecodeError): + return Score( + value=INCORRECT, + answer=code, + explanation="LiveBench runner returned an invalid result.", + ) + passed = evaluation.get("passed") is True + explanation = ( + f"Passed all {evaluation['tests_run']} tests." + if passed + else ( + f"Failed after {evaluation.get('tests_run', 0)} test(s): " + f"{evaluation.get('error', 'unknown error')}." + ) + ) + return Score( + value=CORRECT if passed else INCORRECT, + answer=code, + explanation=explanation, + ) + + return score diff --git a/tests/test_bigcodebench.py b/tests/test_bigcodebench.py new file mode 100644 index 00000000..9964f072 --- /dev/null +++ b/tests/test_bigcodebench.py @@ -0,0 +1,151 @@ +"""Tests for the BigCodeBench adapter.""" + +from unittest.mock import patch + +import pytest +from inspect_ai.dataset import MemoryDataset, Sample + +from openbench.datasets.bigcodebench import ( + DATASET_REVISIONS, + DATASET_VERSION, + INSTRUCTION_PREFIX, + format_bigcodebench_prompt, + get_bigcodebench_dataset, + load_bigcodebench_execution_fields, + record_to_sample, +) +from openbench.evals.bigcodebench import bigcodebench +from openbench.evals.bigcodebench.bigcodebench import compose_path_for_runtime +from openbench.scorers.bigcodebench import build_bigcodebench_payload + + +def _record(**overrides): + record = { + "task_id": "BigCodeBench/0", + "complete_prompt": "def task_func():\n pass", + "instruct_prompt": "Create task_func.", + "canonical_solution": "def task_func():\n return 1", + "code_prompt": "def task_func():", + "test": "class TestCases: pass", + "entry_point": "task_func", + } + record.update(overrides) + return record + + +def test_format_bigcodebench_prompt_matches_api_backend_shape(): + assert format_bigcodebench_prompt("Do it", "instruct") == ( + f"{INSTRUCTION_PREFIX}\nDo it\n" + ) + assert format_bigcodebench_prompt("def f(): pass", "complete") == ( + f"{INSTRUCTION_PREFIX}\n```\ndef f(): pass\n```\n" + ) + + +def test_record_to_sample_hides_execution_fields_from_metadata(): + sample = record_to_sample(_record(), split="instruct", subset="full") + assert sample.id == "BigCodeBench/0" + assert sample.input[0].content == f"{INSTRUCTION_PREFIX}\nCreate task_func.\n" + assert sample.metadata == { + "task_id": "BigCodeBench/0", + "split": "instruct", + "subset": "full", + "dataset_version": DATASET_VERSION, + "dataset_revision": DATASET_REVISIONS["full"], + } + + +def test_get_bigcodebench_dataset_validates_options_and_limit(): + with patch( + "openbench.datasets.bigcodebench._bigcodebench_records", + return_value={"BigCodeBench/0": _record()}, + ): + dataset = get_bigcodebench_dataset(split="complete", subset="hard", limit=1) + + assert isinstance(dataset, MemoryDataset) + assert dataset.name == "bigcodebench_complete_hard" + assert len(dataset) == 1 + + with pytest.raises(ValueError, match="split"): + get_bigcodebench_dataset(split="bad", subset="hard") + with pytest.raises(ValueError, match="limit"): + get_bigcodebench_dataset(limit=0) + + +def test_load_bigcodebench_execution_fields_resolves_from_revisioned_record(): + metadata = { + "task_id": "BigCodeBench/0", + "subset": "full", + "dataset_version": DATASET_VERSION, + "dataset_revision": DATASET_REVISIONS["full"], + } + with patch( + "openbench.datasets.bigcodebench._bigcodebench_records", + return_value={"BigCodeBench/0": _record()}, + ): + fields = load_bigcodebench_execution_fields(metadata) + + assert fields == ( + "def task_func():", + "class TestCases: pass", + "task_func", + "def task_func():\n pass", + "def task_func():\n return 1", + ) + + +def test_task_uses_official_greedy_generation_settings(): + dataset = MemoryDataset([Sample(input="prompt", target="")]) + with patch( + "openbench.evals.bigcodebench.bigcodebench.get_bigcodebench_dataset", + return_value=dataset, + ): + task = bigcodebench(epochs=10, limit=1) + + assert task.name == "bigcodebench_instruct_full" + assert task.epochs == 10 + assert task.config.temperature == 0 + assert task.config.top_p == 0.95 + assert task.config.max_tokens == 1280 + + +def test_bigcodebench_runtime_selects_expected_compose_files(): + assert compose_path_for_runtime("official").name == "compose.yaml" + assert compose_path_for_runtime("arm64").name == "compose.arm64.yaml" + + with patch( + "openbench.evals.bigcodebench.bigcodebench.platform_machine", + return_value="arm64", + ): + assert compose_path_for_runtime("auto").name == "compose.arm64.yaml" + + with patch( + "openbench.evals.bigcodebench.bigcodebench.platform_machine", + return_value="x86_64", + ): + assert compose_path_for_runtime("auto").name == "compose.yaml" + + with pytest.raises(ValueError, match="runtime"): + compose_path_for_runtime("bad") # type: ignore[arg-type] + + +def test_build_bigcodebench_payload_preserves_official_execution_fields(): + payload = build_bigcodebench_payload( + "```python\ndef task_func(): return 1\n```", + ( + "def task_func():", + "class TestCases: pass", + "task_func", + "def task_func():\n pass", + "def task_func():\n return 1", + ), + task_id="BigCodeBench/0", + calibrated=True, + min_time_limit=1, + max_as_limit=30 * 1024, + max_data_limit=30 * 1024, + max_stack_limit=10, + ) + assert payload["task_id"] == "BigCodeBench/0" + assert payload["entry_point"] == "task_func" + assert payload["calibrated"] is True diff --git a/tests/test_livebench.py b/tests/test_livebench.py new file mode 100644 index 00000000..de3779d4 --- /dev/null +++ b/tests/test_livebench.py @@ -0,0 +1,111 @@ +"""Tests for the pinned public LiveBench coding adapter.""" + +from datetime import datetime +from unittest.mock import patch + +import pyarrow as pa +from inspect_ai.dataset import MemoryDataset, Sample + +from openbench.datasets.livebench import ( + DATASET_REVISION, + RELEASE_DATE, + _cached_test_rows, + get_livebench_coding_dataset, + load_livebench_test_fields, + record_to_sample, +) +from openbench.evals.livebench import livebench_coding_2024_11_25 +from openbench.scorers.livecodebench import extract_livebench_code + + +def _record(**overrides): + record = { + "question_id": "live-add", + "turns": ["Write a Python program that adds two integers."], + "question_title": "Add", + "task": "LCB_generation", + "livebench_release_date": datetime(2024, 6, 24), + "livebench_removal_date": datetime(2025, 4, 2), + "partial_solution": "", + } + record.update(overrides) + return record + + +def test_record_to_sample_applies_official_release_membership(): + sample = record_to_sample(_record(), 7) + assert isinstance(sample, Sample) + assert sample.id == "live-add" + assert sample.metadata["source_row"] == 7 + assert sample.metadata["release_date"] == RELEASE_DATE + assert sample.metadata["dataset_revision"] == DATASET_REVISION + + assert ( + record_to_sample(_record(livebench_release_date=datetime(2025, 4, 2)), 0) + is None + ) + assert ( + record_to_sample(_record(livebench_removal_date=datetime(2024, 11, 25)), 0) + is None + ) + assert record_to_sample(_record(task="agentic_coding"), 0) is None + + +def test_dataset_keeps_hidden_tests_out_of_sample_metadata(tmp_path): + path = tmp_path / "coding.parquet" + index_table = pa.Table.from_pylist([_record()]) + with ( + patch("openbench.datasets.livebench._dataset_path", return_value=path), + patch("openbench.datasets.livebench._read_table", return_value=index_table), + ): + dataset = get_livebench_coding_dataset() + + assert len(dataset) == 1 + assert dataset[0].input == "Write a Python program that adds two integers." + assert "private_test_cases" not in dataset[0].metadata + + +def test_hidden_tests_resolve_from_pinned_source(tmp_path): + path = tmp_path / "coding.parquet" + test_table = pa.Table.from_pylist( + [ + { + "question_id": "live-add", + "public_test_cases": "[]", + "private_test_cases": "[]", + "original_json": {"metadata": '{"func_name": "add"}'}, + "partial_solution": "def add(a, b):", + } + ] + ) + metadata = { + "source_row": 0, + "source_question_id": "live-add", + } + _cached_test_rows.cache_clear() + with ( + patch("openbench.datasets.livebench._dataset_path", return_value=path), + patch("openbench.datasets.livebench._read_table", return_value=test_table), + ): + fields = load_livebench_test_fields(metadata) + _cached_test_rows.cache_clear() + + assert fields == ("[]", "[]", '{"func_name": "add"}', "def add(a, b):") + + +def test_extract_livebench_code_preserves_official_unfenced_fallback(): + assert extract_livebench_code("print(1)") == "print(1)" + assert extract_livebench_code("```python\nprint(2)\n```") == "print(2)" + assert extract_livebench_code("print(3)\n```") == "print(3)" + + +def test_task_uses_public_release_generation_defaults(): + dataset = MemoryDataset([Sample(input="question", target="")]) + with patch( + "openbench.evals.livebench.livebench.get_livebench_coding_dataset", + return_value=dataset, + ): + task = livebench_coding_2024_11_25() + + assert task.config.temperature == 0 + assert task.config.max_tokens == 4096 diff --git a/tests/test_registry.py b/tests/test_registry.py index abaf9de8..4bc0571e 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -19,6 +19,10 @@ def test_task_registry_contents(): assert TASK_REGISTRY["livecodebench_v6"] == ( "openbench.evals.livecodebench.livecodebench_v6" ) + assert TASK_REGISTRY["bigcodebench"] == "openbench.evals.bigcodebench.bigcodebench" + assert TASK_REGISTRY["livebench_coding_2024_11_25"] == ( + "openbench.evals.livebench.livebench_coding_2024_11_25" + ) def test_load_task_valid(): From 0ec227a664023e402de511f933ad0656e37d11f6 Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 7 Aug 2026 09:54:58 +0800 Subject: [PATCH 2/6] fix: complete BigCodeBench hard runtime dependencies --- src/openbench/evals/bigcodebench/Dockerfile.arm64 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/openbench/evals/bigcodebench/Dockerfile.arm64 b/src/openbench/evals/bigcodebench/Dockerfile.arm64 index dfc3e16c..3f7eecdc 100644 --- a/src/openbench/evals/bigcodebench/Dockerfile.arm64 +++ b/src/openbench/evals/bigcodebench/Dockerfile.arm64 @@ -37,18 +37,25 @@ RUN python -m pip install --no-cache-dir \ datasets==2.17.0 \ Faker==20.1.0 \ Flask==3.0.3 \ + Flask-Login==0.6.3 \ + Flask-Mail==0.9.1 \ + gensim==4.3.2 \ geopandas==0.13.2 \ Levenshtein==0.25.0 \ + librosa==0.10.1 \ lxml==4.9.3 \ matplotlib==3.7.0 \ multipledispatch==1.0.0 \ nltk==3.8 \ + numba==0.57.1 \ numpy==1.23.5 \ opencv-python-headless==4.9.0.80 \ + openpyxl==3.1.2 \ pqdm==0.2.0 \ psutil==5.9.5 \ pyarrow==14.0.1 \ pyquery==1.4.3 \ + pycryptodome==3.14.1 \ pytesseract==0.3.10 \ python-docx==1.1.0 \ rsa==4.9 \ @@ -65,8 +72,11 @@ RUN python -m pip install --no-cache-dir \ tree-sitter-python==0.23.2 \ wget==3.2 \ wordcloud==1.9.3 \ + xlrd==2.0.1 \ xlwt==1.3.0 +RUN python -m nltk.downloader -d /usr/local/share/nltk_data punkt stopwords + RUN useradd --create-home --uid 1000 bigcodebenchuser \ && mkdir --parents /app \ && chown bigcodebenchuser:bigcodebenchuser /app From 4b903eaf9b6899bb905b7e10f1d6fe4d010f124a Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 7 Aug 2026 10:15:32 +0800 Subject: [PATCH 3/6] fix: restore final BigCodeBench hard imports --- src/openbench/evals/bigcodebench/Dockerfile.arm64 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/openbench/evals/bigcodebench/Dockerfile.arm64 b/src/openbench/evals/bigcodebench/Dockerfile.arm64 index 3f7eecdc..602110e9 100644 --- a/src/openbench/evals/bigcodebench/Dockerfile.arm64 +++ b/src/openbench/evals/bigcodebench/Dockerfile.arm64 @@ -19,7 +19,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zip \ && rm -rf /var/lib/apt/lists/* -RUN python -m pip install --upgrade pip setuptools wheel +RUN python -m pip install --upgrade \ + pip==26.2.1 \ + setuptools==80.9.0 \ + wheel==0.47.0 RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench \ && cd /bigcodebench \ @@ -39,6 +42,7 @@ RUN python -m pip install --no-cache-dir \ Flask==3.0.3 \ Flask-Login==0.6.3 \ Flask-Mail==0.9.1 \ + Flask-WTF==1.2.1 \ gensim==4.3.2 \ geopandas==0.13.2 \ Levenshtein==0.25.0 \ From 42c86c2abd29fe6530663090c1d2b3530a991c84 Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 7 Aug 2026 10:34:13 +0800 Subject: [PATCH 4/6] fix: route BigCodeBench numba cache to tmpfs --- src/openbench/evals/bigcodebench/compose.arm64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/openbench/evals/bigcodebench/compose.arm64.yaml b/src/openbench/evals/bigcodebench/compose.arm64.yaml index e6e60117..c9655903 100644 --- a/src/openbench/evals/bigcodebench/compose.arm64.yaml +++ b/src/openbench/evals/bigcodebench/compose.arm64.yaml @@ -13,6 +13,7 @@ services: read_only: true environment: MPLCONFIGDIR: /tmp/matplotlib + NUMBA_CACHE_DIR: /tmp/numba tmpfs: - /app:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=2147483648 - /tmp:rw,nosuid,nodev,uid=1000,gid=1000,mode=0700,size=4294967296 From 200492c29109cdf6ada8e6dfe1ea70491a590d97 Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 7 Aug 2026 11:25:52 +0800 Subject: [PATCH 5/6] fix: harden BigCodeBench arm64 evaluation --- benchmark/wave2-code/bigcodebench.md | 58 +++++-- benchmark/wave2-code/codeforces_elo.md | 40 +++++ benchmark/wave2-code/matrix.tsv | 4 +- benchmark/wave2-code/tir_bench.md | 53 +++++++ packages/openbench-core/pyproject.toml | 6 + pyproject.toml | 7 +- src/openbench/datasets/livebench.py | 4 +- .../evals/bigcodebench/Dockerfile.arm64 | 66 ++------ .../evals/bigcodebench/bigcodebench.py | 8 +- .../bigcodebench/requirements-arm64.lock | 143 ++++++++++++++++++ src/openbench/scorers/bigcodebench.py | 40 +++-- src/openbench/scorers/bigcodebench_runner.py | 19 ++- tests/test_bigcodebench.py | 24 ++- 13 files changed, 383 insertions(+), 89 deletions(-) create mode 100644 benchmark/wave2-code/codeforces_elo.md create mode 100644 benchmark/wave2-code/tir_bench.md create mode 100644 src/openbench/evals/bigcodebench/requirements-arm64.lock diff --git a/benchmark/wave2-code/bigcodebench.md b/benchmark/wave2-code/bigcodebench.md index e6674748..d666e241 100644 --- a/benchmark/wave2-code/bigcodebench.md +++ b/benchmark/wave2-code/bigcodebench.md @@ -3,10 +3,12 @@ ## Decision BigCodeBench now has an OpenBench adapter and an arm64 source-equivalent scorer -image that passes a local Docker runner smoke on Apple Silicon. The pinned -official evaluator image is still `linux/amd64` only, so official-image parity -remains blocked until it is smoke-tested on a usable native `linux/amd64` Docker -host. +image validated against all 148 canonical Hard records on Apple Silicon. It +passes 145 records under the hardened no-network sandbox; the remaining three +require live external downloads and therefore fail closed instead of being +counted as model errors. The pinned official evaluator image is still +`linux/amd64` only, so official-image parity remains blocked until it is +smoke-tested on a usable native `linux/amd64` Docker host. ## Canonical sources @@ -20,14 +22,15 @@ host. - Official evaluator image: `bigcodebench/bigcodebench-evaluate` manifest `sha256:1327bddf60be9bc241648c59e6060cac4ca50248a0588ab735cd0200b17cc8c2` for `linux/amd64`. -- License: MIT. +- License: Apache-2.0. ## Implemented OpenBench surface - Registry ID: `bigcodebench`. - Parameters: `split="complete" | "instruct"`, `subset="full" | "hard"`, `runtime="auto" | "official" | "arm64"`, optional `limit`, `epochs`, and - `total_timeout`. + `total_timeout`. The source-equivalent arm64 runtime currently accepts only + `subset="hard"`; the unvalidated full subset fails at task construction. - Dataset loader: immutable Hugging Face revisions, with hidden execution fields resolved only at score time. - Prompting: follows BigCodeBench's OpenAI/API chat backend wrapper by applying @@ -36,6 +39,8 @@ host. - Scoring: sandbox runner uses BigCodeBench's own `sanitize`, `trusted_check`, and `untrusted_check`. Canonical solution timing is computed per task and used to calibrate the generated solution timeout, matching the upstream evaluator. + If canonical calibration fails, the scorer raises an evaluation error rather + than silently assigning an incorrect model score. - Docker: `runtime="official"` pins the official `linux/amd64` evaluator image; `runtime="arm64"` builds OpenBench's source-equivalent scorer image from the pinned upstream source commit for Apple Silicon; `runtime="auto"` selects @@ -51,7 +56,7 @@ source .venv/bin/activate && ruff check src/openbench/datasets/bigcodebench.py s All checks passed! source .venv/bin/activate && pytest tests/test_bigcodebench.py tests/test_registry.py -21 passed +23 passed source .venv/bin/activate && mypy src/openbench/datasets/bigcodebench.py src/openbench/scorers/bigcodebench.py src/openbench/scorers/bigcodebench_runner.py src/openbench/evals/bigcodebench tests/test_bigcodebench.py Success: no issues found in 6 source files @@ -74,10 +79,33 @@ docker compose -f src/openbench/evals/bigcodebench/compose.arm64.yaml -p openben compose-imports-ok ``` -The arm64 image intentionally does not install the full upstream -`requirements-eval.txt`, because several old scientific pins are amd64-oriented -and brittle on arm64. It installs the pinned BigCodeBench source plus the -minimal dependencies required by OpenBench's official sanitizer/evaluator path. +The arm64 image intentionally omits BigCodeBench's generation-only API clients +and vLLM dependency because OpenBench generates responses outside the scorer +container. Its evaluator dependencies are fully resolved in +`requirements-arm64.lock`; the Python base image, BigCodeBench source commit, +toolchain, direct/transitive Python packages, and NLTK asset hashes are pinned. + +## Canonical Hard preflight + +The source-equivalent image was iterated against all 148 canonical Hard records. +The machine-readable experiment log remains outside Git at +`/tmp/openbench-bcb-arm64-validation/results.tsv`; prompts, tests, and canonical +solutions are not included in the log. + +| Image state | Canonical pass rate | Wall time | +| --- | ---: | ---: | +| Minimal scorer baseline | 79/148 (53.38%) | 170.22 s | +| First dependency layer | 133/148 (89.86%) | 607.98 s | +| Second dependency/NLTK layer | 143/148 (96.62%) | 422.87 s | +| Restored legacy imports | 144/148 (97.30%) | 437.43 s | +| Hardened tmpfs caches | 145/148 (97.97%) | 500.89 s | + +The final three failures are `BigCodeBench/101`, `BigCodeBench/590`, and +`BigCodeBench/1012`. Their canonical solutions access live resources hosted by +CMU, Wikibooks, Google Drive, or learningcontainer. OpenBench keeps networking +disabled for untrusted generated programs and does not replace these resources +with invented fixtures. These samples therefore produce a canonical validation +error and no model score. Blocked locally: @@ -107,6 +135,8 @@ installation was explicitly approved for this validation attempt. For strict upstream-image parity, run the official Docker scorer smoke on a native `linux/amd64` host with Docker privileges sufficient for image extraction -and container creation. Until that passes, BigCodeBench should be described as -implemented with an arm64 source-equivalent smoke, not as validated against the -official amd64 image. +and container creation. A faithful full 148-task hardened run additionally needs +immutable, audited fixtures for the three live-network records; enabling network +for generated programs is not an acceptable fallback. Until both requirements +are satisfied, BigCodeBench should be described as arm64 Hard 145/148 validated, +not as full official-image parity. diff --git a/benchmark/wave2-code/codeforces_elo.md b/benchmark/wave2-code/codeforces_elo.md new file mode 100644 index 00000000..8f2e92f7 --- /dev/null +++ b/benchmark/wave2-code/codeforces_elo.md @@ -0,0 +1,40 @@ +# Codeforces ELO audit + +## Decision + +`Codeforces ELO` is recorded as unsupported because the cited Gemma 4 artifact +publishes final rating values rather than a fixed benchmark protocol. No task +window, contest/problem manifest, submission policy, judge environment, rating +calculation, or canonical evaluation repository is linked from the model card. + +## Source evidence + +- Model card: `google/gemma-4-E4B-it` +- Model-card revision inspected: + `ee0ef6023621cff504d758262d4e04895a5af4a2` +- The benchmark table contains one `Codeforces ELO` row with final values for six + Gemma variants. +- The card contains no other `Codeforces ELO` occurrence and supplies no + methodology note or source link for that row. + +An exact-name GitHub search found downstream catalog/ranking references, but no +repository identified as the Gemma 4 evaluator. LiveOIBench and similarly named +competitive-programming evaluations are distinct protocols and cannot be used +as silent substitutes. + +## Missing reproducibility contract + +ELO is a derived rating, not a dataset-level metric. Reproducing it requires at +least: + +- an immutable problem/contest/date window; +- language, compiler, time, memory, and submission limits; +- prompt and code-extraction rules; +- sample count and generation settings; +- the online/offline judge and hidden test assets; +- the opponent/reference population and exact rating update formula. + +Without those pieces, an OpenBench `codeforces_elo` ID would assign the same name +to an independently invented evaluation whose score is not comparable to the +published row. The candidate remains blocked pending a canonical protocol from +the benchmark publisher. diff --git a/benchmark/wave2-code/matrix.tsv b/benchmark/wave2-code/matrix.tsv index 8eff8eca..6ffb446f 100644 --- a/benchmark/wave2-code/matrix.tsv +++ b/benchmark/wave2-code/matrix.tsv @@ -2,10 +2,12 @@ candidate case rep metric resource status notes ojbench registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 ojbench official_protocol_audit 464_prompts_232_problems 0 dmoj+g++17+pypy3+git_lfs_testdata unsupported Official OJBench repo commit 5e94480b1e135b98855cf5bc81213c256aff5b17 and HF testdata HEAD 61cf9986f22c25d08e1657b03742124099c74353 expose 464 prompts with sha256 bcc8c94eb1fefb856355aa8b5a3e20cc0a2112f5436c5d83ab686edb417bce2c, but faithful judging requires DMOJ 4.1.0 at judge-server commit f098cd3a49a60186d1fadde5132329ec5f4f2213 plus g++17/PyPy3 and LFS problem zips; no hardened OpenBench Docker image has been validated for DMOJ under cap-drop/no-network, and OpenCompass only loads prompts without scoring. tir_bench registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 +tir_bench official_protocol_audit 1215_examples_13_tasks 0 agentic_image_harness+gpt4.1_extractor unsupported Official repository commit f79c7562b59e4f8142b0437fc725eb3ee1aec76c and HF dataset revision ae9976e81e86c4797fd0d28f9119393b37d9d006 publish data and post-generation scoring, but the released scripts require a GPT-4.1 answer extractor and do not publish the agentic image/tool generation harness underlying with-CI/without-CI runs. codeforces_elo registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 +codeforces_elo model_card_protocol_audit 6_reported_model_ratings 0 missing_problem_window+judge+rating_formula unsupported Gemma 4 model card google/gemma-4-E4B-it revision ee0ef6023621cff504d758262d4e04895a5af4a2 publishes one Codeforces ELO row but no immutable problem manifest, generation protocol, judge environment, opponent population, rating formula, or canonical evaluator source; downstream competitive-programming benchmarks are not interchangeable. livebench registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 bigcodebench registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 -bigcodebench openbench_adapter 1140_full+148_hard 0 arm64_source_equiv+x86_official arm64_smoke_passed_official_amd64_blocked Implemented registry ID bigcodebench for official v0.1.4 complete/instruct and full/hard axes using HF dataset revisions b74c0d0bf70d2c0bc459be537895cca163007f1a and 298d2cc7b96612e15e47313c3603ee124cee0c1f plus pinned official amd64 evaluator image bigcodebench/bigcodebench-evaluate@sha256:1327bddf60be9bc241648c59e6060cac4ca50248a0588ab735cd0200b17cc8c2; OpenBench now also provides a source-equivalent arm64 scorer image built from upstream commit 09dd993f46c3fbf3a799465bb96d524edcb0b199, with Docker runner smoke passing on BigCodeBench/13 under network none/read-only/cap-drop/no-new-privileges. Unit tests, ruff, targeted mypy, and compose arm64 import smoke pass; official amd64 image remains unvalidated locally because it segfaults under QEMU on arm64 and the x86_64 GPU instance disallows Docker layer registration/unshare. +bigcodebench openbench_adapter 1140_full+148_hard 0 arm64_hard_145_of_148+x86_official arm64_network_safe_validated_official_amd64_blocked Implemented registry ID bigcodebench for official v0.1.4 complete/instruct and full/hard axes using HF dataset revisions b74c0d0bf70d2c0bc459be537895cca163007f1a and 298d2cc7b96612e15e47313c3603ee124cee0c1f plus pinned official amd64 evaluator image bigcodebench/bigcodebench-evaluate@sha256:1327bddf60be9bc241648c59e6060cac4ca50248a0588ab735cd0200b17cc8c2. The pinned source-equivalent arm64 image passes 145/148 canonical Hard records under network-none/read-only/cap-drop/no-new-privileges; records 101, 590, and 1012 require live external downloads and fail canonical calibration without producing model scores. Official amd64 image remains unvalidated locally because it segfaults under QEMU on arm64 and the x86_64 GPU instance disallows Docker layer registration/unshare. spider registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 spider official_protocol_audit 1034_dev 0 5.15GB_testsuite+missing_llm_prompt unsupported Official Spider repo commit b7b5b8c890cd30e35427348bb9eb8c6d1350ca7c and official test-suite-sql-eval commit e97acc546ecbee8fa27fa8dbf025ef61493a876c define data and test-suite execution accuracy, but no canonical LLM prompt/sampling protocol is published; the required test-suite DB archive is external Google Drive data sha256 9ec24ea8debc6bd04abfe137b5f1a739b5a8836f32c0464e4dfc94eb7f41da96, 1.2GB compressed and 5.15GB uncompressed, so OpenBench should not report a hand-prompted approximation. vita_bench registry_baseline 1 0 0 unsupported No implementation or registry ID at baseline 7f11867 diff --git a/benchmark/wave2-code/tir_bench.md b/benchmark/wave2-code/tir_bench.md new file mode 100644 index 00000000..4269de64 --- /dev/null +++ b/benchmark/wave2-code/tir_bench.md @@ -0,0 +1,53 @@ +# TIR-Bench audit + +## Decision + +TIR-Bench now has a public canonical repository and dataset, so the earlier +"source unavailable" assessment is obsolete. It remains unsupported in this +OpenBench wave because the released code scores pre-generated response files; +it does not publish the agentic image-manipulation harness needed to reproduce +the reported with-CI/without-CI model runs. Its answer extraction also requires +a separately configured GPT-4.1 judge. + +## Canonical sources + +- Repository: `https://github.com/agents-x-project/TIR-Bench` +- Repository commit inspected: + `f79c7562b59e4f8142b0437fc725eb3ee1aec76c` +- Dataset: `Agents-X/TIR-Bench` +- Dataset revision: `ae9976e81e86c4797fd0d28f9119393b37d9d006` +- Dataset license: Apache-2.0. +- Published scope: 1,215 examples across 13 image-reasoning tasks. + +The repository itself does not include a license file. The Apache-2.0 label +above comes from the immutable Hugging Face dataset metadata and should not be +assumed to license the repository code. + +## Released evaluation path + +The repository provides two post-generation stages: + +1. `extract_answer.py` reads a JSON result file, asserts that it contains 1,215 + entries, and calls an OpenAI-compatible GPT-4.1 endpoint with task-specific + few-shot extraction prompts. +2. `calculate_score.py` applies deterministic task-specific comparisons to the + extracted answers, including exact choice/integer/float checks, list IoU, + jigsaw-position accuracy, OCR substring checks, Levenshtein normalization, + and `math_verify` fallbacks. + +The scripts expect model responses to exist already. They do not define how the +evaluated model receives images, creates or invokes image-processing tools, +iterates over intermediate images, limits tool calls, or converts that trajectory +into the final response. The Qwen3.5 model card reports TIR-Bench as "with CI / +without CI", but neither that card nor this repository defines a reproducible CI +runtime. + +## OpenBench compatibility finding + +A faithful integration needs two missing protocol components: the canonical +agent/tool generation harness and explicit authority/configuration for the +GPT-4.1 extraction judge. A static VQA prompt would measure a different task, +while substituting a hand-authored deterministic extractor would change the +published metric. OpenBench therefore records the public data and scorer but +does not expose a `tir_bench` task until those choices can be reproduced or made +explicit in the benchmark identity. diff --git a/packages/openbench-core/pyproject.toml b/packages/openbench-core/pyproject.toml index d2d13b5a..333fc07d 100644 --- a/packages/openbench-core/pyproject.toml +++ b/packages/openbench-core/pyproject.toml @@ -50,6 +50,12 @@ include = ["openbench*"] [tool.setuptools.package-data] "openbench.evals.livecodebench" = ["Dockerfile", "compose.yaml"] +"openbench.evals.bigcodebench" = [ + "Dockerfile.arm64", + "compose.yaml", + "compose.arm64.yaml", + "requirements-arm64.lock", +] "openbench.evals.evalplus" = ["Dockerfile", "compose.yaml"] "openbench.evals.bfcl" = [ "Dockerfile", diff --git a/pyproject.toml b/pyproject.toml index 33f1203d..a3171a31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,12 @@ include = ["openbench*"] [tool.setuptools.package-data] "openbench.evals.livecodebench" = ["Dockerfile", "compose.yaml"] -"openbench.evals.bigcodebench" = ["Dockerfile.arm64", "compose.yaml", "compose.arm64.yaml"] +"openbench.evals.bigcodebench" = [ + "Dockerfile.arm64", + "compose.yaml", + "compose.arm64.yaml", + "requirements-arm64.lock", +] "openbench.evals.evalplus" = ["Dockerfile", "compose.yaml"] "openbench.evals.bfcl" = [ "Dockerfile", diff --git a/src/openbench/datasets/livebench.py b/src/openbench/datasets/livebench.py index 0eb00c4e..5c4535a9 100644 --- a/src/openbench/datasets/livebench.py +++ b/src/openbench/datasets/livebench.py @@ -7,8 +7,8 @@ from pathlib import Path from typing import Any -import pyarrow as pa -import pyarrow.parquet as pq +import pyarrow as pa # type: ignore[import-untyped] +import pyarrow.parquet as pq # type: ignore[import-untyped] from huggingface_hub import hf_hub_download from inspect_ai.dataset import Dataset, MemoryDataset, Sample diff --git a/src/openbench/evals/bigcodebench/Dockerfile.arm64 b/src/openbench/evals/bigcodebench/Dockerfile.arm64 index 602110e9..fc2fd698 100644 --- a/src/openbench/evals/bigcodebench/Dockerfile.arm64 +++ b/src/openbench/evals/bigcodebench/Dockerfile.arm64 @@ -1,4 +1,4 @@ -FROM python:3.10-slim +FROM python:3.10-slim@sha256:34a2c9467a0231d8c29a5ecadc219733a9393b026882b44d91616b9dae6088b6 ARG BIGCODEBENCH_COMMIT=09dd993f46c3fbf3a799465bb96d524edcb0b199 @@ -29,57 +29,21 @@ RUN git clone https://github.com/bigcode-project/bigcodebench.git /bigcodebench && git checkout "${BIGCODEBENCH_COMMIT}" \ && python -m pip install . --no-deps -# Source-equivalent scorer dependencies exercised by the canonical Hard suite. -# Upstream pins that conflict on Python 3.10/arm64 are adjusted together: -# numpy 1.23.5 satisfies TensorFlow 2.11, while scipy 1.10.1 provides an arm64 -# wheel compatible with scikit-learn, statsmodels, and the selected NumPy. +# Fully resolved from the arm64 image that passed the canonical Hard preflight. +# The source package is installed separately above because its generation-only +# dependencies (API clients and vLLM) are not part of the evaluator runtime. +COPY requirements-arm64.lock /tmp/requirements-arm64.lock RUN python -m pip install --no-cache-dir \ - appdirs==1.4.4 \ - beautifulsoup4==4.8.2 \ - chardet==5.2.0 \ - datasets==2.17.0 \ - Faker==20.1.0 \ - Flask==3.0.3 \ - Flask-Login==0.6.3 \ - Flask-Mail==0.9.1 \ - Flask-WTF==1.2.1 \ - gensim==4.3.2 \ - geopandas==0.13.2 \ - Levenshtein==0.25.0 \ - librosa==0.10.1 \ - lxml==4.9.3 \ - matplotlib==3.7.0 \ - multipledispatch==1.0.0 \ - nltk==3.8 \ - numba==0.57.1 \ - numpy==1.23.5 \ - opencv-python-headless==4.9.0.80 \ - openpyxl==3.1.2 \ - pqdm==0.2.0 \ - psutil==5.9.5 \ - pyarrow==14.0.1 \ - pyquery==1.4.3 \ - pycryptodome==3.14.1 \ - pytesseract==0.3.10 \ - python-docx==1.1.0 \ - rsa==4.9 \ - scikit-learn==1.3.1 \ - scipy==1.10.1 \ - seaborn==0.13.2 \ - soundfile==0.12.1 \ - statsmodels==0.14.0 \ - tempdir==0.7.1 \ - tensorflow==2.11.0 \ - termcolor==3.3.0 \ - tqdm==4.70.0 \ - tree-sitter==0.23.0 \ - tree-sitter-python==0.23.2 \ - wget==3.2 \ - wordcloud==1.9.3 \ - xlrd==2.0.1 \ - xlwt==1.3.0 - -RUN python -m nltk.downloader -d /usr/local/share/nltk_data punkt stopwords + --requirement /tmp/requirements-arm64.lock \ + && rm /tmp/requirements-arm64.lock + +RUN python -m nltk.downloader -d /usr/local/share/nltk_data punkt stopwords \ + && printf '%s %s\n%s %s\n' \ + '51c3078994aeaf650bfc8e028be4fb42b4a0d177d41c012b6a983979653660ec' \ + '/usr/local/share/nltk_data/tokenizers/punkt.zip' \ + '48c0e52d8b52546e827f53761fb30300c0ab94f70660d28bd65ba0a86270946b' \ + '/usr/local/share/nltk_data/corpora/stopwords.zip' \ + | sha256sum --check --strict RUN useradd --create-home --uid 1000 bigcodebenchuser \ && mkdir --parents /app \ diff --git a/src/openbench/evals/bigcodebench/bigcodebench.py b/src/openbench/evals/bigcodebench/bigcodebench.py index 58f6cc2c..4ddc05c3 100644 --- a/src/openbench/evals/bigcodebench/bigcodebench.py +++ b/src/openbench/evals/bigcodebench/bigcodebench.py @@ -53,6 +53,12 @@ def bigcodebench( if epochs <= 0: raise ValueError("epochs must be positive") + compose_path = compose_path_for_runtime(runtime) + if compose_path == ARM64_COMPOSE_PATH and subset != "hard": + raise ValueError( + "The arm64 BigCodeBench runtime is validated only for subset='hard'; " + "use runtime='official' for subset='full'" + ) reducers = ["mean", "pass_at_1"] if epochs >= 5: reducers.append("pass_at_5") @@ -64,7 +70,7 @@ def bigcodebench( dataset=get_bigcodebench_dataset(split=split, subset=subset, limit=limit), solver=generate(), scorer=bigcodebench_scorer(total_timeout=total_timeout), - sandbox=("docker", str(compose_path_for_runtime(runtime))), + sandbox=("docker", str(compose_path)), epochs=Epochs(epochs, reducer=reducers), config=GenerateConfig( temperature=0, diff --git a/src/openbench/evals/bigcodebench/requirements-arm64.lock b/src/openbench/evals/bigcodebench/requirements-arm64.lock new file mode 100644 index 00000000..f4ed477c --- /dev/null +++ b/src/openbench/evals/bigcodebench/requirements-arm64.lock @@ -0,0 +1,143 @@ +absl-py==2.5.0 +aiohappyeyeballs==2.7.1 +aiohttp==3.14.3 +aiosignal==1.4.0 +anyio==4.14.2 +appdirs==1.4.4 +astunparse==1.6.3 +async-timeout==5.0.1 +attrs==26.1.0 +audioread==3.1.0 +beautifulsoup4==4.8.2 +blinker==1.9.0 +bounded-pool-executor==0.0.3 +certifi==2026.7.22 +cffi==2.1.1 +chardet==5.2.0 +charset-normalizer==3.4.9 +click-plugins==1.1.1.2 +click==8.4.2 +cligj==0.7.2 +contourpy==1.3.2 +cryptography==50.0.0 +cssselect==1.5.0 +cycler==0.12.1 +datasets==2.17.0 +decorator==5.3.1 +dill==0.3.8 +et_xmlfile==2.0.0 +exceptiongroup==1.3.1 +Faker==20.1.0 +filelock==3.32.2 +fiona==1.10.1 +Flask-Login==0.6.3 +Flask-Mail==0.9.1 +Flask-WTF==1.2.1 +Flask==3.0.3 +flatbuffers==25.12.19 +fonttools==4.63.0 +frozenlist==1.8.0 +fsspec==2023.10.0 +gast==0.4.0 +gensim==4.3.2 +geopandas==0.13.2 +google-auth-oauthlib==0.4.6 +google-auth==2.56.3 +google-pasta==0.2.0 +grpcio==1.83.0 +h11==0.16.0 +h5py==3.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.26.1 +idna==3.18 +itsdangerous==2.2.0 +Jinja2==3.1.6 +joblib==1.5.3 +keras==2.11.0 +kiwisolver==1.5.0 +lazy-loader==0.5 +Levenshtein==0.25.0 +libclang==18.1.1 +librosa==0.10.1 +llvmlite==0.40.1 +lxml==4.9.3 +Markdown==3.10.3 +MarkupSafe==3.0.3 +matplotlib==3.7.0 +msgpack==1.2.1 +multidict==6.7.1 +multipledispatch==1.0.0 +multiprocess==0.70.16 +nltk==3.8 +numba==0.57.1 +numpy==1.23.5 +oauthlib==3.3.1 +opencv-python-headless==4.9.0.80 +openpyxl==3.1.2 +opt_einsum==3.4.0 +packaging==26.3 +pandas==2.3.3 +patsy==1.0.2 +pillow==12.3.0 +platformdirs==4.11.0 +pooch==1.9.0 +pqdm==0.2.0 +propcache==0.5.2 +protobuf==3.19.6 +psutil==5.9.5 +pyarrow-hotfix==0.7 +pyarrow==14.0.1 +pyasn1==0.6.4 +pyasn1_modules==0.4.2 +pycparser==3.0 +pycryptodome==3.14.1 +pyparsing==3.3.2 +pyproj==3.7.1 +pyquery==1.4.3 +pytesseract==0.3.10 +python-dateutil==2.9.0.post0 +python-docx==1.1.0 +pytz==2026.3.post1 +PyYAML==6.0.3 +RapidFuzz==3.14.5 +regex==2026.7.19 +requests-oauthlib==2.0.0 +requests==2.34.2 +rsa==4.9 +scikit-learn==1.3.1 +scipy==1.10.1 +seaborn==0.13.2 +shapely==2.1.2 +six==1.17.0 +smart_open==8.0.1 +soundfile==0.12.1 +soupsieve==2.9.2 +soxr==1.1.0 +statsmodels==0.14.0 +tempdir==0.7.1 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.11.2 +tensorflow-cpu-aws==2.11.0 +tensorflow-estimator==2.11.0 +tensorflow-io-gcs-filesystem==0.37.1 +tensorflow==2.11.0 +termcolor==3.3.0 +threadpoolctl==3.6.0 +tqdm==4.70.0 +tree-sitter-python==0.23.2 +tree-sitter==0.23.0 +typing_extensions==4.16.0 +tzdata==2026.3 +urllib3==2.7.0 +Werkzeug==3.1.8 +wget==3.2 +wordcloud==1.9.3 +wrapt==2.3.0 +WTForms==3.2.2 +xlrd==2.0.1 +xlwt==1.3.0 +xxhash==3.8.1 +yarl==1.24.5 diff --git a/src/openbench/scorers/bigcodebench.py b/src/openbench/scorers/bigcodebench.py index 889826e8..71955af4 100644 --- a/src/openbench/scorers/bigcodebench.py +++ b/src/openbench/scorers/bigcodebench.py @@ -52,6 +52,29 @@ def build_bigcodebench_payload( } +def score_bigcodebench_evaluation(evaluation: dict[str, Any], task_id: str) -> Score: + """Convert a runner result while refusing uncalibrated canonical failures.""" + + status = evaluation.get("status", "unknown") + if status == "canonical_error": + raise RuntimeError( + f"BigCodeBench canonical validation failed for {task_id}; " + "refusing to report a model score" + ) + + passed = evaluation.get("passed") is True + explanation = ( + "Passed official BigCodeBench tests." + if passed + else f"Failed official BigCodeBench tests with status: {status}." + ) + return Score( + value=CORRECT if passed else INCORRECT, + answer=evaluation.get("solution", ""), + explanation=explanation, + ) + + @scorer(metrics=[accuracy(), stderr()]) def bigcodebench_scorer( *, @@ -121,20 +144,9 @@ async def score(state: TaskState, target: Target) -> Score: explanation="BigCodeBench runner returned an invalid result.", ) - status = evaluation.get("status", "unknown") - passed = evaluation.get("passed") is True - canonical_time = evaluation.get("canonical_time") - explanation = ( - "Passed official BigCodeBench tests." - if passed - else f"Failed official BigCodeBench tests with status: {status}." - ) - if canonical_time is None: - explanation += " Canonical timing failed; used official fallback timeout." - return Score( - value=CORRECT if passed else INCORRECT, - answer=evaluation.get("solution", ""), - explanation=explanation, + return score_bigcodebench_evaluation( + evaluation, + task_id=str(state.metadata["task_id"]), ) return score diff --git a/src/openbench/scorers/bigcodebench_runner.py b/src/openbench/scorers/bigcodebench_runner.py index 7bc52a65..52e020d6 100644 --- a/src/openbench/scorers/bigcodebench_runner.py +++ b/src/openbench/scorers/bigcodebench_runner.py @@ -30,7 +30,20 @@ def main() -> None: payload["min_time_limit"], ) canonical_time = canonical["time"] - status, details = untrusted_check( + if canonical_time is None: + print( + json.dumps( + { + "passed": False, + "status": "canonical_error", + "details": {}, + "canonical_time": None, + "solution": solution, + } + ) + ) + return + status, _details = untrusted_check( solution, payload["test"], payload["entry_point"], @@ -38,14 +51,14 @@ def main() -> None: payload["max_data_limit"], payload["max_stack_limit"], payload["min_time_limit"], - canonical_time if canonical_time is not None else 20, + canonical_time, ) print( json.dumps( { "passed": status == PASS, "status": status, - "details": details, + "details": {}, "canonical_time": canonical_time, "solution": solution, } diff --git a/tests/test_bigcodebench.py b/tests/test_bigcodebench.py index 9964f072..02e18077 100644 --- a/tests/test_bigcodebench.py +++ b/tests/test_bigcodebench.py @@ -16,7 +16,10 @@ ) from openbench.evals.bigcodebench import bigcodebench from openbench.evals.bigcodebench.bigcodebench import compose_path_for_runtime -from openbench.scorers.bigcodebench import build_bigcodebench_payload +from openbench.scorers.bigcodebench import ( + build_bigcodebench_payload, + score_bigcodebench_evaluation, +) def _record(**overrides): @@ -100,7 +103,7 @@ def test_task_uses_official_greedy_generation_settings(): "openbench.evals.bigcodebench.bigcodebench.get_bigcodebench_dataset", return_value=dataset, ): - task = bigcodebench(epochs=10, limit=1) + task = bigcodebench(epochs=10, limit=1, runtime="official") assert task.name == "bigcodebench_instruct_full" assert task.epochs == 10 @@ -129,6 +132,11 @@ def test_bigcodebench_runtime_selects_expected_compose_files(): compose_path_for_runtime("bad") # type: ignore[arg-type] +def test_arm64_runtime_rejects_unvalidated_full_subset(): + with pytest.raises(ValueError, match="validated only for subset='hard'"): + bigcodebench(subset="full", runtime="arm64") + + def test_build_bigcodebench_payload_preserves_official_execution_fields(): payload = build_bigcodebench_payload( "```python\ndef task_func(): return 1\n```", @@ -149,3 +157,15 @@ def test_build_bigcodebench_payload_preserves_official_execution_fields(): assert payload["task_id"] == "BigCodeBench/0" assert payload["entry_point"] == "task_func" assert payload["calibrated"] is True + + +def test_canonical_failure_refuses_to_report_model_score(): + with pytest.raises(RuntimeError, match="refusing to report a model score"): + score_bigcodebench_evaluation( + { + "passed": False, + "status": "canonical_error", + "canonical_time": None, + }, + task_id="BigCodeBench/101", + ) From fccf47b540f017c1c3296618399681746ae8fb82 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 7 Aug 2026 03:27:11 +0000 Subject: [PATCH 6/6] chore: update benchmark docs [skip ci] --- docs/snippets/benchmarks.data.mdx | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/snippets/benchmarks.data.mdx b/docs/snippets/benchmarks.data.mdx index a4c2dd46..e9f098e0 100644 --- a/docs/snippets/benchmarks.data.mdx +++ b/docs/snippets/benchmarks.data.mdx @@ -4059,6 +4059,20 @@ export const benchmarksData = [ "function_name": "bigbench_winowhy", "is_alpha": false }, + { + "name": "BigCodeBench", + "description": "Practical Python code generation with diverse function calls and complex instructions", + "category": "core", + "tags": [ + "coding", + "generation", + "execution", + "docker", + "python" + ], + "function_name": "bigcodebench", + "is_alpha": false + }, { "name": "BoolQ", "description": "BoolQ: A Question Answering Dataset for Boolean Reasoning", @@ -5508,6 +5522,20 @@ export const benchmarksData = [ "function_name": "legalsupport", "is_alpha": false }, + { + "name": "LiveBench Coding 2024-11-25", + "description": "Latest fully public LiveBench contamination-free coding release", + "category": "core", + "tags": [ + "coding", + "generation", + "execution", + "docker", + "live" + ], + "function_name": "livebench_coding_2024_11_25", + "is_alpha": false + }, { "name": "LiveCodeBench v6", "description": "Contamination-aware Python code generation through the April 2025 release",