Skip to content

[Klaud Cold] refactor: remove SWE-bench Lite eval suite #3506

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
functionstackx wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

[Klaud Cold] refactor: remove SWE-bench Lite eval suite #3506

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1de72f4
refactor: remove SWE-bench Lite eval suite
functionstackx Sep 26, 2026
41ea1b9
Stop passing MODAL_TOKEN_* to benchmark templates
functionstackx Sep 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 3 additions & 18 deletions .github/workflows/benchmark-multinode-tmpl.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,6 @@ on:
secrets:
INFERENCEX_OFFICIAL_RO_HF_TOKEN:
required: true
MODAL_TOKEN_ID:
required: false
MODAL_TOKEN_SECRET:
required: false
inputs:
config:
description: "Benchmark configuration as JSON"
Expand Down Expand Up @@ -52,12 +48,12 @@ on:
required: false
default: false
eval-framework:
description: "Eval runner (lm-eval, swebench, kimi-vendor, minimax-vendor, or bfcl)"
description: "Eval runner (lm-eval, kimi-vendor, minimax-vendor, or bfcl)"
type: string
required: false
default: "lm-eval"
eval-suite:
description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval"
type: string
required: false
default: ""
Expand All @@ -71,11 +67,6 @@ on:
required: false
type: string
default: ""
swebench-gen-mode:
description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
required: false
type: string
default: ""
require-power:
description: "Fail fixed-sequence result processing when GPU power is invalid"
type: boolean
Expand Down Expand Up @@ -156,11 +147,6 @@ env:
EVAL_SUITE: ${{ inputs.eval-suite }}
EVAL_CONC: ${{ inputs.eval-conc }}
EVAL_LIMIT: ${{ inputs.eval-limit }}
SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode || 'agentic' }}
# GPU/multi-node runners lack Docker for SWE-bench scoring.
SWEBENCH_USE_MODAL: 'true'
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }}
POWER_PRODUCER_SHA: ${{ inputs.power-producer-sha }}
# Agentic-coding env. Fixed-seq-len jobs leave these empty.
Expand Down Expand Up @@ -491,7 +477,6 @@ jobs:
sample*.jsonl
agent_preds.json
predictions.jsonl
swebench_report_*.json
*_artifacts.tar.gz
*.traj*
if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}
Expand All @@ -518,7 +503,7 @@ jobs:
rm -f -- ./*_results.jsonl || true
rm -f -- ./*_artifacts.tar.gz || true
rm -f sample*.jsonl || true
rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true
rm -f agent_preds.json predictions.jsonl *.traj* || true

- name: Slurm cleanup (post-run)
if: always()
Expand Down
21 changes: 3 additions & 18 deletions .github/workflows/benchmark-tmpl.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,6 @@ on:
secrets:
INFERENCEX_OFFICIAL_RO_HF_TOKEN:
required: true
MODAL_TOKEN_ID:
required: false
MODAL_TOKEN_SECRET:
required: false
inputs:
config:
description: "Benchmark configuration as JSON"
Expand Down Expand Up @@ -46,12 +42,12 @@ on:
required: false
default: false
eval-framework:
description: "Eval runner (lm-eval, swebench, kimi-vendor, minimax-vendor, or bfcl)"
description: "Eval runner (lm-eval, kimi-vendor, minimax-vendor, or bfcl)"
type: string
required: false
default: "lm-eval"
eval-suite:
description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval"
type: string
required: false
default: ""
Expand Down Expand Up @@ -89,11 +85,6 @@ on:
required: false
type: string
default: ""
swebench-gen-mode:
description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
required: false
type: string
default: ""
env:
PORT: '8888'
INFMAX_CONTAINER_WORKSPACE: '/workspace'
Expand Down Expand Up @@ -147,15 +138,10 @@ env:
REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }}
AIPERF_EXPERIMENTAL_FAST: ${{ inputs.agentx-fast && '1' || '0' }}
EVAL_LIMIT: ${{ inputs.eval-limit }}
SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode || 'agentic' }}
AIPERF_FAILED_REQUEST_THRESHOLD: '0.10'
RESULT_DIR: /workspace/results
PYTHONDONTWRITEBYTECODE: '1'
PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache
# GPU runners lack Docker for SWE-bench scoring.
SWEBENCH_USE_MODAL: 'true'
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

permissions:
contents: read
Expand Down Expand Up @@ -471,7 +457,6 @@ jobs:
sample*.jsonl
agent_preds.json
predictions.jsonl
swebench_report_*.json
*.traj*
if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}

Expand All @@ -494,7 +479,7 @@ jobs:
rm -f -- ./*_results.jsonl || true
rm -f sample*.jsonl || true
rm -f -- ./*_artifacts.tar.gz || true
rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true
rm -f agent_preds.json predictions.jsonl *.traj* || true

- name: Repair root-owned artifacts (post-run)
if: ${{ always() && inputs.runner == 'cluster:mi355x-amds' }}
Expand Down
54 changes: 16 additions & 38 deletions .github/workflows/e2e-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,17 +52,12 @@ on: # zizmor: ignore[concurrency-limits]
type: string
default: ""
eval-framework:
description: "Eval runner override (auto uses model-aware selection; lm-eval, swebench, kimi-vendor, minimax-vendor, or bfcl)"
description: "Eval runner override (auto uses model-aware selection; lm-eval, kimi-vendor, minimax-vendor, or bfcl)"
required: false
type: string
default: "auto"
eval-suite:
description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
required: false
type: string
default: ""
swebench-gen-mode:
description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval"
required: false
type: string
default: ""
Expand Down Expand Up @@ -105,10 +100,6 @@ on: # zizmor: ignore[concurrency-limits]
secrets:
INFERENCEX_OFFICIAL_RO_HF_TOKEN:
required: true
MODAL_TOKEN_ID:
required: false
MODAL_TOKEN_SECRET:
required: false
inputs:
generate-cli-command:
description: "Command passed to generate matrix script"
Expand Down Expand Up @@ -154,17 +145,12 @@ on: # zizmor: ignore[concurrency-limits]
type: string
default: ""
eval-framework:
description: "Eval runner override (auto uses model-aware selection; lm-eval, swebench, kimi-vendor, minimax-vendor, or bfcl)"
description: "Eval runner override (auto uses model-aware selection; lm-eval, kimi-vendor, minimax-vendor, or bfcl)"
required: false
type: string
default: "auto"
eval-suite:
description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
required: false
type: string
default: ""
swebench-gen-mode:
description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval"
required: false
type: string
default: ""
Expand Down Expand Up @@ -370,8 +356,7 @@ jobs:
config: ${{ fromJson(needs.get-jobs.outputs.multi-node-config) }}
secrets:
INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

with:
config: ${{ toJSON(matrix.config) }}
klaud-run: ${{ inputs.klaud-run }}
Expand All @@ -396,8 +381,7 @@ jobs:
config: ${{ fromJson(needs.get-jobs.outputs.multi-node-eval-config) }}
secrets:
INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

with:
config: ${{ toJSON(matrix.config) }}
klaud-run: ${{ inputs.klaud-run }}
Expand All @@ -412,7 +396,7 @@ jobs:
eval-limit: ${{ inputs.eval-limit }}
eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }}
eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }}
swebench-gen-mode: ${{ inputs.swebench-gen-mode }}

ref: ${{ needs.get-jobs.outputs.benchmark-ref }}

test-sweep-agentic:
Expand All @@ -426,8 +410,7 @@ jobs:
config: ${{ fromJson(needs.get-jobs.outputs.agentic-config) }}
secrets:
INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

with:
config: ${{ toJSON(matrix.config) }}
klaud-run: ${{ inputs.klaud-run }}
Expand All @@ -453,8 +436,7 @@ jobs:
config: ${{ fromJson(needs.get-jobs.outputs.agentic-eval-config) }}
secrets:
INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

with:
config: ${{ toJSON(matrix.config) }}
klaud-run: ${{ inputs.klaud-run }}
Expand All @@ -467,7 +449,7 @@ jobs:
run-eval: true
eval-only: true
eval-limit: ${{ inputs.eval-limit }}
swebench-gen-mode: ${{ inputs.swebench-gen-mode }}

eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }}
eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }}
scenario-type: agentic-coding
Expand All @@ -484,8 +466,7 @@ jobs:
config: ${{ fromJson(needs.get-jobs.outputs.multi-node-agentic-config) }}
secrets:
INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

with:
config: ${{ toJSON(matrix.config) }}
klaud-run: ${{ inputs.klaud-run }}
Expand Down Expand Up @@ -514,8 +495,7 @@ jobs:
config: ${{ fromJson(needs.get-jobs.outputs.multi-node-agentic-eval-config) }}
secrets:
INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

with:
config: ${{ toJSON(matrix.config) }}
klaud-run: ${{ inputs.klaud-run }}
Expand All @@ -530,7 +510,7 @@ jobs:
eval-only: true
eval-conc: ${{ matrix.config['eval-conc'] }}
eval-limit: ${{ inputs.eval-limit }}
swebench-gen-mode: ${{ inputs.swebench-gen-mode }}

eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }}
eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }}
scenario-type: agentic-coding
Expand All @@ -547,8 +527,7 @@ jobs:
config: ${{ fromJson(needs.get-jobs.outputs.single-node-config) }}
secrets:
INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

with:
config: ${{ toJSON(matrix.config) }}
klaud-run: ${{ inputs.klaud-run }}
Expand All @@ -571,8 +550,7 @@ jobs:
config: ${{ fromJson(needs.get-jobs.outputs.eval-config) }}
secrets:
INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}

with:
config: ${{ toJSON(matrix.config) }}
klaud-run: ${{ inputs.klaud-run }}
Expand All @@ -585,7 +563,7 @@ jobs:
eval-limit: ${{ inputs.eval-limit }}
eval-framework: ${{ inputs.eval-framework == 'auto' && (matrix.config['eval-framework'] || 'lm-eval') || inputs.eval-framework }}
eval-suite: ${{ inputs.eval-suite != '' && inputs.eval-suite || (inputs.eval-framework == 'auto' && matrix.config['eval-suite'] || '') }}
swebench-gen-mode: ${{ inputs.swebench-gen-mode }}

ref: ${{ needs.get-jobs.outputs.benchmark-ref }}

collect-results:
Expand Down
Loading
Loading