feat: validate BFCL through native APIs and guard eval parity / 通过原生 API 验证 BFCL 并保障跨硬件评估一致性

.github/workflows/benchmark-multinode-tmpl.yml

@@ -57,7 +57,7 @@ on:
         required: false
         default: "lm-eval"
       eval-suite:
-        description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
+        description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_responses_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
         type: string
         required: false
         default: ""
@@ -152,7 +152,7 @@ env:
   DISAGG: ${{ fromJSON(inputs.config).disagg }}
   PREFILL_HARDWARE: ${{ fromJSON(inputs.config).prefill.hardware }}
   DECODE_HARDWARE: ${{ fromJSON(inputs.config).decode.hardware }}
-  RUN_EVAL: ${{ inputs.run-eval }}
+  RUN_EVAL: ${{ inputs.run-eval && !(inputs.eval-framework == 'bfcl' && (fromJSON(inputs.config).framework == 'trt' || fromJSON(inputs.config).framework == 'dynamo-trt')) }}
   EVAL_ONLY: ${{ inputs.eval-only }}
   EVAL_FRAMEWORK: ${{ inputs.eval-framework }}
   EVAL_SUITE: ${{ inputs.eval-suite }}
@@ -202,6 +202,8 @@ permissions:
 
 jobs:
   benchmark:
+    # BFCL is disabled for TRT; skip eval-only jobs before reserving a GPU runner.
+    if: ${{ !(inputs.eval-only && inputs.eval-framework == 'bfcl' && (fromJSON(inputs.config).framework == 'trt' || fromJSON(inputs.config).framework == 'dynamo-trt')) }}
     runs-on: >-
       ${{ fromJSON(
         vars.PRIORITY_SCHEDULER_ENABLED == 'true' &&
@@ -517,7 +519,7 @@ jobs:
           "$INFERENCEX_RESULTS_PYTHON" -P -m infx.evals.validate_scores --expected-concs "${expected_concs}"
 
       - name: Cleanup eval outputs (post-upload)
-        if: ${{ always() && (inputs.run-eval || inputs.eval-only) }}
+        if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
         run: |
           if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
           rm -f meta_env.json || true

.github/workflows/benchmark-tmpl.yml

@@ -51,7 +51,7 @@ on:
         required: false
         default: "lm-eval"
       eval-suite:
-        description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
+        description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_responses_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
         type: string
         required: false
         default: ""
@@ -131,7 +131,7 @@ env:
   CONC: ${{ fromJSON(inputs.config).conc }}
   SPEC_DECODING: ${{ fromJSON(inputs.config).spec-decoding }}
   DISAGG: ${{ inputs.scenario-type == 'agentic-coding' && 'false' || fromJSON(inputs.config).disagg }}
-  RUN_EVAL: ${{ inputs.run-eval }}
+  RUN_EVAL: ${{ inputs.run-eval && !(inputs.eval-framework == 'bfcl' && (fromJSON(inputs.config).framework == 'trt' || fromJSON(inputs.config).framework == 'dynamo-trt')) }}
   EVAL_ONLY: ${{ inputs.eval-only }}
   EVAL_FRAMEWORK: ${{ inputs.eval-framework }}
   EVAL_SUITE: ${{ inputs.eval-suite }}
@@ -164,6 +164,8 @@ permissions:
 
 jobs:
   benchmark:
+    # BFCL is disabled for TRT; skip eval-only jobs before reserving a GPU runner.
+    if: ${{ !(inputs.eval-only && inputs.eval-framework == 'bfcl' && (fromJSON(inputs.config).framework == 'trt' || fromJSON(inputs.config).framework == 'dynamo-trt')) }}
     runs-on: >-
       ${{ fromJSON(
         vars.PRIORITY_SCHEDULER_ENABLED == 'true' &&

.github/workflows/e2e-tests.yml

@@ -57,7 +57,7 @@ on: # zizmor: ignore[concurrency-limits]
                 type: string
                 default: "auto"
             eval-suite:
-                description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
+                description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_responses_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
                 required: false
                 type: string
                 default: ""
@@ -159,7 +159,7 @@ on: # zizmor: ignore[concurrency-limits]
                 type: string
                 default: "auto"
             eval-suite:
-                description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
+                description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_responses_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval and swebench"
                 required: false
                 type: string
                 default: ""

inferencex-e2e/benchmarks/benchmark_lib.sh

@@ -363,21 +363,6 @@ if not speed_bench_ok or not sample_ok:
 PYEOF
 }
 
-disable_trtllm_detailed_perf_metrics() {
-    local trtllm_root
-    local py_executor
-    local detailed_metrics_gate="enabled=getattr(self.llm_args, 'return_perf_metrics', False))"
-
-    trtllm_root=$(python3 -c 'from importlib.util import find_spec; from pathlib import Path; print(Path(find_spec("tensorrt_llm").origin).parent)')
-    py_executor="$trtllm_root/_torch/pyexecutor/py_executor.py"
-    if ! grep -Fq "$detailed_metrics_gate" "$py_executor"; then
-        echo "Error: unsupported TensorRT-LLM detailed metrics implementation in $py_executor" >&2
-        return 1
-    fi
-    sed -i "s/enabled=getattr(self.llm_args, 'return_perf_metrics', False))/enabled=False)/" "$py_executor"
-    grep -Fq "self.perf_manager = PerfMetricsManager(" "$py_executor"
-    grep -Fq "enabled=False)" "$py_executor"
-}
 
 # Agentic replays must use the model's native context limit. Ignore inherited
 # workflow or shell overrides so neither the server nor AIPerf applies a cap.
@@ -1714,20 +1699,39 @@ _run_bfcl_smoke_eval() {
     _run_bfcl_suite_eval bfcl_smoke 4 900 false "$@"
 }
 
+_skip_bfcl_for_trt() {
+    case "${FRAMEWORK:-}" in
+        trt|dynamo-trt)
+            echo "SKIP: BFCL is disabled for ${FRAMEWORK}; no evaluation score was produced."
+            return 0
+            ;;
+        *) return 1 ;;
+    esac
+}
+
 run_bfcl_eval() {
+    if _skip_bfcl_for_trt; then
+        return 0
+    fi
     local eval_suite="${EVAL_SUITE:-bfcl_smoke}"
     export EVAL_SUITE="$eval_suite"
 
     case "$eval_suite" in
         bfcl_smoke)
             _run_bfcl_smoke_eval "$@"
             ;;
+        bfcl_responses_smoke)
+            _run_bfcl_suite_eval "$eval_suite" 4 900 false "$@"
+            ;;
         bfcl_vllm_minimax_m3)
             _run_bfcl_suite_eval "$eval_suite" 8 7200 true "$@"
             ;;
         bfcl_vllm_kimi)
             _run_bfcl_suite_eval "$eval_suite" 16 14400 true "$@"
             ;;
+        bfcl_kimi_diagnostic)
+            _run_bfcl_suite_eval "$eval_suite" 16 600 true "$@"
+            ;;
         *)
             echo "ERROR: unsupported BFCL suite '${eval_suite}'" >&2
             export EVAL_RESULT_DIR=""
@@ -2922,6 +2926,9 @@ run_eval() {
     fi
 
     local framework="${EVAL_FRAMEWORK:-${cli_framework:-$scenario_default}}"
+    if [ "$framework" = "bfcl" ] && _skip_bfcl_for_trt; then
+        return 0
+    fi
     case "$framework" in
         kimi-vendor)
             [ -n "${EVAL_SUITE:-}" ] || EVAL_SUITE="kimi_tool_call_schema"

inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/minimaxm3/trtllm/gb200-fp4/agentx/agg-variants.yaml

@@ -7,7 +7,7 @@ schema: 2
 base:
   model:
     path: minimax-m3-nvfp4
-    container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23.post1
+    container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc28.dev202609220000
     precision: fp4
   dynamo:
     install: true
@@ -72,11 +72,11 @@ base:
           implementation: msa
           indexer_kv_dtype: fp8
           sparse_disable_index_value: true
-          fuse_qkv_index_projection: true
         kv_cache_config:
           free_gpu_memory_fraction: 0.94
           enable_block_reuse: true
-          block_reuse_policy: per_conversation
+          block_reuse_config:
+            policy: per_conversation
           tokens_per_block: 128
           use_kv_cache_manager_v2: true
           dtype: fp8

inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml

@@ -1,15 +1,14 @@
 # Kimi-K3 MXFP4 AgentX on MI355X with vLLM DSpark
 # (https://recipes.vllm.ai/moonshotai/Kimi-K3). TP8 only: the 1.56 TB checkpoint
 # is ~195 GB per GPU. The KV cache is GPU-resident through c4 and backed by
-# vLLM's SimpleCPUOffloadConnector from c8. The DCP8 arm (c44-c70) ran without a
-# draft model on the legacy script and was removed with it (#3461). No master
-# config uses this recipe yet.
+# vLLM's SimpleCPUOffloadConnector from c8. The DCP8 arm (c44-c70) runs without
+# a draft model, preserving the configured MTP label.
 base:
   schema: 2
   name: kimik3-fp4-mi355x-vllm-agentic
   model:
     path: hf:moonshotai/Kimi-K3
-    container: vllm/vllm-openai-rocm:nightly-rocm100-af1c01499b289be555c475669ba50a88e96d846e
+    container: vllm/vllm-openai-rocm:nightly-rocm100-7f1a5398e9610d96c473931a26c0e12bbe0d0423
     precision: fp4
   resources:
     gpu_type: mi355x
@@ -184,3 +183,66 @@ override_tp8_c14_simple:
       CONC: '14'
       KV_OFFLOADING: 'dram'
       TOTAL_CPU_DRAM_GB: '1799'
+
+override_tp8_dcp8_c44_simple:
+  roles:
+    agg:
+      args:
+        max-num-seqs: 88
+        max-num-batched-tokens: 8192
+        compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":88,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88]}'
+        kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}'
+        decode-context-parallel-size: 8
+        dcp-comm-backend: a2a
+        attention-backend: ROCM_AITER_MLA
+      env:
+        VLLM_USE_BREAKABLE_CUDAGRAPH: '0'
+        PYTHONHASHSEED: '42'
+  benchmark:
+    env:
+      CONC: '44'
+      KV_OFFLOADING: dram
+      TOTAL_CPU_DRAM_GB: '1799'
+      SPEC_DECODING: mtp
+
+override_tp8_dcp8_c48_simple:
+  roles:
+    agg:
+      args:
+        max-num-seqs: 96
+        max-num-batched-tokens: 8192
+        compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":96,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96]}'
+        kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}'
+        decode-context-parallel-size: 8
+        dcp-comm-backend: a2a
+        attention-backend: ROCM_AITER_MLA
+      env:
+        VLLM_USE_BREAKABLE_CUDAGRAPH: '0'
+        PYTHONHASHSEED: '42'
+  benchmark:
+    env:
+      CONC: '48'
+      KV_OFFLOADING: dram
+      TOTAL_CPU_DRAM_GB: '1799'
+      SPEC_DECODING: mtp
+
+override_tp8_dcp8_c70_simple:
+  roles:
+    agg:
+      args:
+        max-num-seqs: 140
+        max-num-batched-tokens: 8192
+        compilation-config: '{"mode":3,"cudagraph_mode":"FULL","max_cudagraph_capture_size":140,"custom_ops":["+fused_rms_norm_gated"],"cudagraph_capture_sizes":[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140]}'
+        kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":224875000000,"lazy_offload":false}}'
+        decode-context-parallel-size: 8
+        dcp-comm-backend: a2a
+        attention-backend: ROCM_AITER_MLA
+      env:
+        VLLM_USE_BREAKABLE_CUDAGRAPH: '0'
+        PYTHONHASHSEED: '42'
+  benchmark:
+    env:
+      CONC: '70'
+      KV_OFFLOADING: dram
+      TOTAL_CPU_DRAM_GB: '1799'
+      SPEC_DECODING: mtp

inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b200-fp4-mtp/agentic.yaml

@@ -6,7 +6,7 @@ base:
   name: minimaxm3-fp4-b200-trtllm-agentic
   model:
     path: hf:nvidia/MiniMax-M3-NVFP4
-    container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1
+    container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc28.dev202609220000
     precision: fp4
   resources:
     gpu_type: b200
@@ -21,9 +21,6 @@ base:
   engine:
     type: trtllm
     served_model_name: nvidia/MiniMax-M3-NVFP4
-  # Keep Prometheus without rc23's per-step timing collector and accept BFCL's
-  # store=false field.
-  setup_script: minimaxm3-trtllm-rc23.sh
   # TP8 loads, autotunes and captures graphs past the 30-minute default.
   health_check:
     max_attempts: 1440
@@ -33,7 +30,7 @@ base:
       nodes: 1
       workers: 1
       # Server-layer flag; the local checkpoint mounts at /model.
-      extra_args: [--chat_template, /model/chat_template.jinja]
+      extra_args: [--chat_template, /model/chat_template.jinja, --tool_parser, minimax_m3]
       args:
         moe_expert_parallel_size: 1
         max_seq_len: 1048576
@@ -55,11 +52,11 @@ base:
           implementation: msa
           indexer_kv_dtype: fp8
           sparse_disable_index_value: true
-          fuse_qkv_index_projection: true
         kv_cache_config:
           free_gpu_memory_fraction: 0.94
           enable_block_reuse: true
-          block_reuse_policy: per_conversation
+          block_reuse_config:
+            policy: per_conversation
           tokens_per_block: 128
           use_kv_cache_manager_v2: true
           dtype: fp8

inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/trtllm/b300-fp4-mtp/agentic.yaml

@@ -5,7 +5,7 @@ base:
   name: minimaxm3-fp4-b300-trtllm-agentic
   model:
     path: hf:nvidia/MiniMax-M3-NVFP4
-    container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc23.post1
+    container: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc28.dev202609220000
     precision: fp4
   resources:
     gpu_type: b300
@@ -25,7 +25,7 @@ base:
     agg:
       nodes: 1
       workers: 1
-      extra_args: [--chat_template, /model/chat_template.jinja]
+      extra_args: [--chat_template, /model/chat_template.jinja, --tool_parser, minimax_m3]
       args:
         moe_expert_parallel_size: 1
         max_seq_len: 1048576
@@ -47,11 +47,11 @@ base:
           implementation: msa
           indexer_kv_dtype: fp8
           sparse_disable_index_value: true
-          fuse_qkv_index_projection: true
         kv_cache_config:
           free_gpu_memory_fraction: 0.94
           enable_block_reuse: true
-          block_reuse_policy: per_conversation
+          block_reuse_config:
+            policy: per_conversation
           tokens_per_block: 128
           use_kv_cache_manager_v2: true
           dtype: fp8