From f2af79667f9ebce5317cdc5e412ea122f104a435 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 10 Sep 2026 18:04:55 -0700 Subject: [PATCH 01/16] feat(config): add compact GLM-5.2 B300 AgentX recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 GLM-5.2 B300 AgentX 紧凑配方，并更新 B300 DSXE 启动路径。 --- .../b300-fp4/agentic/b300_ctx_hca_pin.sh | 63 +++++ ...agg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 248 +++++++++++++++++ ...gg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml | 250 +++++++++++++++++ ...gg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml | 248 +++++++++++++++++ ...gg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml | 250 +++++++++++++++++ ...-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml | 251 ++++++++++++++++++ ...-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 250 +++++++++++++++++ configs/nvidia-master.yaml | 105 ++++++++ perf-changelog.yaml | 7 + runners/launch_b300-dsxe.sh | 40 ++- 10 files changed, 1710 insertions(+), 2 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh new file mode 100644 index 0000000000..a99166c3f6 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh @@ -0,0 +1,63 @@ +# Worker preamble for GLM-5.2 dynamo-trt on B300 DSXE. Prefill ranks select active +# HCA pairs by physical GPU; decode remains unpinned. + +unset UCX_TLS # Preserve CUDA memory registration for NIXL transfers. + +_srt_live_devices() { + set -- /sys/class/infiniband/mlx5_* + [ -e "$1" ] || { printf '%s' "$_srt_in"; return 0; } # fail open + _srt_out=""; _srt_oIFS="$IFS"; IFS=, + for _srt_d in $_srt_in; do + _srt_n="${_srt_d%%:*}" + case "$(cat "/sys/class/infiniband/$_srt_n/ports/1/state" 2>/dev/null)" in + *ACTIVE*) _srt_out="${_srt_out:+$_srt_out,}$_srt_d" ;; + esac + done + IFS="$_srt_oIFS"; printf '%s' "$_srt_out" +} + +# `symmetric` shares four rails. `bia_faithful` pins each prefill rank to its +# physical GPU's rail pair. Other values leave ranks unchanged. +case "${BASH_EXECUTION_STRING:-}" in + *SRT_FABRIC_MODE=symmetric*) + export UCX_NET_DEVICES="mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1" + echo "CTX_HCA_PIN mode=symmetric localid=${SLURM_LOCALID:-0} UCX_NET_DEVICES=$UCX_NET_DEVICES" + return 0 2>/dev/null || true + ;; + *SRT_FABRIC_MODE=bia_faithful*) ;; + *) return 0 2>/dev/null || true ;; +esac + +case "${BASH_EXECUTION_STRING:-}" in + *trtllm_config_prefill*) ;; # context rank: pin below + *) return 0 2>/dev/null || true ;; # decode/frontend: unpinned +esac + +_srt_cvd=$(printf '%s' "${BASH_EXECUTION_STRING:-}" \ + | grep -oE 'CUDA_VISIBLE_DEVICES=[0-9,]+' | head -1 | cut -d= -f2) +[ -n "$_srt_cvd" ] || return 0 2>/dev/null || true + +IFS=, read -r -a _srt_g <<< "$_srt_cvd" +_srt_phys="${_srt_g[${SLURM_LOCALID:-0}]}" +case "$_srt_phys" in + 0) _srt_hca="mlx5_2:1,mlx5_3:1" ;; + 1) _srt_hca="mlx5_8:1,mlx5_9:1" ;; + 2) _srt_hca="mlx5_4:1,mlx5_5:1" ;; + 3) _srt_hca="mlx5_0:1,mlx5_1:1" ;; + 4) _srt_hca="mlx5_16:1,mlx5_17:1" ;; + 5) _srt_hca="mlx5_22:1,mlx5_23:1" ;; + 6) _srt_hca="mlx5_20:1,mlx5_21:1" ;; + 7) _srt_hca="mlx5_10:1,mlx5_11:1" ;; + *) echo "CTX_HCA_PIN: no mapping for physical GPU $_srt_phys" >&2; _srt_hca="" ;; +esac + +if [ -n "$_srt_hca" ]; then + _srt_in="$_srt_hca"; _srt_hca="$(_srt_live_devices)" +fi +if [ -n "$_srt_hca" ]; then + export UCX_NET_DEVICES="$_srt_hca" + echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=$UCX_NET_DEVICES" +else + echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=" +fi +return 0 2>/dev/null || true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml new file mode 100644 index 0000000000..f70390c11d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -0,0 +1,248 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + enable_padding: true + enable_attention_dp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 1 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 412316860416 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 5 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '1' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_1p1d-compact_c1 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + enable_multiple_frontends: false + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + orchestrator_placement: first_decode + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc26.dev202609040000 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 +name: dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: b300 + gpus_per_decode: 8 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 +sbatch_directives: + exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml new file mode 100644 index 0000000000..ee828239b1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml @@ -0,0 +1,250 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 5 + enable_padding: true + enable_attention_dp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 412316860416 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 5 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '20' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_1p1d-compact_c20 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + enable_multiple_frontends: false + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + orchestrator_placement: first_decode + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc26.dev202609040000 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 +name: dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: b300 + gpus_per_decode: 8 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 +sbatch_directives: + exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml new file mode 100644 index 0000000000..4393b38769 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml @@ -0,0 +1,248 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + enable_padding: true + enable_attention_dp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 2 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 274877906944 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 5 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '30' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_1p4d-compact_c30 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + orchestrator_placement: first_decode + enable_multiple_frontends: false + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc26.dev202609040000 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 +name: dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5 +resources: + decode_nodes: 2 + decode_workers: 4 + gpu_type: b300 + gpus_per_decode: 4 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 +sbatch_directives: + exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml new file mode 100644 index 0000000000..040cea5936 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml @@ -0,0 +1,250 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: symmetric + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: symmetric + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 5 + enable_padding: true + enable_attention_dp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 5 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 197568495616 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 5 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '60' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_3p4d-compact_c60 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + orchestrator_placement: first_decode + enable_multiple_frontends: false + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc26.dev202609040000 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 +name: dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5 +resources: + decode_nodes: 2 + decode_workers: 4 + gpu_type: b300 + gpus_per_decode: 4 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 2 + prefill_workers: 3 +sbatch_directives: + exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml new file mode 100644 index 0000000000..919c72efdb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml @@ -0,0 +1,251 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 197568495616 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + context_chunking_policy: EQUAL_PROGRESS + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '227' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_6p1d-compact_c227 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + enable_multiple_frontends: true + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc26.dev202609040000 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 +name: dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: b300 + gpus_per_decode: 8 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 3 + prefill_workers: 6 +sbatch_directives: + exclude: b300-001,b300-005,b300-006,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml new file mode 100644 index 0000000000..774777b5ae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -0,0 +1,250 @@ +backend: + decode_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + prefill_environment: + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_PUBLISH_KV_EVENTS: '0' + DYN_TOKENIZER: fastokens + DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + HF_HUB_DISABLE_PROGRESS_BARS: '1' + HF_HUB_OFFLINE: '1' + HOME: /trtllm-jit-cache + MIMALLOC_PURGE_DELAY: '0' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + SRT_FABRIC_MODE: bia_faithful + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' + TLLM_EXECUTOR_USE_FILE_SOCKET: '1' + TLLM_LOG_LEVEL: INFO + TLLM_PREFIX_TOKEN_CACHE: '1' + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' + TQDM_DISABLE: '1' + TRANSFORMERS_OFFLINE: '1' + TRTLLM_CTX_LOCAL_HCA_PIN: '1' + TRTLLM_DSA_INDEXER_BF16: '1' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_SERVE_ENABLE_MSGSPEC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_LOG_LEVEL: info + UCX_MAX_HCA_PER_GPU: inf + UCX_MAX_RNDV_RAILS: '2' + UCX_RNDV_SCHEME: put_zcopy + publish_events_and_metrics: false + trtllm_config: + decode: + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.9 + host_cache_size: 0 + tokens_per_block: 64 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + speculative_config: + decoding_type: MTP + max_draft_len: 3 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + backend: pytorch + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: '5120' + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 1048576 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + gpus_per_node: 8 + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 197568495616 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + perf_metrics_max_requests: 100000 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + num_nextn_predict_layers: 3 + tensor_parallel_size: 4 + trust_remote_code: true + type: trtllm +benchmark: + type: custom + client_placement: first_decode + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '3600' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT: '1800' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + CONC: '233' + DURATION: '3600' + FRAMEWORK: dynamo-trt + HF_HUB_CACHE: /hf_hub_cache + HF_HUB_DISABLE_PROGRESS_BARS: '1' + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + IS_MULTINODE: 'true' + KV_OFFLOADING: none + MAX_MODEL_LEN: '1048576' + MODEL: nvidia/GLM-5.2-NVFP4 + MODEL_PREFIX: glm5.2 + OPENAI_API_KEY: EMPTY + PORT: '8000' + PRECISION: fp4 + RESULT_DIR: /logs/agentic + RESULT_FILENAME: glm52_fp4_dynamo-trt_8p2d-compact_c233 + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + TQDM_DISABLE: '1' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 +dynamo: + remap_root: false # Preserve the invoking UID for PMIx authentication. + install: true + request_plane: tcp + version: 1.4.0 +frontend: + args: + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + no-kv-events: true + router-mode: kv + enable_multiple_frontends: true + env: + DYN_LOG: warn + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_TOKENIZER: fastokens + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + ETCD_LEASE_TTL: '120' + type: dynamo +health_check: + interval_seconds: 10 + max_attempts: 540 +identity: + model: + repo: nvidia/GLM-5.2-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + frameworks: + dynamo: 1.4.0 + tensorrt_llm: 1.3.0rc26.dev202609040000 +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + precision: fp4 +name: dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3 +resources: + decode_nodes: 2 + decode_workers: 2 + gpu_type: b300 + gpus_per_decode: 8 + gpus_per_node: 8 + gpus_per_prefill: 4 + prefill_nodes: 4 + prefill_workers: 8 +sbatch_directives: + exclude: b300-001,b300-002,b300-004,b300-005,b300-009,b300-010,b300-012,b300-013,b300-014,b300-015,b300-018 +srun_options: + cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 526fbec480..5667f4c8eb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9076,6 +9076,111 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: ep: 1 dp-attn: true +glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 + model: nvidia/GLM-5.2-NVFP4 + model-prefix: glm5.2 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "1.4.0.dev20260807" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [1] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [20] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [30] + kv-offloading: none + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [60] + kv-offloading: none + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [227] + kv-offloading: none + prefill: + num-worker: 6 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: mtp + conc-list: [233] + kv-offloading: none + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml" + decode: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + # GLM-5.2 B300 NVFP4 AgentX with EAGLE/MTP speculative decoding, following the # AgentX speculative-decoding policy in MODELS.md. SGLang EAGLE runs off # GLM-5.2's built-in nextn head (num-steps 3, diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b0dc6a14d4..f017d0353b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7145,3 +7145,10 @@ - "Update SGLang image from lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 (2026-09-07 cu13 dev nightly, build commit sgl-project/sglang@30705c00) to the v0.5.19 release image lmsysorg/sglang:v0.5.19-cu130 (digest sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9, build commit sgl-project/sglang@0bcd822377da7b5718e674eaf9c870d349424dd1, Docker Hub last pushed 2026-09-04T22:50:19Z)." - "The release image ships the same CUDA 13.0.3, FlashInfer 0.6.18 and sgl-kernel 0.4.6.post1 as the nightly. benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh is unchanged: SGLANG_ENABLE_SPEC_V2 EAGLE MTP at 3 steps, golden acceptance length 3.39, flashinfer attention with allreduce fusion, fp8 quantization and fp8_e4m3 KV, HiCache kernel IO / page_first layout. TP8/EP1 DRAM HiCache concurrency 2 through 24 unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2966 + +- config-keys: + - glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact + description: + - "Add six compact GLM-5.2 NVFP4 AgentX recipes on B300 with Dynamo and TensorRT-LLM disaggregated serving." + - "Use TensorRT-LLM 1.3.0rc26, enable the DSA metadata and indexer settings, and provide a persistent JIT cache for the recipe workers." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 70254deb65..0beac7cd19 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -145,8 +145,9 @@ fi # Default is the newest tag. Add a branch here to pin a ref per model / precision / # framework when a recipe needs one, so results stay reproducible. select_srt_slurm_version() { - if false; then - : + if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-trt" && "$MODEL_PREFIX" == "glm5.2" ]]; then + SRT_SLURM_REPO="https://github.com/Thunderbeee/srt-slurm.git" + SRT_SLURM_REF="824c15e8eccd447bdf79c39d264ebafd10b1dba3" else SRT_SLURM_REPO="https://github.com/NVIDIA/srt-slurm.git" SRT_SLURM_REF="v1.0.87" @@ -221,6 +222,38 @@ export ISL="$ISL" export OSL="$OSL" export EVAL_ONLY="${EVAL_ONLY:-false}" +SRT_EXTRA_CLUSTER_CONFIG="" +if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-trt" && "$MODEL_PREFIX" == "glm5.2" ]]; then + CTX_HCA_PIN_HOST_DIR="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic" + CTX_HCA_PIN_SCRIPT="$CTX_HCA_PIN_HOST_DIR/b300_ctx_hca_pin.sh" + if [[ ! -f "$CTX_HCA_PIN_SCRIPT" ]]; then + echo "Error: context HCA pinning preamble not found at $CTX_HCA_PIN_SCRIPT" >&2 + exit 1 + fi + + AIPERF_MMAP_CACHE_HOST_PATH="/data/home/sa-gha-runner/aiperf-cache" + HF_HUB_CACHE_HOST_PATH="/data/home/sa-gha-runner/hf-hub-cache" + TRTLLM_JIT_CACHE_HOST_PATH="/data/home/sa-gha-runner/trtllm-jit-cache" + mkdir -p \ + "$AIPERF_MMAP_CACHE_HOST_PATH" \ + "$HF_HUB_CACHE_HOST_PATH" \ + "$TRTLLM_JIT_CACHE_HOST_PATH" + chmod 0777 \ + "$AIPERF_MMAP_CACHE_HOST_PATH" \ + "$HF_HUB_CACHE_HOST_PATH" \ + "$TRTLLM_JIT_CACHE_HOST_PATH" 2>/dev/null || true + + SRT_EXTRA_CLUSTER_CONFIG=$(cat < srtslurm.yaml echo "Generated srtslurm.yaml:" From 427c61d588d3ccaf78780f08ec962d3006839918 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 10 Sep 2026 18:05:44 -0700 Subject: [PATCH 02/16] chore(changelog): link PR #2993 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将性能变更日志条目链接到 PR #2993。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f017d0353b..b1ce68d288 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp From 5f2b9e341d6a405f1ff96bd60503ea0acb6db7ee Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 10 Sep 2026 18:06:21 -0700 Subject: [PATCH 03/16] fix(changelog): correct PR #2993 link placement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 恢复历史条目的占位链接，并仅在新条目中填写 PR #2993。 --- perf-changelog.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b1ce68d288..2340f44bdd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5595,7 +5595,7 @@ - "Image ghcr.io/tile-ai/tilert:0.1.5 (tilert 0.1.5.post2 installed at container start); commands aligned to TileRT README Topology A -- NIXL KV transfer, --kv-cache-dtype fp8_ds_mla (prefill) <-> fp8 (decode), max-seq-len 202752; MTP speculative-config wired via spec-decoding=mtp" - "Topology: 1 prefill node (TP8) + 1 decode node (TP8), each 8xB200 exclusive; TileRT decode is bs=1 only so conc-list is a single point [1], ISL 1k/8k OSL 1k" - "Runner: launch_b200-dgxc.sh tilert early-return branch (zero impact on the dynamo path); tilert_utils/submit.sh issues two srun --ntasks=1, one per role, because prefill and decode need different container images; roles are dispatched by the TILERT_ROLE it exports, and torn down across nodes via a sentinel file on the shared /workspace" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX - config-keys: - qwen3.5-fp8-b200-sglang-agentic-mtp @@ -7151,4 +7151,4 @@ description: - "Add six compact GLM-5.2 NVFP4 AgentX recipes on B300 with Dynamo and TensorRT-LLM disaggregated serving." - "Use TensorRT-LLM 1.3.0rc26, enable the DSA metadata and indexer settings, and provide a persistent JIT cache for the recipe workers." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 From fe9a40ec92297884a0e60f74ac04a9bac1ad2566 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 14 Sep 2026 12:57:43 -0700 Subject: [PATCH 04/16] fix(recipes): remove stale B300 node exclusions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移除六个 GLM-5.2 B300 配方中过期的节点排除列表，让 DSXE 调度器选择有效节点。 --- .../dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 2 -- .../dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml | 2 -- .../dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml | 2 -- .../dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml | 2 -- .../dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml | 2 -- .../dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 2 -- 6 files changed, 12 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml index f70390c11d..873e140065 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -242,7 +242,5 @@ resources: gpus_per_prefill: 4 prefill_nodes: 1 prefill_workers: 1 -sbatch_directives: - exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 srun_options: cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml index ee828239b1..f790980b3a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml @@ -244,7 +244,5 @@ resources: gpus_per_prefill: 4 prefill_nodes: 1 prefill_workers: 1 -sbatch_directives: - exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 srun_options: cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml index 4393b38769..ef43307d6d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml @@ -242,7 +242,5 @@ resources: gpus_per_prefill: 4 prefill_nodes: 1 prefill_workers: 1 -sbatch_directives: - exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 srun_options: cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml index 040cea5936..7d4e0ec933 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml @@ -244,7 +244,5 @@ resources: gpus_per_prefill: 4 prefill_nodes: 2 prefill_workers: 3 -sbatch_directives: - exclude: b300-001,b300-005,b300-006,b300-007,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 srun_options: cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml index 919c72efdb..57b9ea3e14 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml @@ -245,7 +245,5 @@ resources: gpus_per_prefill: 4 prefill_nodes: 3 prefill_workers: 6 -sbatch_directives: - exclude: b300-001,b300-005,b300-006,b300-009,b300-010,b300-014,b300-015,b300-016,b300-017 srun_options: cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml index 774777b5ae..d037b7cd5b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -244,7 +244,5 @@ resources: gpus_per_prefill: 4 prefill_nodes: 4 prefill_workers: 8 -sbatch_directives: - exclude: b300-001,b300-002,b300-004,b300-005,b300-009,b300-010,b300-012,b300-013,b300-014,b300-015,b300-018 srun_options: cpu-bind: verbose,mask_cpu:0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0x0000000000000000ffffffffffffffff0000000000000000ffffffffffffffff,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000,0xffffffffffffffff0000000000000000ffffffffffffffff0000000000000000 From 7e5ed3181e10a1ee6d994151f9e61cdac87d309b Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 14 Sep 2026 18:07:24 -0700 Subject: [PATCH 05/16] fix(recipes): fall back to available B300 fabric devices MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validate requested UCX devices before exporting them and use automatic fabric discovery when the requested names are unavailable. Increase pip download retry limits for transient runtime installation interruptions. 中文：导出 UCX 设备前先验证其可用性；当指定设备名称不存在时，改用自动网络设备发现。同时提高运行时安装过程中临时下载中断的重试上限。 --- .../b300-fp4/agentic/b300_ctx_hca_pin.sh | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh index a99166c3f6..48e40343d8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh @@ -1,11 +1,17 @@ -# Worker preamble for GLM-5.2 dynamo-trt on B300 DSXE. Prefill ranks select active -# HCA pairs by physical GPU; decode remains unpinned. +# Worker preamble for GLM-5.2 dynamo-trt on B300 DSXE. Use the requested HCA +# layout only when those devices are active; otherwise let UCX discover the +# available fabric devices. unset UCX_TLS # Preserve CUDA memory registration for NIXL transfers. +unset UCX_NET_DEVICES + +# Dynamo is installed at worker startup. Allow pip to resume transiently truncated +# package downloads instead of failing the whole multi-node allocation. +export PIP_DEFAULT_TIMEOUT="${PIP_DEFAULT_TIMEOUT:-120}" +export PIP_RETRIES="${PIP_RETRIES:-20}" +export PIP_RESUME_RETRIES="${PIP_RESUME_RETRIES:-20}" _srt_live_devices() { - set -- /sys/class/infiniband/mlx5_* - [ -e "$1" ] || { printf '%s' "$_srt_in"; return 0; } # fail open _srt_out=""; _srt_oIFS="$IFS"; IFS=, for _srt_d in $_srt_in; do _srt_n="${_srt_d%%:*}" @@ -20,8 +26,14 @@ _srt_live_devices() { # physical GPU's rail pair. Other values leave ranks unchanged. case "${BASH_EXECUTION_STRING:-}" in *SRT_FABRIC_MODE=symmetric*) - export UCX_NET_DEVICES="mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1" - echo "CTX_HCA_PIN mode=symmetric localid=${SLURM_LOCALID:-0} UCX_NET_DEVICES=$UCX_NET_DEVICES" + _srt_in="mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1" + _srt_hca="$(_srt_live_devices)" + if [ -n "$_srt_hca" ]; then + export UCX_NET_DEVICES="$_srt_hca" + echo "CTX_HCA_PIN mode=symmetric localid=${SLURM_LOCALID:-0} UCX_NET_DEVICES=$UCX_NET_DEVICES" + else + echo "CTX_HCA_PIN mode=symmetric localid=${SLURM_LOCALID:-0} UCX_NET_DEVICES=" + fi return 0 2>/dev/null || true ;; *SRT_FABRIC_MODE=bia_faithful*) ;; @@ -58,6 +70,6 @@ if [ -n "$_srt_hca" ]; then export UCX_NET_DEVICES="$_srt_hca" echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=$UCX_NET_DEVICES" else - echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=" + echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=" fi return 0 2>/dev/null || true From 6bb87443823a539e8984d324ff10bd341247ef6f Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 14 Sep 2026 20:58:49 -0700 Subject: [PATCH 06/16] fix(recipes): extend GLM-5.2 B300 Slurm limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为六个 GLM-5.2 B300 AgentX 配方显式设置 12 小时 Slurm 时限，避免作业使用一小时默认值。 --- .../dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 3 +++ .../dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml | 3 +++ .../dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml | 3 +++ .../dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml | 3 +++ .../dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml | 3 +++ .../dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 3 +++ 6 files changed, 18 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml index d9f2d199b1..96d3c8adf5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -1,3 +1,6 @@ +slurm: + time_limit: '12:00:00' + backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml index b4e4622870..375b4fc655 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml @@ -1,3 +1,6 @@ +slurm: + time_limit: '12:00:00' + backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml index b208517638..188bd988a8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml @@ -1,3 +1,6 @@ +slurm: + time_limit: '12:00:00' + backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml index 9393aa3224..958a271750 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml @@ -1,3 +1,6 @@ +slurm: + time_limit: '12:00:00' + backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml index 2280a216d6..706bf25bff 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml @@ -1,3 +1,6 @@ +slurm: + time_limit: '12:00:00' + backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml index 18898fa0b3..18cbf0faa0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -1,3 +1,6 @@ +slurm: + time_limit: '12:00:00' + backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' From 0f4a194638be1d2aa38adc98bd9a0dadc9a42bcf Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 14 Sep 2026 23:32:38 -0700 Subject: [PATCH 07/16] fix(eval): co-locate GLM-5.2 lm-eval frontend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在仅评估运行中，将 GLM-5.2 前端与 lm-eval 放置在同一节点。 --- runners/launch_b300-dsxe.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 6f1b0ff5b1..792084fe04 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -318,6 +318,17 @@ fi # Override the job name in the recipe with the runner name. sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" if [[ "${EVAL_ONLY:-false}" == "true" ]]; then + if [[ "$IS_AGENTIC" == "1" && + "$FRAMEWORK" == "dynamo-trt" && + "$MODEL_PREFIX" == "glm5.2" && + "${EVAL_FRAMEWORK:-lm-eval}" == "lm-eval" ]]; then + # This pinned srt-slurm revision launches lm-eval on the allocation head + # and targets localhost. Keep throughput frontends on first_decode, but + # co-locate the eval-only frontend so the loopback endpoint is reachable. + sed -i \ + 's/^ orchestrator_placement: first_decode$/ orchestrator_placement: head/' \ + "$CONFIG_PATH" + fi python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ "$CONFIG_PATH" "$FRAMEWORK" || exit 1 fi From a9d7b2ab6a7afda15014debb335cf85b71985ede Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 15 Sep 2026 03:56:25 -0700 Subject: [PATCH 08/16] fix(recipes): retry truncated Dynamo installs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 重试被截断的 Dynamo 安装，避免多进程启动缺少工作进程。 --- .../glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh index 48e40343d8..87877209ff 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh @@ -11,6 +11,22 @@ export PIP_DEFAULT_TIMEOUT="${PIP_DEFAULT_TIMEOUT:-120}" export PIP_RETRIES="${PIP_RETRIES:-20}" export PIP_RESUME_RETRIES="${PIP_RESUME_RETRIES:-20}" +# pip does not retry every truncated response even with its network retry +# settings. Retry the complete command so a single worker does not leave an +# otherwise healthy multi-rank launch permanently short of one rank. +pip() { + local _srt_attempt=1 _srt_max_attempts=5 + while ! command pip "$@"; do + if [ "$_srt_attempt" -ge "$_srt_max_attempts" ]; then + echo "pip failed after $_srt_attempt attempts" >&2 + return 1 + fi + echo "pip failed; retrying complete command (attempt $((_srt_attempt + 1))/$_srt_max_attempts)" >&2 + sleep $((_srt_attempt * 5)) + _srt_attempt=$((_srt_attempt + 1)) + done +} + _srt_live_devices() { _srt_out=""; _srt_oIFS="$IFS"; IFS=, for _srt_d in $_srt_in; do From 813d5abf1cb62601d8c9f919078f90be85b02f2e Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 15 Sep 2026 08:04:03 -0700 Subject: [PATCH 09/16] fix(recipes): allow c1 profiling to drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为 c1 分析阶段增加可配置的请求排空宽限期。 --- benchmarks/benchmark_lib.sh | 5 +++++ .../dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 1 + 2 files changed, 6 insertions(+) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 305a118ef3..c514d4816b 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -3088,6 +3088,11 @@ build_replay_cmd() { REPLAY_CMD+=" --tokenizer $MODEL" REPLAY_CMD+=" --concurrency $CONC" REPLAY_CMD+=" --benchmark-duration $duration" + # Let recipes with long low-concurrency requests drain work admitted before + # the profiling window closes. Omitted recipes retain AIPerf's default. + if [ -n "${AGENTIC_BENCHMARK_GRACE_PERIOD:-}" ]; then + REPLAY_CMD+=" --benchmark-grace-period $AGENTIC_BENCHMARK_GRACE_PERIOD" + fi REPLAY_CMD+=" --stats-interval 30" REPLAY_CMD+=" --random-seed 42" # Fail runs early once the live error ratio crosses the configured limit. diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml index 96d3c8adf5..148ac34759 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -168,6 +168,7 @@ benchmark: client_placement: first_decode command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + AGENTIC_BENCHMARK_GRACE_PERIOD: '1800' AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' From 9ead89c267d1e77bc12b02939eb7ab4fc0356427 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 15 Sep 2026 11:58:22 -0700 Subject: [PATCH 10/16] fix(recipes): use LIBFABRIC for GLM-5.2 KV transfer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为 GLM-5.2 KV 传输启用 LIBFABRIC，并移除临时的超时、重试和排空缓解配置。 --- benchmarks/benchmark_lib.sh | 5 --- .../b300-fp4/agentic/b300_ctx_hca_pin.sh | 42 ++++--------------- ...agg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 10 ++--- ...gg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml | 9 ++-- ...gg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml | 9 ++-- ...gg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml | 9 ++-- ...-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml | 9 ++-- ...-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 9 ++-- 8 files changed, 31 insertions(+), 71 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index c514d4816b..305a118ef3 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -3088,11 +3088,6 @@ build_replay_cmd() { REPLAY_CMD+=" --tokenizer $MODEL" REPLAY_CMD+=" --concurrency $CONC" REPLAY_CMD+=" --benchmark-duration $duration" - # Let recipes with long low-concurrency requests drain work admitted before - # the profiling window closes. Omitted recipes retain AIPerf's default. - if [ -n "${AGENTIC_BENCHMARK_GRACE_PERIOD:-}" ]; then - REPLAY_CMD+=" --benchmark-grace-period $AGENTIC_BENCHMARK_GRACE_PERIOD" - fi REPLAY_CMD+=" --stats-interval 30" REPLAY_CMD+=" --random-seed 42" # Fail runs early once the live error ratio crosses the configured limit. diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh index 87877209ff..a99166c3f6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh @@ -1,33 +1,11 @@ -# Worker preamble for GLM-5.2 dynamo-trt on B300 DSXE. Use the requested HCA -# layout only when those devices are active; otherwise let UCX discover the -# available fabric devices. +# Worker preamble for GLM-5.2 dynamo-trt on B300 DSXE. Prefill ranks select active +# HCA pairs by physical GPU; decode remains unpinned. unset UCX_TLS # Preserve CUDA memory registration for NIXL transfers. -unset UCX_NET_DEVICES - -# Dynamo is installed at worker startup. Allow pip to resume transiently truncated -# package downloads instead of failing the whole multi-node allocation. -export PIP_DEFAULT_TIMEOUT="${PIP_DEFAULT_TIMEOUT:-120}" -export PIP_RETRIES="${PIP_RETRIES:-20}" -export PIP_RESUME_RETRIES="${PIP_RESUME_RETRIES:-20}" - -# pip does not retry every truncated response even with its network retry -# settings. Retry the complete command so a single worker does not leave an -# otherwise healthy multi-rank launch permanently short of one rank. -pip() { - local _srt_attempt=1 _srt_max_attempts=5 - while ! command pip "$@"; do - if [ "$_srt_attempt" -ge "$_srt_max_attempts" ]; then - echo "pip failed after $_srt_attempt attempts" >&2 - return 1 - fi - echo "pip failed; retrying complete command (attempt $((_srt_attempt + 1))/$_srt_max_attempts)" >&2 - sleep $((_srt_attempt * 5)) - _srt_attempt=$((_srt_attempt + 1)) - done -} _srt_live_devices() { + set -- /sys/class/infiniband/mlx5_* + [ -e "$1" ] || { printf '%s' "$_srt_in"; return 0; } # fail open _srt_out=""; _srt_oIFS="$IFS"; IFS=, for _srt_d in $_srt_in; do _srt_n="${_srt_d%%:*}" @@ -42,14 +20,8 @@ _srt_live_devices() { # physical GPU's rail pair. Other values leave ranks unchanged. case "${BASH_EXECUTION_STRING:-}" in *SRT_FABRIC_MODE=symmetric*) - _srt_in="mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1" - _srt_hca="$(_srt_live_devices)" - if [ -n "$_srt_hca" ]; then - export UCX_NET_DEVICES="$_srt_hca" - echo "CTX_HCA_PIN mode=symmetric localid=${SLURM_LOCALID:-0} UCX_NET_DEVICES=$UCX_NET_DEVICES" - else - echo "CTX_HCA_PIN mode=symmetric localid=${SLURM_LOCALID:-0} UCX_NET_DEVICES=" - fi + export UCX_NET_DEVICES="mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1" + echo "CTX_HCA_PIN mode=symmetric localid=${SLURM_LOCALID:-0} UCX_NET_DEVICES=$UCX_NET_DEVICES" return 0 2>/dev/null || true ;; *SRT_FABRIC_MODE=bia_faithful*) ;; @@ -86,6 +58,6 @@ if [ -n "$_srt_hca" ]; then export UCX_NET_DEVICES="$_srt_hca" echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=$UCX_NET_DEVICES" else - echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=" + echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=" fi return 0 2>/dev/null || true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml index 148ac34759..de7249919d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -1,6 +1,3 @@ -slurm: - time_limit: '12:00:00' - backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' @@ -29,6 +26,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -63,6 +61,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -76,7 +75,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON @@ -124,7 +123,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON @@ -168,7 +167,6 @@ benchmark: client_placement: first_decode command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: - AGENTIC_BENCHMARK_GRACE_PERIOD: '1800' AIPERF_DATASET_CONFIGURATION_TIMEOUT: '1800' AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml index 375b4fc655..0f4c2ee14e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml @@ -1,6 +1,3 @@ -slurm: - time_limit: '12:00:00' - backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' @@ -29,6 +26,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -63,6 +61,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -76,7 +75,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON @@ -126,7 +125,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml index 188bd988a8..933c65d2e0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml @@ -1,6 +1,3 @@ -slurm: - time_limit: '12:00:00' - backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' @@ -29,6 +26,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -63,6 +61,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -76,7 +75,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON @@ -124,7 +123,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml index 958a271750..d9b89ed5db 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml @@ -1,6 +1,3 @@ -slurm: - time_limit: '12:00:00' - backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' @@ -29,6 +26,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -63,6 +61,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -76,7 +75,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON @@ -126,7 +125,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml index 706bf25bff..389556aa09 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml @@ -1,6 +1,3 @@ -slurm: - time_limit: '12:00:00' - backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' @@ -29,6 +26,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -63,6 +61,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -76,7 +75,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON @@ -128,7 +127,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml index 18cbf0faa0..ad0e954077 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -1,6 +1,3 @@ -slurm: - time_limit: '12:00:00' - backend: decode_environment: DYN_ENGINE_CONV_AFFINITY: '1' @@ -29,6 +26,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -63,6 +61,7 @@ backend: TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' + TRTLLM_NIXL_KVCACHE_BACKEND: LIBFABRIC TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -76,7 +75,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON @@ -128,7 +127,7 @@ backend: backend: pytorch cache_transceiver_config: backend: NIXL - kv_cache_bounce_size_mb: '5120' + kv_cache_bounce_size_mb: '0' kv_transfer_timeout_ms: 600000 max_tokens_in_buffer: 1048576 transceiver_runtime: PYTHON From d714eb65873dffe1923f389a7582b1a8f6496f0e Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 15 Sep 2026 14:28:07 -0700 Subject: [PATCH 11/16] fix: stage EFA runtime for GLM-5.2 LIBFABRIC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为 GLM-5.2 LIBFABRIC 暂存匹配的 EFA 运行时，并移除临时 UCX HCA 固定配置。 --- .../b300-fp4/agentic/b300_ctx_hca_pin.sh | 63 ------------------- ...agg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 12 ---- ...gg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml | 12 ---- ...gg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml | 12 ---- ...gg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml | 12 ---- ...-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml | 12 ---- ...-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 12 ---- runners/launch_b300-dsxe.sh | 61 +++++++++++++++--- runners/test_slurm_utils.py | 16 +++++ 9 files changed, 69 insertions(+), 143 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh deleted file mode 100644 index a99166c3f6..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/b300_ctx_hca_pin.sh +++ /dev/null @@ -1,63 +0,0 @@ -# Worker preamble for GLM-5.2 dynamo-trt on B300 DSXE. Prefill ranks select active -# HCA pairs by physical GPU; decode remains unpinned. - -unset UCX_TLS # Preserve CUDA memory registration for NIXL transfers. - -_srt_live_devices() { - set -- /sys/class/infiniband/mlx5_* - [ -e "$1" ] || { printf '%s' "$_srt_in"; return 0; } # fail open - _srt_out=""; _srt_oIFS="$IFS"; IFS=, - for _srt_d in $_srt_in; do - _srt_n="${_srt_d%%:*}" - case "$(cat "/sys/class/infiniband/$_srt_n/ports/1/state" 2>/dev/null)" in - *ACTIVE*) _srt_out="${_srt_out:+$_srt_out,}$_srt_d" ;; - esac - done - IFS="$_srt_oIFS"; printf '%s' "$_srt_out" -} - -# `symmetric` shares four rails. `bia_faithful` pins each prefill rank to its -# physical GPU's rail pair. Other values leave ranks unchanged. -case "${BASH_EXECUTION_STRING:-}" in - *SRT_FABRIC_MODE=symmetric*) - export UCX_NET_DEVICES="mlx5_0:1,mlx5_1:1,mlx5_10:1,mlx5_11:1" - echo "CTX_HCA_PIN mode=symmetric localid=${SLURM_LOCALID:-0} UCX_NET_DEVICES=$UCX_NET_DEVICES" - return 0 2>/dev/null || true - ;; - *SRT_FABRIC_MODE=bia_faithful*) ;; - *) return 0 2>/dev/null || true ;; -esac - -case "${BASH_EXECUTION_STRING:-}" in - *trtllm_config_prefill*) ;; # context rank: pin below - *) return 0 2>/dev/null || true ;; # decode/frontend: unpinned -esac - -_srt_cvd=$(printf '%s' "${BASH_EXECUTION_STRING:-}" \ - | grep -oE 'CUDA_VISIBLE_DEVICES=[0-9,]+' | head -1 | cut -d= -f2) -[ -n "$_srt_cvd" ] || return 0 2>/dev/null || true - -IFS=, read -r -a _srt_g <<< "$_srt_cvd" -_srt_phys="${_srt_g[${SLURM_LOCALID:-0}]}" -case "$_srt_phys" in - 0) _srt_hca="mlx5_2:1,mlx5_3:1" ;; - 1) _srt_hca="mlx5_8:1,mlx5_9:1" ;; - 2) _srt_hca="mlx5_4:1,mlx5_5:1" ;; - 3) _srt_hca="mlx5_0:1,mlx5_1:1" ;; - 4) _srt_hca="mlx5_16:1,mlx5_17:1" ;; - 5) _srt_hca="mlx5_22:1,mlx5_23:1" ;; - 6) _srt_hca="mlx5_20:1,mlx5_21:1" ;; - 7) _srt_hca="mlx5_10:1,mlx5_11:1" ;; - *) echo "CTX_HCA_PIN: no mapping for physical GPU $_srt_phys" >&2; _srt_hca="" ;; -esac - -if [ -n "$_srt_hca" ]; then - _srt_in="$_srt_hca"; _srt_hca="$(_srt_live_devices)" -fi -if [ -n "$_srt_hca" ]; then - export UCX_NET_DEVICES="$_srt_hca" - echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=$UCX_NET_DEVICES" -else - echo "CTX_HCA_PIN localid=${SLURM_LOCALID:-0} phys_gpu=$_srt_phys UCX_NET_DEVICES=" -fi -return 0 2>/dev/null || true diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml index de7249919d..1417571082 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -10,7 +10,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: bia_faithful TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -19,7 +18,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -30,10 +28,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy prefill_environment: DYN_ENGINE_CONV_AFFINITY: '1' DYN_PUBLISH_KV_EVENTS: '0' @@ -45,7 +39,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: bia_faithful TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -54,7 +47,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -65,10 +57,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy publish_events_and_metrics: false trtllm_config: decode: diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml index 0f4c2ee14e..77dd505328 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml @@ -10,7 +10,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: bia_faithful TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -19,7 +18,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -30,10 +28,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy prefill_environment: DYN_ENGINE_CONV_AFFINITY: '1' DYN_PUBLISH_KV_EVENTS: '0' @@ -45,7 +39,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: bia_faithful TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -54,7 +47,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -65,10 +57,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy publish_events_and_metrics: false trtllm_config: decode: diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml index 933c65d2e0..f7bf40b4e9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml @@ -10,7 +10,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: bia_faithful TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -19,7 +18,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -30,10 +28,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy prefill_environment: DYN_ENGINE_CONV_AFFINITY: '1' DYN_PUBLISH_KV_EVENTS: '0' @@ -45,7 +39,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: bia_faithful TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -54,7 +47,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -65,10 +57,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy publish_events_and_metrics: false trtllm_config: decode: diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml index d9b89ed5db..6c9163f53e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml @@ -10,7 +10,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: symmetric TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -19,7 +18,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -30,10 +28,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy prefill_environment: DYN_ENGINE_CONV_AFFINITY: '1' DYN_PUBLISH_KV_EVENTS: '0' @@ -45,7 +39,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: symmetric TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -54,7 +47,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '2.61' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -65,10 +57,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy publish_events_and_metrics: false trtllm_config: decode: diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml index 389556aa09..1841d0ba72 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml @@ -10,7 +10,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: bia_faithful TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -19,7 +18,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -30,10 +28,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy prefill_environment: DYN_ENGINE_CONV_AFFINITY: '1' DYN_PUBLISH_KV_EVENTS: '0' @@ -45,7 +39,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: bia_faithful TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -54,7 +47,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -65,10 +57,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy publish_events_and_metrics: false trtllm_config: decode: diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml index ad0e954077..ac495f3acd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -10,7 +10,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: bia_faithful TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -19,7 +18,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -30,10 +28,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy prefill_environment: DYN_ENGINE_CONV_AFFINITY: '1' DYN_PUBLISH_KV_EVENTS: '0' @@ -45,7 +39,6 @@ backend: MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - SRT_FABRIC_MODE: bia_faithful TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' TLLM_EXECUTOR_USE_FILE_SOCKET: '1' @@ -54,7 +47,6 @@ backend: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '1.99' TQDM_DISABLE: '1' TRANSFORMERS_OFFLINE: '1' - TRTLLM_CTX_LOCAL_HCA_PIN: '1' TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' @@ -65,10 +57,6 @@ backend: TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_SERVE_ENABLE_MSGSPEC: '1' TRTLLM_WORKER_DISABLE_GC: '1' - UCX_LOG_LEVEL: info - UCX_MAX_HCA_PER_GPU: inf - UCX_MAX_RNDV_RAILS: '2' - UCX_RNDV_SCHEME: put_zcopy publish_events_and_metrics: false trtllm_config: decode: diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 03074af838..d92cdf5436 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -225,12 +225,57 @@ export EVAL_ONLY="${EVAL_ONLY:-false}" SRT_EXTRA_CLUSTER_CONFIG="" if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-trt" && "$MODEL_PREFIX" == "glm5.2" ]]; then - CTX_HCA_PIN_HOST_DIR="$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic" - CTX_HCA_PIN_SCRIPT="$CTX_HCA_PIN_HOST_DIR/b300_ctx_hca_pin.sh" - if [[ ! -f "$CTX_HCA_PIN_SCRIPT" ]]; then - echo "Error: context HCA pinning preamble not found at $CTX_HCA_PIN_SCRIPT" >&2 - exit 1 - fi + # TRT-LLM rc26 ships NIXL v1.4.0 without its LIBFABRIC plugin. Stage the + # matching wheel plugin and EFA userspace runtime in a shared, immutable + # cache, then mount only those runtime libraries. + NIXL_LIBFABRIC_HOST_DIR="/data/home/sa-gha-runner/nixl-libfabric/nixl-1.4.0-efa-1.47.0" + mkdir -p "$(dirname "$NIXL_LIBFABRIC_HOST_DIR")" + ( + exec 9>"${NIXL_LIBFABRIC_HOST_DIR}.lock" + flock -w 1800 9 || exit 1 + + if [[ ! -r "$NIXL_LIBFABRIC_HOST_DIR/nixl/libplugin_LIBFABRIC.so" || + ! -r "$NIXL_LIBFABRIC_HOST_DIR/efa/opt/amazon/efa/lib/libfabric.so.1" || + ! -r "$NIXL_LIBFABRIC_HOST_DIR/efa/usr/lib/x86_64-linux-gnu/libibverbs/libefa-rdmav59.so" ]]; then + if [[ -e "$NIXL_LIBFABRIC_HOST_DIR" ]]; then + echo "Error: incomplete NIXL LIBFABRIC cache: $NIXL_LIBFABRIC_HOST_DIR" >&2 + exit 1 + fi + + _nixl_stage=$(mktemp -d "${NIXL_LIBFABRIC_HOST_DIR}.tmp.XXXXXX") + trap 'rm -rf -- "$_nixl_stage"' EXIT + mkdir -p "$_nixl_stage/runtime/nixl" "$_nixl_stage/runtime/efa" \ + "$_nixl_stage/installer" + + curl -LfsS --retry 3 -o "$_nixl_stage/nixl.whl" \ + "https://files.pythonhosted.org/packages/8b/7c/b79fb09e832233c90f1e9d9b953e88c2b92096d968f2444839c6aa92b645/nixl_cu13-1.4.0-cp312-cp312-manylinux_2_28_x86_64.whl" + echo "3e606fbe80c39ce14899726fad0cb0fec53c6bac9f34168492692c4166b2fabb $_nixl_stage/nixl.whl" | sha256sum -c - + unzip -p "$_nixl_stage/nixl.whl" \ + nixl_cu13.libs/nixl/libplugin_LIBFABRIC.so \ + > "$_nixl_stage/runtime/nixl/libplugin_LIBFABRIC.so" + unzip -p "$_nixl_stage/nixl.whl" \ + nixl_cu13.libs/libnuma-3387f5e3.so.1.0.0 \ + > "$_nixl_stage/runtime/nixl/libnuma-3387f5e3.so.1.0.0" + + curl -LfsS --retry 3 -o "$_nixl_stage/efa.tar.gz" \ + "https://efa-installer.amazonaws.com/aws-efa-installer-1.47.0.tar.gz" + echo "2df4201e046833c7dc8160907bee7f52b76ff80ed147376a2d0ed8a0dd66b2db $_nixl_stage/efa.tar.gz" | sha256sum -c - + tar -xzf "$_nixl_stage/efa.tar.gz" -C "$_nixl_stage/installer" \ + aws-efa-installer/DEBS/UBUNTU2404/x86_64/libfabric1-aws_2.4.0amzn1.0_amd64.deb \ + aws-efa-installer/DEBS/UBUNTU2404/x86_64/rdma-core/ibverbs-providers_61.0-1_amd64.deb \ + aws-efa-installer/DEBS/UBUNTU2404/x86_64/rdma-core/libibverbs1_61.0-1_amd64.deb \ + aws-efa-installer/DEBS/UBUNTU2404/x86_64/rdma-core/librdmacm1_61.0-1_amd64.deb \ + aws-efa-installer/DEBS/UBUNTU2404/x86_64/rdma-core/rdma-core_61.0-1_amd64.deb + while IFS= read -r -d '' _efa_deb; do + dpkg-deb -x "$_efa_deb" "$_nixl_stage/runtime/efa" + done < <(find "$_nixl_stage/installer" -name '*.deb' -print0) + mv "$_nixl_stage/runtime" "$NIXL_LIBFABRIC_HOST_DIR" + fi + ) || exit 1 + + # The cluster has EFA and Mellanox HCAs. Its Mellanox Enroot hook otherwise + # masks the EFA sysfs devices before the LIBFABRIC backend starts. + export MELLANOX_VISIBLE_DEVICES=void AIPERF_MMAP_CACHE_HOST_PATH="/data/home/sa-gha-runner/aiperf-cache" HF_HUB_CACHE_HOST_PATH="/data/home/sa-gha-runner/hf-hub-cache" @@ -246,11 +291,11 @@ if [[ "$IS_AGENTIC" == "1" && "$FRAMEWORK" == "dynamo-trt" && "$MODEL_PREFIX" == SRT_EXTRA_CLUSTER_CONFIG=$(cat < None assert '"$HF_CACHE_HOST_DIR:$HF_CACHE_CONTAINER_DIR"' in launcher +def test_b300_dsxe_glm52_stages_the_libfabric_runtime() -> None: + launcher = (REPO_ROOT / "runners/launch_b300-dsxe.sh").read_text() + + assert "nixl_cu13-1.4.0-cp312-cp312-manylinux_2_28_x86_64.whl" in launcher + assert "aws-efa-installer-1.47.0.tar.gz" in launcher + assert "libfabric1-aws_2.4.0amzn1.0_amd64.deb" in launcher + assert "ibverbs-providers_61.0-1_amd64.deb" in launcher + assert 'export MELLANOX_VISIBLE_DEVICES=void' in launcher + assert '"${NIXL_LIBFABRIC_HOST_DIR}": "/nixl-libfabric"' in launcher + assert "export NIXL_PLUGIN_DIR=/nixl-libfabric/nixl" in launcher + assert "export IBV_DRIVERS_PATH=/nixl-libfabric/efa/" in launcher + assert "export FI_PROVIDER=efa" in launcher + assert "export FI_EFA_USE_DEVICE_RDMA=1" in launcher + assert "NIXL_PLUGINS_DIR" not in launcher + + def test_patch_srt_eval_dispatch_forwards_selection_and_is_idempotent( tmp_path: Path, ) -> None: From 01c106fe62160cd34476e6a494a6156d8ac595fe Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Wed, 16 Sep 2026 23:50:18 -0700 Subject: [PATCH 12/16] fix(glm52-agentx): preserve MPI worker identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留 MPI 工作进程的 Slurm 用户身份，并在作业步骤独立的虚拟环境中安装 Dynamo。 --- .../configs/prepare-dynamo-venv.sh | 31 +++++++++++++++++++ ...agg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 2 ++ ...gg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml | 2 ++ ...gg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml | 2 ++ ...gg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml | 2 ++ ...-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml | 2 ++ ...-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 2 ++ configs/nvidia-master.yaml | 1 + perf-changelog.yaml | 7 +++++ runners/launch_b300-dsxe.sh | 2 +- 10 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/configs/prepare-dynamo-venv.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/prepare-dynamo-venv.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/prepare-dynamo-venv.sh new file mode 100644 index 0000000000..10ff55ae93 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/prepare-dynamo-venv.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Source before the SRT installer so its lock and packages share a writable prefix. +source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only || return 1 +check_env_vars SLURM_JOB_ID SLURM_STEP_ID || return 1 + +INFX_DYNAMO_VENV=$(python3 - <<'PY' +import fcntl +import os +import re +import venv +from pathlib import Path + +job = os.environ["SLURM_JOB_ID"] +step = os.environ["SLURM_STEP_ID"] +if not re.fullmatch(r"[0-9]+", job) or not re.fullmatch(r"[0-9]+", step): + raise ValueError("Dynamo virtual environment requires numeric Slurm job and step IDs") +root = Path("/tmp") / f"infx-dynamo-{job}.{step}" +root.mkdir(mode=0o700, exist_ok=True) +environment = root / "venv" +with (root / "create.lock").open("w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if not (root / "complete").exists(): + venv.EnvBuilder(system_site_packages=True, with_pip=True, symlinks=True).create(environment) + (root / "complete").touch() +print(environment) +PY +) || return 1 +export VIRTUAL_ENV="$INFX_DYNAMO_VENV" +export PATH="$VIRTUAL_ENV/bin:$PATH" +unset INFX_DYNAMO_VENV diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml index c7e1b34d0e..7e30cfb917 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -1,4 +1,6 @@ schema: 2 +srun_options: + no-container-remap-root: "" engine: type: trtllm publish_events_and_metrics: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml index 8a2c2fb58c..3d66db78ea 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml @@ -1,4 +1,6 @@ schema: 2 +srun_options: + no-container-remap-root: "" setup_script: diagnose-pmix.sh engine: type: trtllm diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml index 089e2216c8..7410784904 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml @@ -1,4 +1,6 @@ schema: 2 +srun_options: + no-container-remap-root: "" engine: type: trtllm publish_events_and_metrics: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml index 90bf9e1e2c..629ccb5062 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml @@ -1,4 +1,6 @@ schema: 2 +srun_options: + no-container-remap-root: "" engine: type: trtllm publish_events_and_metrics: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml index 457b5a9f05..8cd8da7ffd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml @@ -1,4 +1,6 @@ schema: 2 +srun_options: + no-container-remap-root: "" engine: type: trtllm publish_events_and_metrics: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml index 872deb1032..c189a53c46 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -1,4 +1,6 @@ schema: 2 +srun_options: + no-container-remap-root: "" engine: type: trtllm publish_events_and_metrics: false diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8c0e4c298c..3941b2b12c 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6825,6 +6825,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: dp-attn: true glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact: + # Preserve the Slurm worker identity and install Dynamo in a writable virtual environment. image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 model: nvidia/GLM-5.2-NVFP4 model-prefix: glm5.2 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c8cc5387d3..bc3967153a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8017,3 +8017,10 @@ - "Record bounded PMIx launch metadata before starting the concurrency-20 TensorRT-LLM workers." - "在启动并发 20 的 TensorRT-LLM 工作进程前，记录限定范围的 PMIx 启动元数据。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 + +- config-keys: + - glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact + description: + - "Preserve the Slurm identity for TensorRT-LLM MPI workers and install Dynamo in a separate virtual environment for each job step." + - "为 TensorRT-LLM MPI 工作进程保留 Slurm 身份，并在各作业步骤的独立虚拟环境中安装 Dynamo。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 78fe6d179f..434703170c 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -247,7 +247,7 @@ default_mounts: "${AIPERF_MMAP_CACHE_HOST_PATH}": "/aiperf_mmap_cache" "${HF_HUB_CACHE_HOST_PATH}": "/hf_hub_cache" "${TRTLLM_JIT_CACHE_HOST_PATH}": "/trtllm-jit-cache" -default_bash_preamble: "export NIXL_PLUGIN_DIR=/nixl-libfabric/nixl; export LD_LIBRARY_PATH=/nixl-libfabric/efa/opt/amazon/efa/lib:/nixl-libfabric/efa/usr/lib/x86_64-linux-gnu:/nixl-libfabric/nixl:\${LD_LIBRARY_PATH:-}; export IBV_DRIVERS_PATH=/nixl-libfabric/efa/usr/lib/x86_64-linux-gnu/libibverbs; export FI_PROVIDER=efa; export FI_EFA_USE_DEVICE_RDMA=1" +default_bash_preamble: "if command -v trtllm-llmapi-launch >/dev/null 2>&1; then source /configs/prepare-dynamo-venv.sh || exit; fi; export NIXL_PLUGIN_DIR=/nixl-libfabric/nixl; export LD_LIBRARY_PATH=/nixl-libfabric/efa/opt/amazon/efa/lib:/nixl-libfabric/efa/usr/lib/x86_64-linux-gnu:/nixl-libfabric/nixl:\${LD_LIBRARY_PATH:-}; export IBV_DRIVERS_PATH=/nixl-libfabric/efa/usr/lib/x86_64-linux-gnu/libibverbs; export FI_PROVIDER=efa; export FI_EFA_USE_DEVICE_RDMA=1" EOF ) fi From 0c6e465d73ce8b04b42ccbf87b95ba84c550aeac Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 17 Sep 2026 05:21:49 -0700 Subject: [PATCH 13/16] fix(glm52-agentx): allow time for the complete sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为 GLM-5.2 AgentX 作业显式设置 Slurm 时限，涵盖服务启动、完整回放和结果收集。 --- .../dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 2 ++ .../dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml | 2 ++ .../dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml | 2 ++ .../dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml | 2 ++ ...dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml | 2 ++ ...dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 2 ++ configs/nvidia-master.yaml | 1 + perf-changelog.yaml | 7 +++++++ 8 files changed, 20 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml index 7e30cfb917..2e4e6c77cf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -1,4 +1,6 @@ schema: 2 +slurm: + time_limit: "06:00:00" srun_options: no-container-remap-root: "" engine: diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml index 3d66db78ea..6f430c7379 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml @@ -1,4 +1,6 @@ schema: 2 +slurm: + time_limit: "06:00:00" srun_options: no-container-remap-root: "" setup_script: diagnose-pmix.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml index 7410784904..ae8c16c71e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml @@ -1,4 +1,6 @@ schema: 2 +slurm: + time_limit: "06:00:00" srun_options: no-container-remap-root: "" engine: diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml index 629ccb5062..e50f4fc033 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml @@ -1,4 +1,6 @@ schema: 2 +slurm: + time_limit: "06:00:00" srun_options: no-container-remap-root: "" engine: diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml index 8cd8da7ffd..e58e4a78ed 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml @@ -1,4 +1,6 @@ schema: 2 +slurm: + time_limit: "06:00:00" srun_options: no-container-remap-root: "" engine: diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml index c189a53c46..f4d42aa926 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -1,4 +1,6 @@ schema: 2 +slurm: + time_limit: "06:00:00" srun_options: no-container-remap-root: "" engine: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3941b2b12c..3913066b32 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6825,6 +6825,7 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache: dp-attn: true glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact: + # Allow Slurm time for server startup and the complete AgentX replay. # Preserve the Slurm worker identity and install Dynamo in a writable virtual environment. image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 model: nvidia/GLM-5.2-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index bc3967153a..a2cd4d55fc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8024,3 +8024,10 @@ - "Preserve the Slurm identity for TensorRT-LLM MPI workers and install Dynamo in a separate virtual environment for each job step." - "为 TensorRT-LLM MPI 工作进程保留 Slurm 身份，并在各作业步骤的独立虚拟环境中安装 Dynamo。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 + +- config-keys: + - glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact + description: + - "Set an explicit Slurm time limit for server startup, AgentX replay, and result collection." + - "显式设置 Slurm 时限，为服务启动、AgentX 回放和结果收集预留时间。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 From 921c7a6aa7973f7cef874c050537125918c98b9b Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Fri, 18 Sep 2026 09:49:20 -0700 Subject: [PATCH 14/16] fix(glm52): repair Dynamo install and KV transfer setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复 Dynamo 安装重试和 KV 传输上下文设置，并对齐评估端点的节点放置。 --- .../configs/prepare-dynamo-venv.sh | 26 +++++++++++++++++++ ...agg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 4 +++ ...gg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml | 8 ++++-- ...gg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml | 8 ++++-- ...gg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml | 8 ++++-- ...-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml | 4 +++ ...-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 4 +++ configs/nvidia-master.yaml | 2 ++ docs/configuration-procedures.md | 2 ++ docs/configuration-procedures_zh.md | 2 ++ perf-changelog.yaml | 6 +++++ 11 files changed, 68 insertions(+), 6 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/prepare-dynamo-venv.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/prepare-dynamo-venv.sh index 10ff55ae93..3134349590 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/prepare-dynamo-venv.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/prepare-dynamo-venv.sh @@ -29,3 +29,29 @@ PY export VIRTUAL_ENV="$INFX_DYNAMO_VENV" export PATH="$VIRTUAL_ENV/bin:$PATH" unset INFX_DYNAMO_VENV + +# Keep transient package download failures inside SRT's existing install lock. +# Other pip invocations retain their original behavior. +pip() { + if [[ $# -ne 7 || "$1" != install || "$2" != --break-system-packages || + "$3" != --quiet || "$4" != --extra-index-url || + "$5" != https://pypi.nvidia.com || "$6" != ai-dynamo-runtime==?* || + "$7" != "ai-dynamo==${6#ai-dynamo-runtime==}" ]]; then + command pip "$@" + return $? + fi + + local attempt install_status + for attempt in 1 2 3; do + if command pip "$@"; then + return 0 + else + install_status=$? + fi + if [[ "$attempt" -lt 3 ]]; then + echo "Dynamo install attempt $attempt failed; retrying unchanged packages." >&2 + sleep 5 || return $? + fi + done + return "$install_status" +} diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml index 2e4e6c77cf..cb257faf29 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -21,6 +21,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -100,6 +102,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml index 6f430c7379..a36774d9a2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml @@ -22,6 +22,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -101,6 +103,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -198,7 +202,7 @@ benchmark: TQDM_DISABLE: '1' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 placement: - node: first_decode + node: head dynamo: install: true source: @@ -224,7 +228,7 @@ frontend: ETCD_LEASE_TTL: '120' type: dynamo placement: - node: first_decode + node: head health_check: interval_seconds: 10 max_attempts: 540 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml index ae8c16c71e..526fbe5e9b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml @@ -21,6 +21,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -100,6 +102,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -195,7 +199,7 @@ benchmark: TQDM_DISABLE: '1' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 placement: - node: first_decode + node: head dynamo: install: true source: @@ -221,7 +225,7 @@ frontend: ETCD_LEASE_TTL: '120' type: dynamo placement: - node: first_decode + node: head health_check: interval_seconds: 10 max_attempts: 540 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml index e50f4fc033..4f067d4fe3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml @@ -21,6 +21,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -100,6 +102,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -197,7 +201,7 @@ benchmark: TQDM_DISABLE: '1' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 placement: - node: first_decode + node: head dynamo: install: true source: @@ -223,7 +227,7 @@ frontend: ETCD_LEASE_TTL: '120' type: dynamo placement: - node: first_decode + node: head health_check: interval_seconds: 10 max_attempts: 540 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml index e58e4a78ed..66c161d0ee 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml @@ -21,6 +21,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -100,6 +102,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml index f4d42aa926..632a339e3e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -21,6 +21,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -99,6 +101,8 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + NIXL_DISABLE_CUDA_ADDR_WA: '1' + OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1ace1c9400..10368897c2 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8152,6 +8152,8 @@ dsv41flash-fp4-h200-vllm-agentic-dspark: - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact: + # Use the LIBFABRIC primary CUDA context and colocate single-frontend eval endpoints. + # Bound worker OpenMP threads and retry Dynamo installs with unchanged package verification. # Allow Slurm time for server startup and the complete AgentX replay. # Preserve the Slurm worker identity and install Dynamo in a writable virtual environment. image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26.dev202609040000 diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index dc3498d46e..93b42cccc1 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -160,6 +160,8 @@ Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchm Do not ship one side alone. `srtctl` reads the recipe, while matrix generation reads the master config. Recipe-only changes can mislabel results. Master-only changes do not alter the deployed recipe. +The GLM-5.2 B300 compact Dynamo recipes prepare a writable virtual environment for workers and frontends. The helper makes up to three attempts to install the exact versioned Dynamo packages inside SRT's existing install lock, preserving package hash checks and the final failure status. Prefill and decode workers retain the `LIBFABRIC` NIXL backend and set `NIXL_DISABLE_CUDA_ADDR_WA=1`. They also set `OMP_NUM_THREADS=1` explicitly. For the single-frontend recipes selected for eval, `frontend.placement.node` and `benchmark.placement.node` are `head`, matching the pinned launcher's loopback eval endpoint. + ## Register an llm-d recipe Sources: [`benchmarks/llm-d/README.md`](../benchmarks/llm-d/README.md), [`benchmarks/multi_node/llm-d/README.md`](../benchmarks/multi_node/llm-d/README.md), [`llm-d-recipes/`](../benchmarks/multi_node/llm-d-recipes/), and the current [`llmd-vllm` benchmark wrapper](../benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 801a453598..513d688314 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -158,6 +158,8 @@ B200 Nscale 的 GLM-5.1 可用 `MODEL_PATH` 指定已有共享权重，覆盖默 不得只提交一侧：`srtctl` 读取配方，而矩阵生成读取主配置。仅改配方可能给结果贴错标签；仅改主配置不会改变实际部署的配方。 +GLM-5.2 B300 紧凑型 Dynamo 配方为工作进程和前端准备可写的虚拟环境。辅助脚本在 SRT 现有安装锁内，最多尝试三次安装相同的指定版本 Dynamo 软件包，并保留软件包哈希校验和最终失败状态。prefill 和 decode 工作进程继续使用 `LIBFABRIC` NIXL 后端，并设置 `NIXL_DISABLE_CUDA_ADDR_WA=1`，同时显式设置 `OMP_NUM_THREADS=1`。对于选中进行 eval 的单前端配方，`frontend.placement.node` 和 `benchmark.placement.node` 均设为 `head`，与固定版本启动脚本使用的本地回环 eval 端点保持一致。 + ## 注册 llm-d 配方 来源：[`benchmarks/llm-d/README.md`](../benchmarks/llm-d/README.md)、[`benchmarks/multi_node/llm-d/README.md`](../benchmarks/multi_node/llm-d/README.md)、[`llm-d-recipes/`](../benchmarks/multi_node/llm-d-recipes/) 和当前 [`llmd-vllm` 基准 wrapper](../benchmarks/multi_node/dsv4_fp4_gb200_llmd-vllm-disagg.sh)。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b7b678f651..3a86e9fbd2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8110,3 +8110,9 @@ - "Set an explicit Slurm time limit for server startup, AgentX replay, and result collection." - "显式设置 Slurm 时限，为服务启动、AgentX 回放和结果收集预留时间。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 + +- config-keys: + - glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact + description: + - "Use the LIBFABRIC primary CUDA context, set worker OpenMP threads explicitly, colocate single-frontend evaluation endpoints, and retry verified Dynamo installs. / 使用 LIBFABRIC 主 CUDA 上下文，显式设置工作进程的 OpenMP 线程数，将单前端评估端点与评估进程放在同一节点，并重试保留校验的 Dynamo 安装。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 From a91f7bb1d2c293edd9bfaa1cac51fb54a82ed40c Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Fri, 18 Sep 2026 11:26:06 -0700 Subject: [PATCH 15/16] fix: use standard CUDA KV allocations for GLM-5.2 B300 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为 GLM-5.2 B300 的 prefill 和 decode 工作进程采用旧版 KV 管理器和标准 CUDA KV 内存分配，并启用 LIBFABRIC 提供程序警告。 --- .../dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 6 ++++++ .../dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml | 6 ++++++ .../dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml | 6 ++++++ .../dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml | 6 ++++++ ...dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml | 6 ++++++ ...dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 6 ++++++ configs/nvidia-master.yaml | 1 + docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- perf-changelog.yaml | 7 +++++++ 10 files changed, 46 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml index cb257faf29..a8ba9878e7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -21,6 +21,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -35,6 +36,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -60,6 +62,7 @@ roles: enable_chunked_prefill: true gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: true event_buffer_max_size: 0 @@ -102,6 +105,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -116,6 +120,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -138,6 +143,7 @@ roles: enable_attention_dp: false gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: false event_buffer_max_size: 0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml index a36774d9a2..c96f6aaac1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml @@ -22,6 +22,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -36,6 +37,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -61,6 +63,7 @@ roles: enable_chunked_prefill: true gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: true event_buffer_max_size: 0 @@ -103,6 +106,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -117,6 +121,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -142,6 +147,7 @@ roles: enable_attention_dp: false gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: false event_buffer_max_size: 0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml index 526fbe5e9b..3af6bef607 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml @@ -21,6 +21,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -35,6 +36,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -60,6 +62,7 @@ roles: enable_chunked_prefill: true gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: true event_buffer_max_size: 0 @@ -102,6 +105,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -116,6 +120,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -139,6 +144,7 @@ roles: enable_attention_dp: false gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: false event_buffer_max_size: 0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml index 4f067d4fe3..5b72317fae 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml @@ -21,6 +21,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -35,6 +36,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -60,6 +62,7 @@ roles: enable_chunked_prefill: true gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: true event_buffer_max_size: 0 @@ -102,6 +105,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -116,6 +120,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -141,6 +146,7 @@ roles: enable_attention_dp: false gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: false event_buffer_max_size: 0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml index 66c161d0ee..ea5a46307f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml @@ -21,6 +21,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -35,6 +36,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -60,6 +62,7 @@ roles: enable_chunked_prefill: true gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: true event_buffer_max_size: 0 @@ -102,6 +105,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -116,6 +120,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -143,6 +148,7 @@ roles: enable_lm_head_tp_in_adp: false gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: false event_buffer_max_size: 0 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml index 632a339e3e..13ec009086 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -21,6 +21,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -35,6 +36,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -60,6 +62,7 @@ roles: enable_chunked_prefill: true gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: true event_buffer_max_size: 0 @@ -101,6 +104,7 @@ roles: HOME: /trtllm-jit-cache MIMALLOC_PURGE_DELAY: '0' NCCL_GRAPH_MIXING_SUPPORT: '0' + FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' OMP_NUM_THREADS: '1' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True @@ -115,6 +119,7 @@ roles: TRTLLM_DSA_INDEXER_BF16: '1' TRTLLM_ENABLE_PDL: '1' TRTLLM_FUSED_DSA_METADATA: '1' + TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY: '0' TRTLLM_KVCACHE_RECV_BUFFER_COUNT: '1' TRTLLM_KVCACHE_SEND_BUFFER_COUNT: '1' TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: '600' @@ -142,6 +147,7 @@ roles: enable_lm_head_tp_in_adp: false gpus_per_node: 8 kv_cache_config: + use_kv_cache_manager_v2: false dtype: fp8 enable_block_reuse: false event_buffer_max_size: 0 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 10368897c2..1a15972068 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8152,6 +8152,7 @@ dsv41flash-fp4-h200-vllm-agentic-dspark: - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact: + # Use legacy KV pools with standard CUDA allocations and emit LIBFABRIC warnings. # Use the LIBFABRIC primary CUDA context and colocate single-frontend eval endpoints. # Bound worker OpenMP threads and retry Dynamo installs with unchanged package verification. # Allow Slurm time for server startup and the complete AgentX replay. diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 93b42cccc1..08af29b0da 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -160,7 +160,7 @@ Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchm Do not ship one side alone. `srtctl` reads the recipe, while matrix generation reads the master config. Recipe-only changes can mislabel results. Master-only changes do not alter the deployed recipe. -The GLM-5.2 B300 compact Dynamo recipes prepare a writable virtual environment for workers and frontends. The helper makes up to three attempts to install the exact versioned Dynamo packages inside SRT's existing install lock, preserving package hash checks and the final failure status. Prefill and decode workers retain the `LIBFABRIC` NIXL backend and set `NIXL_DISABLE_CUDA_ADDR_WA=1`. They also set `OMP_NUM_THREADS=1` explicitly. For the single-frontend recipes selected for eval, `frontend.placement.node` and `benchmark.placement.node` are `head`, matching the pinned launcher's loopback eval endpoint. +The GLM-5.2 B300 compact Dynamo recipes prepare a writable virtual environment for workers and frontends. The helper makes up to three attempts to install the exact versioned Dynamo packages inside SRT's existing install lock, preserving package hash checks and the final failure status. Prefill and decode workers retain the `LIBFABRIC` NIXL backend and set `NIXL_DISABLE_CUDA_ADDR_WA=1`. Both roles select `kv_cache_config.use_kv_cache_manager_v2: false` and set `TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=0` to use the legacy KV manager with standard CUDA KV allocations. The GLM-5.2 indexer and MTP settings remain explicit. They also set `OMP_NUM_THREADS=1` and `FI_LOG_LEVEL=warn` so LIBFABRIC reports provider warnings for both worker roles. For the single-frontend recipes selected for eval, `frontend.placement.node` and `benchmark.placement.node` are `head`, matching the pinned launcher's loopback eval endpoint. ## Register an llm-d recipe diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 513d688314..da66641aa8 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -158,7 +158,7 @@ B200 Nscale 的 GLM-5.1 可用 `MODEL_PATH` 指定已有共享权重，覆盖默 不得只提交一侧：`srtctl` 读取配方，而矩阵生成读取主配置。仅改配方可能给结果贴错标签；仅改主配置不会改变实际部署的配方。 -GLM-5.2 B300 紧凑型 Dynamo 配方为工作进程和前端准备可写的虚拟环境。辅助脚本在 SRT 现有安装锁内，最多尝试三次安装相同的指定版本 Dynamo 软件包，并保留软件包哈希校验和最终失败状态。prefill 和 decode 工作进程继续使用 `LIBFABRIC` NIXL 后端，并设置 `NIXL_DISABLE_CUDA_ADDR_WA=1`，同时显式设置 `OMP_NUM_THREADS=1`。对于选中进行 eval 的单前端配方，`frontend.placement.node` 和 `benchmark.placement.node` 均设为 `head`，与固定版本启动脚本使用的本地回环 eval 端点保持一致。 +GLM-5.2 B300 紧凑型 Dynamo 配方为工作进程和前端准备可写的虚拟环境。辅助脚本在 SRT 现有安装锁内，最多尝试三次安装相同的指定版本 Dynamo 软件包，并保留软件包哈希校验和最终失败状态。prefill 和 decode 工作进程继续使用 `LIBFABRIC` NIXL 后端，并设置 `NIXL_DISABLE_CUDA_ADDR_WA=1`，两类工作进程均选择 `kv_cache_config.use_kv_cache_manager_v2: false` 并设置 `TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=0`，使用旧版 KV 管理器和标准 CUDA KV 内存分配。GLM-5.2 的 indexer 和 MTP 设置保持显式配置，同时设置 `OMP_NUM_THREADS=1` 和 `FI_LOG_LEVEL=warn`，让 LIBFABRIC 输出这两类工作进程的提供程序警告。对于选中进行 eval 的单前端配方，`frontend.placement.node` 和 `benchmark.placement.node` 均设为 `head`，与固定版本启动脚本使用的本地回环 eval 端点保持一致。 ## 注册 llm-d 配方 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 705b5ead86..3aad0b55dc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8125,3 +8125,10 @@ description: - "Use the LIBFABRIC primary CUDA context, set worker OpenMP threads explicitly, colocate single-frontend evaluation endpoints, and retry verified Dynamo installs. / 使用 LIBFABRIC 主 CUDA 上下文，显式设置工作进程的 OpenMP 线程数，将单前端评估端点与评估进程放在同一节点，并重试保留校验的 Dynamo 安装。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 + +- config-keys: + - glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact + description: + - "Use the legacy KV manager with standard CUDA KV allocations and enable LIBFABRIC provider warnings for GLM-5.2 B300 prefill and decode workers." + - "为 GLM-5.2 B300 的 prefill 和 decode 工作进程采用旧版 KV 管理器和标准 CUDA KV 内存分配，并启用 LIBFABRIC 提供程序警告。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 From e683a557f12fbfe957fe922014950d322a5e8879 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Sun, 20 Sep 2026 22:25:06 -0700 Subject: [PATCH 16/16] fix(glm52): harden B300 worker startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为 B300 GLM-5.2 工作进程分配独立节点，清理过期共享内存，设置 CPU 亲和性和工作进程环境，并缩短注册等待时限。 --- .../configs/clean_stale_shm.sh | 44 ++++++++++ .../configs/diagnose-pmix.sh | 2 + ...agg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml | 15 ++-- ...gg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml | 14 ++-- ...gg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml | 17 ++-- ...gg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml | 19 +++-- ...-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml | 17 ++-- ...-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml | 17 ++-- configs/nvidia-master.yaml | 2 + docs/configuration-procedures.md | 2 +- docs/configuration-procedures_zh.md | 2 +- perf-changelog.yaml | 7 ++ utils/test_clean_stale_shm.py | 83 +++++++++++++++++++ 13 files changed, 204 insertions(+), 37 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/configs/clean_stale_shm.sh create mode 100644 utils/test_clean_stale_shm.py diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/clean_stale_shm.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/clean_stale_shm.sh new file mode 100644 index 0000000000..64e2e169f9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/clean_stale_shm.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Use only with exclusive node allocations. Anchor the cutoff to allocation +# start so a later frontend/worker setup cannot remove this job's segments. +clean_stale_shm() { + python3 -S - "$@" <<'PY' +import os +import socket +import stat +import sys +import time +from pathlib import Path + +directory = Path(sys.argv[1]) +job_start = int(sys.argv[2]) +if job_start <= 600 or job_start > time.time(): + raise ValueError("SLURM_JOB_START_TIME must be a valid allocation start timestamp") +cutoff = job_start - 600 +prefixes = ("vader_segment.", "nccl-", "sem.", "psm3", "fe80::") +entries = list(directory.iterdir()) +removed = 0 +for entry in entries: + if not entry.name.startswith(prefixes): + continue + try: + info = entry.lstat() + if (info.st_uid != os.getuid() or not stat.S_ISREG(info.st_mode) + or info.st_mtime >= cutoff): + continue + entry.unlink() + removed += 1 + except FileNotFoundError: + # Another rank may have already removed the same stale segment. + continue +print(f"[clean_stale_shm] {socket.gethostname()}: removed {removed} stale segments " + f"from {len(entries)} entries; cutoff={cutoff}") +PY +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only || exit 1 + check_env_vars SLURM_JOB_START_TIME || exit 1 + clean_stale_shm /dev/shm "$SLURM_JOB_START_TIME" +fi diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/diagnose-pmix.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/diagnose-pmix.sh index f19fa91a4c..23364bf215 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/diagnose-pmix.sh +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/diagnose-pmix.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +bash "$(dirname "${BASH_SOURCE[0]}")/clean_stale_shm.sh" || exit 1 + # Read launch metadata without initializing MPI or changing its configuration. python3 -S - <<'PY' import json diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml index 2d7cee6b5a..237b2cce17 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5.yaml @@ -3,8 +3,10 @@ slurm: time_limit: "06:00:00" srun_options: no-container-remap-root: "" +setup_script: clean_stale_shm.sh engine: type: trtllm + numa_cpu_bind: true publish_events_and_metrics: false roles: prefill: @@ -16,14 +18,15 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -99,14 +102,15 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -231,7 +235,7 @@ frontend: node: first_decode health_check: interval_seconds: 10 - max_attempts: 540 + max_attempts: 180 identity: model: repo: nvidia/GLM-5.2-NVFP4 @@ -246,5 +250,6 @@ model: precision: fp4 name: dynamo-disagg-b300-1p1d-tep8-compact-c1-b1-mtp5 resources: + spread_workers: true gpu_type: b300 gpus_per_node: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml index ed2031fd75..987142faa4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5.yaml @@ -6,6 +6,7 @@ srun_options: setup_script: diagnose-pmix.sh engine: type: trtllm + numa_cpu_bind: true publish_events_and_metrics: false roles: prefill: @@ -17,15 +18,16 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' NIXL_LIBFABRIC_NUM_THREADS: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -102,14 +104,15 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -236,7 +239,7 @@ frontend: node: head health_check: interval_seconds: 10 - max_attempts: 540 + max_attempts: 180 identity: model: repo: nvidia/GLM-5.2-NVFP4 @@ -251,5 +254,6 @@ model: precision: fp4 name: dynamo-disagg-b300-1p1d-tep8-compact-c20-b5-mtp5 resources: + spread_workers: true gpu_type: b300 gpus_per_node: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml index d56e5dad66..b86a68d19e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5.yaml @@ -3,8 +3,10 @@ slurm: time_limit: "06:00:00" srun_options: no-container-remap-root: "" +setup_script: clean_stale_shm.sh engine: type: trtllm + numa_cpu_bind: true publish_events_and_metrics: false roles: prefill: @@ -16,14 +18,15 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -91,7 +94,7 @@ roles: tensor_parallel_size: 4 trust_remote_code: true decode: - nodes: 2 + nodes: 4 workers: 4 gpus: 4 env: @@ -99,14 +102,15 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -231,7 +235,7 @@ frontend: node: head health_check: interval_seconds: 10 - max_attempts: 540 + max_attempts: 180 identity: model: repo: nvidia/GLM-5.2-NVFP4 @@ -246,5 +250,6 @@ model: precision: fp4 name: dynamo-disagg-b300-1p4d-tep4-compact-c30-b2-mtp5 resources: + spread_workers: true gpu_type: b300 gpus_per_node: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml index 3a657fda6a..6d1f7b408f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5.yaml @@ -3,12 +3,14 @@ slurm: time_limit: "06:00:00" srun_options: no-container-remap-root: "" +setup_script: clean_stale_shm.sh engine: type: trtllm + numa_cpu_bind: true publish_events_and_metrics: false roles: prefill: - nodes: 2 + nodes: 3 workers: 3 gpus: 4 env: @@ -16,14 +18,15 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -91,7 +94,7 @@ roles: tensor_parallel_size: 4 trust_remote_code: true decode: - nodes: 2 + nodes: 4 workers: 4 gpus: 4 env: @@ -99,14 +102,15 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -233,7 +237,7 @@ frontend: node: head health_check: interval_seconds: 10 - max_attempts: 540 + max_attempts: 180 identity: model: repo: nvidia/GLM-5.2-NVFP4 @@ -248,5 +252,6 @@ model: precision: fp4 name: dynamo-disagg-b300-3p4d-tep4-compact-c60-b5-mtp5 resources: + spread_workers: true gpu_type: b300 gpus_per_node: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml index cccdaf3c34..fcced4bc56 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3.yaml @@ -3,12 +3,14 @@ slurm: time_limit: "06:00:00" srun_options: no-container-remap-root: "" +setup_script: clean_stale_shm.sh engine: type: trtllm + numa_cpu_bind: true publish_events_and_metrics: false roles: prefill: - nodes: 3 + nodes: 6 workers: 6 gpus: 4 env: @@ -16,14 +18,15 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -99,14 +102,15 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -233,7 +237,7 @@ frontend: type: dynamo health_check: interval_seconds: 10 - max_attempts: 540 + max_attempts: 180 identity: model: repo: nvidia/GLM-5.2-NVFP4 @@ -248,5 +252,6 @@ model: precision: fp4 name: dynamo-disagg-b300-6p1d-dep8-compact-c227-b16-mtp3 resources: + spread_workers: true gpu_type: b300 gpus_per_node: 8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml index aa9d3932bb..d6fb52ed35 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/b300-fp4/agentic/dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3.yaml @@ -3,12 +3,14 @@ slurm: time_limit: "06:00:00" srun_options: no-container-remap-root: "" +setup_script: clean_stale_shm.sh engine: type: trtllm + numa_cpu_bind: true publish_events_and_metrics: false roles: prefill: - nodes: 4 + nodes: 8 workers: 8 gpus: 4 env: @@ -16,14 +18,15 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -98,14 +101,15 @@ roles: DYN_PUBLISH_KV_EVENTS: '0' DYN_TOKENIZER: fastokens DYN_TRTLLM_ENABLE_ATTENTION_DP: '1' + ETCD_LEASE_TTL: '120' HF_HUB_DISABLE_PROGRESS_BARS: '1' HF_HUB_OFFLINE: '1' HOME: /trtllm-jit-cache - MIMALLOC_PURGE_DELAY: '0' + MIMALLOC_PURGE_DELAY: '10000' NCCL_GRAPH_MIXING_SUPPORT: '0' FI_LOG_LEVEL: warn NIXL_DISABLE_CUDA_ADDR_WA: '1' - OMP_NUM_THREADS: '1' + OMP_NUM_THREADS: '12' PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' TLLM_EXECUTOR_BATCH_RESP_IN_AWAIT: '1' @@ -232,7 +236,7 @@ frontend: type: dynamo health_check: interval_seconds: 10 - max_attempts: 540 + max_attempts: 180 identity: model: repo: nvidia/GLM-5.2-NVFP4 @@ -247,5 +251,6 @@ model: precision: fp4 name: dynamo-disagg-b300-8p2d-dep8-compact-c233-b16-mtp3 resources: + spread_workers: true gpu_type: b300 gpus_per_node: 8 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 184fdbc54a..c10ba931a8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8377,6 +8377,8 @@ qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp: name: hicache glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact: + # Place each worker on its own node; recipe node counts supply scheduling labels. + # Set worker CPU affinity and environment, and bound readiness to 30 minutes. # Forward the workflow result identity and preserve the AgentX producer exit status. # Use legacy KV pools with standard CUDA allocations and emit LIBFABRIC warnings. # Use the LIBFABRIC primary CUDA context and colocate single-frontend eval endpoints. diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 6cc9a32b72..72144e7ae0 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -183,7 +183,7 @@ Do not ship one side alone. `srtctl` reads the recipe, while matrix generation r The B300 AgentX launcher forwards the workflow-owned `RESULT_FILENAME` through a native SRT override, and the GLM-5.2 compact recipes leave that name to the caller. It collects the resulting `_concN.json` files from the mounted workspace and requires the Slurm allocation to finish as `COMPLETED` with `ExitCode=0:0`; partial aggregates cannot hide a failed replay or request-error gate. Accounting checks retry briefly for delayed terminal records, and the launcher stages logs and results before returning a failure. -The GLM-5.2 B300 compact Dynamo recipes prepare a writable virtual environment for workers and frontends. The helper makes up to three attempts to install the exact versioned Dynamo packages inside SRT's existing install lock, preserving package hash checks and the final failure status. Prefill and decode workers retain the `LIBFABRIC` NIXL backend and set `NIXL_DISABLE_CUDA_ADDR_WA=1`. Both roles select `kv_cache_config.use_kv_cache_manager_v2: false` and set `TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=0` to use the legacy KV manager with standard CUDA KV allocations. The GLM-5.2 indexer and MTP settings remain explicit. They also set `OMP_NUM_THREADS=1` and `FI_LOG_LEVEL=warn` so LIBFABRIC reports provider warnings for both worker roles. For the single-frontend recipes selected for eval, `frontend.placement.node` and `benchmark.placement.node` are `head`, matching the pinned launcher's loopback eval endpoint. +The GLM-5.2 B300 compact Dynamo recipes prepare a writable virtual environment for workers and frontends. The helper makes up to three attempts to install the exact versioned Dynamo packages inside SRT's existing install lock, preserving package hash checks and the final failure status. Prefill and decode workers retain the `LIBFABRIC` NIXL backend and set `NIXL_DISABLE_CUDA_ADDR_WA=1`. Both roles select `kv_cache_config.use_kv_cache_manager_v2: false` and set `TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=0` to use the legacy KV manager with standard CUDA KV allocations. The GLM-5.2 indexer and MTP settings remain explicit. Both worker roles set `OMP_NUM_THREADS=12`, `MIMALLOC_PURGE_DELAY=10000`, `ETCD_LEASE_TTL=120`, and `FI_LOG_LEVEL=warn`. Each worker occupies a separate node (`resources.spread_workers: true`, with role node counts equal to worker counts). The native `engine.numa_cpu_bind` helper applies each MPI rank's GPU-local NUMA CPU mask before execution, without OpenMP binding variables. The readiness gate requires all workers to register within 30 minutes for both benchmark and eval jobs; the allocation retains its six-hour limit for completed workloads. A setup helper removes matching, same-user stale shared-memory files only when they predate the allocation start by more than ten minutes. This fixed cutoff preserves the current allocation's files when setup runs again for another rank or frontend; concurrency 20 retains its PMIx diagnostics after cleanup. For the single-frontend recipes selected for eval, `frontend.placement.node` and `benchmark.placement.node` are `head`, matching the pinned launcher's loopback eval endpoint. ## Register an llm-d recipe diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index f2afc19f16..36dae84ede 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -160,7 +160,7 @@ B200 Nscale 的 GLM-5.1 可用 `MODEL_PATH` 指定已有共享权重，覆盖默 B300 AgentX 启动脚本通过 SRT 原生覆盖参数传递工作流提供的 `RESULT_FILENAME`，GLM-5.2 紧凑型配方由调用方指定该名称。启动脚本从挂载的工作区收集生成的 `_concN.json` 文件，并要求 Slurm 作业以 `COMPLETED` 状态及 `ExitCode=0:0` 结束，避免部分聚合结果掩盖回放失败或请求错误率检查失败。对于延迟出现的最终记账记录，检查会进行有限次数的重试；启动脚本在返回失败前保留日志和结果。 -GLM-5.2 B300 紧凑型 Dynamo 配方为工作进程和前端准备可写的虚拟环境。辅助脚本在 SRT 现有安装锁内，最多尝试三次安装相同的指定版本 Dynamo 软件包，并保留软件包哈希校验和最终失败状态。prefill 和 decode 工作进程继续使用 `LIBFABRIC` NIXL 后端，并设置 `NIXL_DISABLE_CUDA_ADDR_WA=1`，两类工作进程均选择 `kv_cache_config.use_kv_cache_manager_v2: false` 并设置 `TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=0`，使用旧版 KV 管理器和标准 CUDA KV 内存分配。GLM-5.2 的 indexer 和 MTP 设置保持显式配置，同时设置 `OMP_NUM_THREADS=1` 和 `FI_LOG_LEVEL=warn`，让 LIBFABRIC 输出这两类工作进程的提供程序警告。对于选中进行 eval 的单前端配方，`frontend.placement.node` 和 `benchmark.placement.node` 均设为 `head`，与固定版本启动脚本使用的本地回环 eval 端点保持一致。 +GLM-5.2 B300 紧凑型 Dynamo 配方为工作进程和前端准备可写的虚拟环境。辅助脚本在 SRT 现有安装锁内，最多尝试三次安装相同的指定版本 Dynamo 软件包，并保留软件包哈希校验和最终失败状态。prefill 和 decode 工作进程继续使用 `LIBFABRIC` NIXL 后端，并设置 `NIXL_DISABLE_CUDA_ADDR_WA=1`，两类工作进程均选择 `kv_cache_config.use_kv_cache_manager_v2: false` 并设置 `TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=0`，使用旧版 KV 管理器和标准 CUDA KV 内存分配。GLM-5.2 的 indexer 和 MTP 设置保持显式配置，两类工作进程均设置 `OMP_NUM_THREADS=12`、`MIMALLOC_PURGE_DELAY=10000`、`ETCD_LEASE_TTL=120` 和 `FI_LOG_LEVEL=warn`。每个工作进程独占一个节点（`resources.spread_workers: true`，各角色的节点数等于工作进程数）。原生 `engine.numa_cpu_bind` 辅助脚本在执行前，为各 MPI rank 应用其 GPU 所在 NUMA 节点的 CPU 掩码，不设置 OpenMP 绑定变量。基准测试和 eval 的就绪检查都要求全部工作进程在 30 分钟内完成注册；作业仍保留六小时的总时限以完成负载。启动脚本只清理当前用户拥有、名称匹配且修改时间早于作业分配开始时间十分钟以上的共享内存文件。固定的时间界限可避免其他 rank 或前端再次运行启动脚本时删除本次作业的文件；并发 20 的配方在清理后继续记录 PMIx 诊断信息。对于选中进行 eval 的单前端配方，`frontend.placement.node` 和 `benchmark.placement.node` 均设为 `head`，与固定版本启动脚本使用的本地回环 eval 端点保持一致。 ## 注册 llm-d 配方 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6507da21f2..eec28a8f79 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8510,3 +8510,10 @@ - "Set a single prefill LIBFABRIC posting thread for GLM-5.2 c20 and select acceptance automatically for the compact GLM-5.2 recipes." - "为 GLM-5.2 c20 设置单个预填充 LIBFABRIC 提交线程，并为紧凑型 GLM-5.2 配方自动选择接受长度。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 + +- config-keys: + - glm5.2-fp4-b300-dynamo-trt-agentic-mtp-compact + description: + - "Spread GLM-5.2 B300 workers across nodes, clean stale shared memory before startup, bind worker CPUs by GPU locality, and bound worker registration waits." + - "将 GLM-5.2 B300 工作进程分散到独立节点，启动前清理过期共享内存，按 GPU 的 NUMA 位置绑定工作进程 CPU，并限制工作进程注册等待时间。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2993 diff --git a/utils/test_clean_stale_shm.py b/utils/test_clean_stale_shm.py new file mode 100644 index 0000000000..b0bc8c0bc1 --- /dev/null +++ b/utils/test_clean_stale_shm.py @@ -0,0 +1,83 @@ +"""Exercise shared-memory cleanup against isolated filesystem fixtures.""" + +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + + +SCRIPT = ( + Path(__file__).resolve().parents[1] + / "benchmarks/multi_node/srt-slurm-recipes/configs/clean_stale_shm.sh" +) + + +def run_cleanup(directory: Path, job_start: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "bash", "-c", 'source "$1"; clean_stale_shm "$2" "$3"', + "bash", str(SCRIPT), str(directory), job_start, + ], + env={**os.environ, "PATH": f"{Path(sys.executable).parent}:{os.environ['PATH']}"}, + capture_output=True, + text=True, + check=False, + ) + + +def segment(path: Path, timestamp: int) -> None: + path.write_text("segment") + os.utime(path, (timestamp, timestamp)) + + +def test_cleanup_removes_only_matching_old_regular_files(tmp_path: Path) -> None: + job_start = int(time.time()) + for name in ("vader_segment.old", "nccl-old", "sem.old", "psm3old", "fe80::old"): + segment(tmp_path / name, job_start - 601) + segment(tmp_path / "nccl-boundary", job_start - 600) + segment(tmp_path / "nccl-current", job_start) + segment(tmp_path / "unrelated", job_start - 3600) + directory = tmp_path / "nccl-directory" + directory.mkdir() + segment(directory / "child", job_start - 3600) + os.utime(directory, (job_start - 3600, job_start - 3600)) + (tmp_path / "nccl-link").symlink_to(tmp_path / "unrelated") + + result = run_cleanup(tmp_path, str(job_start)) + + assert result.returncode == 0, result.stderr + assert "removed 5 stale segments" in result.stdout + assert {path.name for path in tmp_path.iterdir()} == { + "nccl-boundary", "nccl-current", "unrelated", "nccl-directory", "nccl-link", + } + assert (directory / "child").read_text() == "segment" + assert (tmp_path / "nccl-link").is_symlink() + + +def test_delayed_repeated_setup_keeps_current_allocation_segments(tmp_path: Path) -> None: + job_start = int(time.time()) - 3600 + segment(tmp_path / "nccl-previous", job_start - 601) + segment(tmp_path / "nccl-this-job", job_start + 1) + + first = run_cleanup(tmp_path, str(job_start)) + second = run_cleanup(tmp_path, str(job_start)) + + assert first.returncode == second.returncode == 0, first.stderr + second.stderr + assert "removed 1 stale segments" in first.stdout + assert "removed 0 stale segments" in second.stdout + assert (tmp_path / "nccl-this-job").read_text() == "segment" + assert not (tmp_path / "nccl-previous").exists() + + +@pytest.mark.parametrize("job_start", ["invalid", "0", "9999999999"]) +def test_invalid_allocation_start_leaves_files_untouched(tmp_path: Path, job_start: str) -> None: + segment(tmp_path / "nccl-old", 1) + + result = run_cleanup(tmp_path, job_start) + + assert result.returncode != 0 + assert "ValueError" in result.stderr + assert (tmp_path / "nccl-old").read_text() == "segment"