From 13de142f3fe6e71d97a2dcbe65cfa4ea89844710 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sun, 27 Sep 2026 15:47:49 -0400 Subject: [PATCH] [Klaud Cold] Add Kimi-K3 B300 Mooncake EFA AgentX disagg 1P1D DCP8 DSpark4 c48 on efa_pr_58768 image Co-Authored-By: Claude Opus 5.5 (1M context) --- ...g-1p1d-dcp8-dcp8-dspark4-mooncake-c48.yaml | 237 ++++++++++++++++++ configs/nvidia-master.yaml | 29 +++ perf-changelog.yaml | 9 + runners/launch_b300-dsxe.sh | 14 +- 4 files changed, 287 insertions(+), 2 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake-c48.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake-c48.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake-c48.yaml new file mode 100644 index 0000000000..dc8f472f69 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/b300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake-c48.yaml @@ -0,0 +1,237 @@ +schema: 2 +name: kimi-k3-vllm-disagg-b300-1p1d-dcp8-dcp8-dspark4-mooncake-c48-agentic +model: + path: moonshotai/Kimi-K3 + container: ghcr.io#semianalysisai/vllm-openai:efa_pr_58768 + precision: fp4 +environment: {LD_LIBRARY_PATH: "/opt/amazon/efa/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib"} +dynamo: + install: true + source: + rev: cfada2fd9d17bfa6bb68dbee9d2f455e12577b8b +slurm: + time_limit: 06:00:00 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +services: +- name: etcd + type: etcd + placement: + node: infra +- name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +- name: mooncake-master + type: mooncake-master + args: + - --default_kv_lease_ttl=60000 + - --eviction_high_watermark_ratio=0.95 + - --eviction_ratio=0.10 + options: + store_config: + metadata_server: P2PHANDSHAKE + global_segment_size: 190GB + local_buffer_size: 4GB + protocol: efa + mode: embedded + enable_offload: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: vllm + trust-remote-code: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + enable-auto-tool-choice: true + router-mode: least-loaded + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: '3600' + DYN_TOKENIZER_CACHE_BYTES: '8589934592' +engine: + type: vllm + connector: null + dp_launch_mode: per_node +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-b300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + GLIBC_TUNABLES: glibc.malloc.hugetlb=1 + MC_STORE_MEMCPY: '1' + MC_WORKERS_PER_CTX: '4' + args: + enable-logging-iteration-details: true + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"backends":["LIBFABRIC"]}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + mamba-ssm-cache-dtype: bfloat16 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + cp-kv-cache-interleave-size: 1 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' + enable-prefix-caching: true + enable-flashinfer-autotune: true + kv_events: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + VLLM_USE_DIRECT_DCP_A2A: '1' + VLLM_USE_DIRECT_DCP_Q_GATHER: '1' + VLLM_USE_DIRECT_DCP_KV_GATHER: '1' + VLLM_ALLREDUCE_USE_FLASHINFER: '1' + VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: '0' + DYN_REQUEST_PLANE: tcp + ETCD_LEASE_TTL: '600' + VLLM_ENGINE_READY_TIMEOUT_S: '3600' + VLLM_RPC_TIMEOUT: '600000' + TILELANG_CLEANUP_TEMP_FILES: '1' + VLLM_USE_NCCL_SYMM_MEM: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '0' + NCCL_NVLS_ENABLE: '1' + VLLM_SERVER_DEV_MODE: '1' + VLLM_USE_V2_MODEL_RUNNER: '1' + VLLM_USE_RUST_FRONTEND: '1' + VLLM_MOONCAKE_LOAD_RECV_THREADS: '4' + MC_SLICE_SIZE: '1048576' + VLLM_CONNECTOR_PREFETCH_DEPTH: '8' + VLLM_CONNECTOR_PREFETCH_KV_CAP: '0.65' + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: '1800' + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: '0' + NCCL_P2P_LEVEL: NVL + MC_ENABLE_DEST_DEVICE_AFFINITY: '1' + WITH_NVIDIA_PEERMEM: '0' + NCCL_NET_PLUGIN: none + UCX_MEMTYPE_CACHE: n + UCX_MEMTYPE_REG_WHOLE: n + UCX_RCACHE_MAX_UNRELEASED: '1024' + UCX_TCP_AF_PRIO: inet + VLLM_SSM_CONV_STATE_LAYOUT: DS + DG_JIT_CACHE_DIR: /tmp/dg-cache-kimi-k3-b300-pd-dspark-mooncake-{job_id} + PYTHONHASHSEED: '42' + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + TORCH_CUDA_ARCH_LIST: '10.0' + MC_STORE_CLIENT_METRIC: '1' + MC_STORE_CLIENT_METRIC_INTERVAL: '5' + MC_TE_METRIC: '0' + GLIBC_TUNABLES: glibc.malloc.hugetlb=1 + MC_STORE_MEMCPY: '1' + MC_WORKERS_PER_CTX: '4' + args: + enable-logging-iteration-details: true + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"backends":["LIBFABRIC"]}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"compact_group_io":true,"max_load_batch_keys":2,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: moonshotai/Kimi-K3 + prefix-match-unit: 128 + load-format: fastsafetensors + kv-cache-dtype: fp8 + mamba-ssm-cache-dtype: bfloat16 + gpu-memory-utilization: 0.94 + tensor-parallel-size: 8 + decode-context-parallel-size: 8 + cp-kv-cache-interleave-size: 1 + dcp-comm-backend: a2a + enable-cumem-allocator: true + trust-remote-code: true + max-cudagraph-capture-size: 512 + stream-interval: 10 + language-model-only: true + attention-backend: TOKENSPEED_MLA + attention-config: '{"mla_prefill_backend": "TRTLLM_RAGGED", "use_prefill_query_quantization": true}' + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","attention_backend":"TOKENSPEED_MLA","method":"dspark","num_speculative_tokens":4,"rejection_sample_method":"block","draft_sample_method":"probabilistic"}' + enable-prefix-caching: true + enable-flashinfer-autotune: true +sbatch_directives: + cpus-per-task: '72' + mem: '0' + comment: '''{"OccupiedIdleGPUsJobReaper":{"exemptIdleTimeMins":"60","reason":"model_loading","description":"Very large model will take extra loading time."}}''' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + client_placement: head + concurrencies: + - 48 +identity: + model: + repo: moonshotai/Kimi-K3 + container: + image: ghcr.io#semianalysisai/vllm-openai:efa_pr_58768 + frameworks: + dynamo: cfada2fd9d17bfa6bb68dbee9d2f455e12577b8b diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b0fc9ef451..caf2be2963 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1528,6 +1528,35 @@ kimik3-fp4-b300-vllm-agentic-dspark: # disjoint across arms so exp-names stay unique. - { tp: 8, ep: 1, dcp-size: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 70], srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/b300-fp4-mtp/agentic.yaml } +kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg-1p1d: + image: ghcr.io#semianalysisai/vllm-openai:efa_pr_58768 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.75 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13.post1" } + conc-list: [48] + prefill: + num-worker: 1 + tp: 8 + dcp-size: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/kimik3/vllm/b300-fp4/agentx/disagg-1p1d-dcp8-dcp8-dspark4-mooncake-c48.yaml" + decode: { num-worker: 1, tp: 8, dcp-size: 8, ep: 1, dp-attn: false } + dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 model: deepseek-ai/DeepSeek-R1-0528 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 352c61fc0d..ab78252856 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8986,3 +8986,12 @@ - "Hopper 使用 FA3，Blackwell FA4 fp8 descale 问题不适用，EAGLE3 draft 保持 attention_backend FLASH_ATTN。" - "This change does not alter the EAGLE3 draft model data type. The draft loads unmodified from the published Inferact/MiniMax-M3-EAGLE3-GQA checkpoint via --speculative-config (method=eagle3). kv-cache-dtype fp8 sets KV-cache storage precision, not the draft weights, and no flag overrides or re-quantizes the draft weights." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3503 + +- config-keys: + - kimik3-fp4-b300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg-1p1d + scenario-type: + - agentic-coding + description: + - "Add the Kimi-K3 B300 Mooncake EFA AgentX disaggregated 1P1D DCP8/DCP8 DSpark4 configuration at c48 on ghcr.io/semianalysisai/vllm-openai:efa_pr_58768 (vllm-project/vllm#58768 vllm-openai-efa target, with AWS EFA libfabric 2.6.0amzn1.0 and mooncake-transfer-engine-efa-cuda13 0.3.13.post1 built in). Taken from #3405's 1P1D arm; drops its kimik3-b300-efa-setup.sh runtime installer." + - "新增基于 ghcr.io/semianalysisai/vllm-openai:efa_pr_58768（vllm-project/vllm#58768 的 vllm-openai-efa 目标，内置 AWS EFA libfabric 2.6.0amzn1.0 与 mooncake-transfer-engine-efa-cuda13 0.3.13.post1）的 Kimi-K3 B300 Mooncake EFA AgentX 分离式 1P1D DCP8/DCP8 DSpark4 配置（c48）。取自 #3405 的 1P1D 配置；删除其 kimik3-b300-efa-setup.sh 运行时安装脚本。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3523 diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index efd0506131..f8fb76cae0 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -112,8 +112,18 @@ import_squash_image() { if unsquashfs -l \"$sqsh\" > /dev/null 2>&1; then exit 0 fi - rm -f \"$sqsh\" - enroot import -o \"$sqsh\" \"docker://$image_ref\" + # Large registry layers (e.g. GHCR) can drop mid-transfer with + # \"curl: (56) Connection reset by peer\". Retry with fewer parallel + # connections; enroot's layer cache keeps finished layers across attempts. + for attempt in 1 2 3 4; do + rm -f \"$sqsh\" + if ENROOT_MAX_CONNECTIONS=\"\${ENROOT_MAX_CONNECTIONS:-4}\" enroot import -o \"$sqsh\" \"docker://$image_ref\"; then + break + fi + [ \"\$attempt\" -eq 4 ] && exit 1 + echo \"enroot import attempt \$attempt failed for $image_ref; retrying in \$((attempt * 30))s\" >&2 + sleep \$((attempt * 30)) + done unsquashfs -l \"$sqsh\" > /dev/null " || { echo "Error: enroot import failed for $image_ref -> $sqsh" >&2; exit 1; }