From f7fbad8cb1c10c07d0558184889898ffc70108a6 Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Thu, 2 Jul 2026 19:34:04 -0700 Subject: [PATCH 01/24] Add --- .../single_node/agentic/kimik2.5_fp4_b300.sh | 111 +++++++++++++++--- configs/nvidia-master.yaml | 13 +- perf-changelog.yaml | 6 + 3 files changed, 103 insertions(+), 27 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index b99299b12..02f1cd586 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -7,7 +7,7 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=native. +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=native or KV_OFFLOAD_BACKEND=mooncake. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -37,39 +37,112 @@ install_agentic_deps # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" +MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" mkdir -p "$RESULT_DIR" +SERVER_PID="" +MOONCAKE_MASTER_PID="" + +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + OFFLOAD_ARGS=() -PREFIX_CACHE_ARGS=() - -if require_agentic_kv_offload_backend native; then - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - OFFLOAD_ARGS=( - --kv_offloading_backend native - --kv_offloading_size "$TOTAL_CPU_DRAM_GB" - --disable-hybrid-kv-cache-manager - ) + +if agentic_kv_offload_enabled; then + case "$KV_OFFLOAD_BACKEND" in + native) + export VLLM_USE_SIMPLE_KV_OFFLOAD=1 + OFFLOAD_ARGS=( + --kv_offloading_backend native + --kv_offloading_size "$TOTAL_CPU_DRAM_GB" + --disable-hybrid-kv-cache-manager + ) + ;; + mooncake) + { set +x; } 2>/dev/null + unset VLLM_USE_SIMPLE_KV_OFFLOAD + + PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) + + MOONCAKE_VERSION=0.3.11.post1 + agentic_pip_install --quiet --no-cache-dir --no-deps \ + --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" + + MOONCAKE_MASTER_PORT=$((PORT + 12000)) + MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" + cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & + MOONCAKE_MASTER_PID=$! + sleep 2 + if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then + echo "Mooncake master died during startup." >&2 + cat "$MOONCAKE_MASTER_LOG" >&2 + exit 1 + fi + OFFLOAD_ARGS=( + --kv-transfer-config + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both"}' + ) + ;; + *) echo "Error: unsupported KV_OFFLOAD_BACKEND value '$KV_OFFLOAD_BACKEND' (expected one of: native, mooncake)" >&2; exit 1 ;; + esac fi echo "Starting vllm server..." export PYTHONNOUSERSITE=1 +DCP_ARGS=() +if [[ "$CONC" -ge 16 ]]; then + DCP_ARGS=(--decode-context-parallel-size "$TP") +fi + { set +x; } 2>/dev/null VLLM_CMD=( vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port "$PORT" - --tensor-parallel-size="$TP" - --gpu-memory-utilization 0.90 - --max-num-seqs "$CONC" - --reasoning-parser kimi_k2 - --tool-call-parser kimi_k2 - --compilation_config.pass_config.fuse_allreduce_rms true --kv-cache-dtype fp8 - --max-cudagraph-capture-size 2048 - --stream-interval 20 --trust-remote-code - "${PREFIX_CACHE_ARGS[@]}" + --block-size 64 + --language-model-only + --attention-config '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + --max-cudagraph-capture-size 2048 + --max-num-batched-tokens 16384 + --stream-interval 10 + --enable-prefix-caching + --tensor-parallel-size "$TP" + "${DCP_ARGS[@]}" "${OFFLOAD_ARGS[@]}" ) printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 46154341c..c48e81bb2 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2965,12 +2965,7 @@ dsr1-fp8-b300-sglang-mtp: - { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp } kimik2.5-fp4-b300-vllm-agentic: - # v0.20.2 (cu129) lacks the flashinfer kernels for B300's reported SM - # (sm_12x); workers hit "Only SM 10.x and 11.x are supported" in the - # trtllm_fp4_block_scale_moe path. v0.20.0-cu130 is the Blackwell-targeted - # build that has the full sm_10x/sm_11x/sm_12x kernel set and is what the - # INT4 B300 sister already uses successfully. - image: vllm/vllm-openai:v0.22.0 + image: vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa model: nvidia/Kimi-K2.5-NVFP4 model-prefix: kimik2.5 runner: cluster:b300-nv @@ -2981,8 +2976,10 @@ kimik2.5-fp4-b300-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4] } + - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [64, 72, 80] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [64, 72, 80] } dsr1-fp8-b200-trt: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 943f4b390..6a7ecbd8f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4440,3 +4440,9 @@ - "Update Minimax M3 b200 vllm image tag" - "Update search space to cover more configs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1978 + +- config-keys: + - kimik2.5-fp4-b300-vllm-agentic + description: + - "Update kimi k2.5 agentx B300" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/ From 9a47359f035c462160b9efa52a7c67ccba568738 Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Thu, 2 Jul 2026 19:34:53 -0700 Subject: [PATCH 02/24] update --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6a7ecbd8f..3f823c520 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4445,4 +4445,4 @@ - kimik2.5-fp4-b300-vllm-agentic description: - "Update kimi k2.5 agentx B300" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/ + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1998 From 2826ba92c9a2eb21826d0ac1831313abf265b28b Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Sun, 5 Jul 2026 20:27:44 -0400 Subject: [PATCH 03/24] Update nvidia-master.yaml --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c48e81bb2..eb101166c 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2966,7 +2966,7 @@ dsr1-fp8-b300-sglang-mtp: kimik2.5-fp4-b300-vllm-agentic: image: vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa - model: nvidia/Kimi-K2.5-NVFP4 + model: Kimi-K2.6-NVFP4 model-prefix: kimik2.5 runner: cluster:b300-nv precision: fp4 From 45306822d6bb219dab7c9bb9144496c1707bc637 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 10:33:37 -0400 Subject: [PATCH 04/24] Update kimik2.5_fp4_b300.sh --- benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index 02f1cd586..c53c3dd02 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -126,6 +126,8 @@ if [[ "$CONC" -ge 16 ]]; then DCP_ARGS=(--decode-context-parallel-size "$TP") fi +export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm + { set +x; } 2>/dev/null VLLM_CMD=( vllm serve "$MODEL_PATH" --served-model-name "$MODEL" From 73b96a5319dbc1123d1ccb9dd871e03cffc3ed32 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 10:35:23 -0400 Subject: [PATCH 05/24] Update perf-changelog.yaml --- perf-changelog.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 69c6c0183..146f52084 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4532,12 +4532,6 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2077 - config-keys: -<<<<<<< wzhao/kimi-agentx-b300 - - kimik2.5-fp4-b300-vllm-agentic - description: - - "Update kimi k2.5 agentx B300" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1998 -======= - dsv4-fp4-gb300-dynamo-vllm description: - "Refresh DSV4 8k/1k vLLM GB300 recipes with new w4a4 container and updated configs" @@ -4562,4 +4556,9 @@ description: - "Add high concurrency configs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 ->>>>>>> main + +- config-keys: + - kimik2.5-fp4-b300-vllm-agentic + description: + - "Update kimi k2.5 agentx B300" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1998 From e1c16ef4d01495d87ea2b8feb8be51942d90a508 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:43:01 -0400 Subject: [PATCH 06/24] Update nvidia-master.yaml --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index fb52e7654..b90dc0e50 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2965,7 +2965,7 @@ dsr1-fp8-b300-sglang-mtp: - { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp } kimik2.5-fp4-b300-vllm-agentic: - image: vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa + image: vllm/vllm-openai:nightly-a16dbd5b8572d4128be9f10b9dcff4999b594b25 model: Kimi-K2.6-NVFP4 model-prefix: kimik2.5 runner: cluster:b300-nv From 56d6201c3cfe655e5541fc7cc8d0f819dcfdbc08 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:43:21 -0400 Subject: [PATCH 07/24] Update kimik2.5_fp4_b300.sh --- benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index c53c3dd02..02f1cd586 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -126,8 +126,6 @@ if [[ "$CONC" -ge 16 ]]; then DCP_ARGS=(--decode-context-parallel-size "$TP") fi -export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm - { set +x; } 2>/dev/null VLLM_CMD=( vllm serve "$MODEL_PATH" --served-model-name "$MODEL" From 65b16b2fbc9ef2d73165c274bd2a2472907e32f4 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:51:38 -0400 Subject: [PATCH 08/24] Update Docker image for kimik2.5-fp4-b300-vllm-agentic --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b90dc0e50..f6228145e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2965,7 +2965,7 @@ dsr1-fp8-b300-sglang-mtp: - { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp } kimik2.5-fp4-b300-vllm-agentic: - image: vllm/vllm-openai:nightly-a16dbd5b8572d4128be9f10b9dcff4999b594b25 + image: nightly-69715823df89b11ee684b84066390cbb9092d5c1 model: Kimi-K2.6-NVFP4 model-prefix: kimik2.5 runner: cluster:b300-nv From cc143d3ded0e0f5cf9deb63138a06086b797cc68 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:52:34 -0400 Subject: [PATCH 09/24] Configure VLLM backend for inference Set the VLLM_FLASHINFER_ALLREDUCE_BACKEND environment variable to 'trtllm'. --- benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index 02f1cd586..c53c3dd02 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -126,6 +126,8 @@ if [[ "$CONC" -ge 16 ]]; then DCP_ARGS=(--decode-context-parallel-size "$TP") fi +export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm + { set +x; } 2>/dev/null VLLM_CMD=( vllm serve "$MODEL_PATH" --served-model-name "$MODEL" From e98a4ed8f40b49b40d16e71af1a67726735b5205 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:55:51 -0400 Subject: [PATCH 10/24] Update Docker image for kimik2.5-fp4-b300-vllm-agentic --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f6228145e..979979238 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2965,7 +2965,7 @@ dsr1-fp8-b300-sglang-mtp: - { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp } kimik2.5-fp4-b300-vllm-agentic: - image: nightly-69715823df89b11ee684b84066390cbb9092d5c1 + image: vllm/vllm-openai:nightly-69715823df89b11ee684b84066390cbb9092d5c1 model: Kimi-K2.6-NVFP4 model-prefix: kimik2.5 runner: cluster:b300-nv From 3c59244f9833a8d784d5e60312406d044d11be1e Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:24:35 -0400 Subject: [PATCH 11/24] Update nvidia-master.yaml --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 979979238..fb52e7654 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2965,7 +2965,7 @@ dsr1-fp8-b300-sglang-mtp: - { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp } kimik2.5-fp4-b300-vllm-agentic: - image: vllm/vllm-openai:nightly-69715823df89b11ee684b84066390cbb9092d5c1 + image: vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa model: Kimi-K2.6-NVFP4 model-prefix: kimik2.5 runner: cluster:b300-nv From 4f8f6c1ab767eaa7fc3cf63ff73905a60815e88e Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:32:43 -0400 Subject: [PATCH 12/24] Remove eviction configuration from kimik2.5_fp4_b300.sh Removed eviction configuration variables from the script. --- benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index c53c3dd02..98834d225 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -94,10 +94,6 @@ EOF export WITH_NVIDIA_PEERMEM=0 export MC_ENABLE_DEST_DEVICE_AFFINITY=1 - MOONCAKE_EVICTION_HIGH_WATERMARK_RATIO=0.80 - MOONCAKE_EVICTION_RATIO=0.10 - MOONCAKE_KV_LEASE_TTL=60s - echo "Starting Mooncake master on port $MOONCAKE_MASTER_PORT..." mooncake_master --port "$MOONCAKE_MASTER_PORT" \ --default_kv_lease_ttl=1h \ From f481562b525e0151e5426263615498c94e8bfdb7 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 23:51:37 -0400 Subject: [PATCH 13/24] Update nvidia-master.yaml --- configs/nvidia-master.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 93f94fdae..b3b86068d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2970,10 +2970,7 @@ kimik2.5-fp4-b300-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4] } - - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [64, 72, 80] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [64, 72, 80] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [1] } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 From 978ff604fc017dccaf6a62b3774496a2219cd346 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Tue, 7 Jul 2026 00:54:44 -0400 Subject: [PATCH 14/24] Update nvidia-master.yaml --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b3b86068d..8bcaa96b1 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2970,7 +2970,7 @@ kimik2.5-fp4-b300-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [1] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [1, 16, 32, 64] } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 From 279e0d46c995649993b2eae67b28d66b57b035dd Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Tue, 7 Jul 2026 01:37:11 -0400 Subject: [PATCH 15/24] Change protocol from 'rdma' to 'tcp' --- benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index 98834d225..57e30177a 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -86,7 +86,7 @@ if agentic_kv_offload_enabled; then "master_server_address": "127.0.0.1:$MOONCAKE_MASTER_PORT", "global_segment_size": "${PER_RANK_GB}GB", "local_buffer_size": "4GB", - "protocol": "rdma", + "protocol": "tcp", "device_name": "" } EOF From 729b1574970e999d734559edca175efb110b1424 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:15:14 -0400 Subject: [PATCH 16/24] Update kimik2.5_fp4_b300.sh --- benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index 57e30177a..baa41339a 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -84,9 +84,9 @@ if agentic_kv_offload_enabled; then "mode": "embedded", "metadata_server": "P2PHANDSHAKE", "master_server_address": "127.0.0.1:$MOONCAKE_MASTER_PORT", - "global_segment_size": "${PER_RANK_GB}GB", + "global_segment_size": "200GB", "local_buffer_size": "4GB", - "protocol": "tcp", + "protocol": "rdma", "device_name": "" } EOF From e9c0d92fc8e6ef405f574f3e6d76bb7c085b0699 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:18:44 -0400 Subject: [PATCH 17/24] Set device_name to 'mlx5_0' in configuration --- benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index baa41339a..79ba360a6 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -87,7 +87,7 @@ if agentic_kv_offload_enabled; then "global_segment_size": "200GB", "local_buffer_size": "4GB", "protocol": "rdma", - "device_name": "" + "device_name": "mlx5_0" } EOF export MOONCAKE_CONFIG_PATH From 6910ed76306aaad1930c1d984c3174053e9590a9 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:39:18 -0400 Subject: [PATCH 18/24] Update kimik2.5_fp4_b300.sh --- benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index 79ba360a6..44bd80c41 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -84,7 +84,7 @@ if agentic_kv_offload_enabled; then "mode": "embedded", "metadata_server": "P2PHANDSHAKE", "master_server_address": "127.0.0.1:$MOONCAKE_MASTER_PORT", - "global_segment_size": "200GB", + "global_segment_size": "${PER_RANK_GB}GB", "local_buffer_size": "4GB", "protocol": "rdma", "device_name": "mlx5_0" From 72a6a96b51f9daae6345ed149abd6181d4d1a904 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:40:36 -0400 Subject: [PATCH 19/24] Modify search space for agentic-coding scenario Updated the search space configuration for agentic-coding scenario in nvidia-master.yaml. --- configs/nvidia-master.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8bcaa96b1..094a4ebe7 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2970,7 +2970,10 @@ kimik2.5-fp4-b300-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [1, 16, 32, 64] } + # - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4] } + # - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80] } + # - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [64, 72, 80] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80] } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 From 77df3aa0d45a82342774afedc3a1eabd2d27a4fc Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Tue, 7 Jul 2026 08:45:49 -0700 Subject: [PATCH 20/24] auto-filter --- .../single_node/agentic/kimik2.5_fp4_b300.sh | 14 +- .../vllm-mooncake-rdma-device-filter.patch | 334 ++++++++++++++++++ 2 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/agentic/patches/vllm-mooncake-rdma-device-filter.patch diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index 44bd80c41..ada5a781e 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -77,6 +77,18 @@ if agentic_kv_offload_enabled; then agentic_pip_install --quiet --no-cache-dir --no-deps \ --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" + # Filter Mooncake RDMA auto-discovery to routable RoCE NICs; B300 + # hosts expose IB-mode fabric ports and IP-less NICs that crash the + # server when left in the device pool. With the filter applied, + # device_name stays empty so auto-discovery selects the devices. + VLLM_RDMA_PATCH="$(dirname "$0")/patches/vllm-mooncake-rdma-device-filter.patch" + VLLM_SITE_DIR="$(python3 -c 'import pathlib, vllm; print(pathlib.Path(vllm.__file__).resolve().parent.parent)')" + if git -C "$VLLM_SITE_DIR" apply --reverse --check "$VLLM_RDMA_PATCH" >/dev/null 2>&1; then + echo "vLLM RDMA device-filter patch already applied." + else + git -C "$VLLM_SITE_DIR" apply --verbose "$VLLM_RDMA_PATCH" + fi + MOONCAKE_MASTER_PORT=$((PORT + 12000)) MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" cat > "$MOONCAKE_CONFIG_PATH" < str | None: + if not isinstance(value, str): +@@ -44,3 +52,283 @@ def get_configured_preferred_segment( + ) + return env_value + return None ++ ++ ++def _normalize_explicit_worker_rnics(device_list: str) -> str: ++ entries = [entry.strip() for entry in device_list.split(",")] ++ if any(not entry for entry in entries): ++ raise ValueError( ++ "Mooncake worker device_name contains an empty RDMA device entry" ++ ) ++ return ",".join(entries) ++ ++ ++def _read_sysfs_text(path: Path) -> str | None: ++ try: ++ return path.read_text().strip() ++ except OSError: ++ return None ++ ++ ++def _get_mooncake_pkey_index() -> int: ++ raw_index = os.environ.get("MC_PKEY_INDEX") ++ if raw_index is None: ++ return _DEFAULT_MOONCAKE_PKEY_INDEX ++ try: ++ pkey_index = int(raw_index) ++ except ValueError: ++ return _DEFAULT_MOONCAKE_PKEY_INDEX ++ if not 0 <= pkey_index <= 65535: ++ return _DEFAULT_MOONCAKE_PKEY_INDEX ++ return pkey_index ++ ++ ++def _has_compatible_pkey(port_path: Path, link_layer: str) -> bool: ++ if link_layer != "InfiniBand": ++ return True ++ ++ pkey = _read_sysfs_text(port_path / "pkeys" / str(_get_mooncake_pkey_index())) ++ if pkey is None: ++ return False ++ try: ++ pkey_value = int(pkey, 16) ++ except ValueError: ++ return False ++ ++ # 0x7FFF: check bits 0-14 for valid partition number ++ # 0x8000: check bit 15 for membership type (full vs limited) ++ # Limited P_Keys can fail peer-to-peer transfers ++ return bool(pkey_value & 0x8000) and bool(pkey_value & 0x7FFF) ++ ++ ++def _is_compatible_port(port_path: Path) -> bool: ++ link_layer = _read_sysfs_text(port_path / "link_layer") ++ state = _read_sysfs_text(port_path / "state") ++ ++ if link_layer not in {"InfiniBand", "Ethernet"}: ++ return False ++ ++ if state != "4: ACTIVE": ++ return False ++ return _has_compatible_pkey(port_path, link_layer) ++ ++ ++def _is_compatible_rdma_device(device_path: Path) -> bool: ++ # node_type must be CA (Channel Adapter) ++ if _read_sysfs_text(device_path / "node_type") != "1: CA": ++ return False ++ ++ # Inspect every port: a device is usable if any of its ports is ACTIVE with ++ # a compatible link layer and P_Key. ++ try: ++ port_paths = sorted( ++ path for path in (device_path / "ports").iterdir() if path.is_dir() ++ ) ++ except OSError: ++ return False ++ ++ return any(_is_compatible_port(port_path) for port_path in port_paths) ++ ++ ++def _ipv4_from_gid(gid: str) -> ipaddress.IPv4Address | None: ++ """Extract IPv4 from an IPv4-mapped RoCE GID.""" ++ try: ++ ip = ipaddress.IPv6Address(gid) ++ except ValueError: ++ return None ++ return ip.ipv4_mapped ++ ++ ++def _first_netdev_name(device_path: Path) -> str | None: ++ """Return the first Linux netdev backing an RDMA device.""" ++ try: ++ netdevs = sorted(path.name for path in (device_path / "device/net").iterdir()) ++ except OSError: ++ return None ++ return netdevs[0] if netdevs else None ++ ++ ++def _first_roce_v2_ipv4_gid(device_path: Path) -> ipaddress.IPv4Address | None: ++ """Return the first non-link-local RoCE v2 IPv4 GID for an RDMA device.""" ++ try: ++ port_paths = sorted( ++ path for path in (device_path / "ports").iterdir() if path.is_dir() ++ ) ++ except OSError: ++ return None ++ ++ for port_path in port_paths: ++ try: ++ gid_paths = sorted( ++ path for path in (port_path / "gids").iterdir() if path.name.isdigit() ++ ) ++ except OSError: ++ continue ++ for gid_path in gid_paths: ++ gid_type = _read_sysfs_text( ++ port_path / "gid_attrs" / "types" / gid_path.name ++ ) ++ if gid_type != "RoCE v2": ++ continue ++ gid_ip = _ipv4_from_gid(_read_sysfs_text(gid_path) or "") ++ if gid_ip is not None and not gid_ip.is_link_local: ++ return gid_ip ++ return None ++ ++ ++def _ipv4_from_proc_route_hex(value: str) -> ipaddress.IPv4Address | None: ++ """Decode /proc/net/route IPv4 hex, e.g. 000010AC -> 172.16.0.0.""" ++ try: ++ return ipaddress.IPv4Address(int.from_bytes(bytes.fromhex(value), "little")) ++ except ValueError: ++ return None ++ ++ ++def _read_ipv4_routes() -> dict[str, list[ipaddress.IPv4Network]]: ++ """Read non-default IPv4 routes from /proc/net/route grouped by netdev.""" ++ try: ++ lines = _PROC_NET_ROUTE.read_text().splitlines() ++ except OSError: ++ return {} ++ ++ # Map each Linux netdev to its non-default IPv4 routes, e.g. ++ # {"spx_p1n1": [IPv4Network("172.16.0.0/13")]}. ++ routes: dict[str, list[ipaddress.IPv4Network]] = defaultdict(list) ++ for line in lines[1:]: ++ fields = line.split() ++ if len(fields) < 8: ++ continue ++ iface = fields[0] ++ dest_hex = fields[1] ++ flags_hex = fields[3] ++ mask_hex = fields[7] ++ try: ++ flags = int(flags_hex, 16) ++ except ValueError: ++ continue ++ # ensure the route is active ++ if not flags & 0x1: ++ continue ++ dest = _ipv4_from_proc_route_hex(dest_hex) ++ mask = _ipv4_from_proc_route_hex(mask_hex) ++ if dest is None or mask is None: ++ continue ++ try: ++ network = ipaddress.IPv4Network(f"{dest}/{mask}", strict=False) ++ except ValueError: ++ continue ++ if network.prefixlen == 0: ++ continue ++ routes[iface].append(network) ++ return routes ++ ++ ++def _route_group_for_device( ++ device_path: Path, ++ routes_by_netdev: Mapping[str, list[ipaddress.IPv4Network]], ++) -> str | None: ++ """Return the broadest route containing the device's RoCE v2 GID IP.""" ++ netdev = _first_netdev_name(device_path) ++ gid_ip = _first_roce_v2_ipv4_gid(device_path) ++ if netdev is None or gid_ip is None: ++ return None ++ ++ candidates = [ ++ route for route in routes_by_netdev.get(netdev, []) if gid_ip in route ++ ] ++ if not candidates: ++ return None ++ ++ # The least-specific non-default route captures the routed rail family. On ++ # split-rail RoCE hosts, direct host/link routes differ per NIC while the ++ # broad rail route is shared by mutually reachable NICs. ++ return str(min(candidates, key=lambda route: route.prefixlen)) ++ ++ ++def _filter_to_largest_routable_rnic_group( ++ device_names: list[str], ++ route_groups: Mapping[str, str], ++) -> list[str]: ++ """Choose the largest routable RNIC group for RDMA device auto discovery.""" ++ if not device_names: ++ return [] ++ ++ # Group devices by routed rail. ++ by_route_group: dict[str, list[str]] = defaultdict(list) ++ for name in device_names: ++ route_group = route_groups.get(name) ++ if route_group is not None: ++ by_route_group[route_group].append(name) ++ if not by_route_group: ++ return sorted(device_names) ++ ++ # Pick the largest routable rail group, with a deterministic tie-break. ++ selected = min( ++ by_route_group.values(), ++ key=lambda names: (-len(names), sorted(names)[0]), ++ ) ++ return sorted(selected) ++ ++ ++def _get_auto_discovered_worker_rnics() -> str: ++ try: ++ device_paths = sorted( ++ path ++ for path in _SYSFS_INFINIBAND_ROOT.iterdir() ++ if path.is_dir() or path.is_symlink() ++ ) ++ except OSError: ++ return "" ++ ++ compatible = [path for path in device_paths if _is_compatible_rdma_device(path)] ++ device_names = [path.name for path in compatible] ++ routes_by_netdev = _read_ipv4_routes() ++ route_groups: dict[str, str] = {} ++ for path in compatible: ++ route_group = _route_group_for_device(path, routes_by_netdev) ++ if route_group is not None: ++ route_groups[path.name] = route_group ++ ++ selected = _filter_to_largest_routable_rnic_group(device_names, route_groups) ++ return ",".join(selected) ++ ++ ++def get_configured_worker_rnic( ++ *, ++ protocol: str, ++ configured_device: str, ++) -> str: ++ normalized_device = normalize_string_override(configured_device) ++ if normalized_device is not None: ++ return _normalize_explicit_worker_rnics(normalized_device) ++ ++ if protocol not in {"rdma", "efa"}: ++ return "" ++ ++ if protocol == "rdma": ++ logger.info( ++ "No RDMA devices specified for Mooncake backend (protocol=%s). " ++ "Running automatic device selection.", ++ protocol, ++ ) ++ discovered_devices = _get_auto_discovered_worker_rnics() ++ if discovered_devices: ++ logger.info( ++ "Mooncake auto-selected compatible RDMA devices for protocol=%s: %s", ++ protocol, ++ discovered_devices, ++ ) ++ return discovered_devices ++ ++ logger.warning( ++ "No compatible RDMA devices were discovered in /sys/class/infiniband. " ++ "Falling back to Mooncake's built-in auto-selection.", ++ ) ++ return "" ++ ++ logger.warning( ++ "No devices specified for Mooncake backend (protocol=%s); falling back " ++ "to Mooncake's built-in auto-selection.", ++ protocol, ++ ) ++ return "" +diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +index 18cae18ee9..0e4053c0da 100644 +--- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py ++++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +@@ -989,6 +989,10 @@ class MooncakeStoreWorker: + if vllm_config.kv_transfer_config + else {} + ) ++ store_config.device_name = rdma_utils.get_configured_worker_rnic( ++ protocol=store_config.protocol, ++ configured_device=store_config.device_name, ++ ) + self.store = MooncakeDistributedStore() + local_ip = get_ip() + local_hostname = rdma_utils.get_requester_local_hostname(local_ip) From 9c86925aad02a1dade45b92b5f06be14549b6b84 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:53:39 -0400 Subject: [PATCH 21/24] Update perf-changelog.yaml --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1c45566c9..fcbdaff61 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4582,7 +4582,7 @@ - "Speeds up the draft forward measured on MI355X TP4 MXFP8, 8k/1k throughput gains growing with concurrency vs no override." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2107 - - config-keys: +- config-keys: - kimik2.5-fp4-b300-vllm-agentic description: - "Update kimi k2.5 agentx B300" From 09230f190f29e9f4baf520621646aac4ee470f86 Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Tue, 7 Jul 2026 09:05:27 -0700 Subject: [PATCH 22/24] fix --- benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index ada5a781e..54395ab37 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -81,7 +81,7 @@ if agentic_kv_offload_enabled; then # hosts expose IB-mode fabric ports and IP-less NICs that crash the # server when left in the device pool. With the filter applied, # device_name stays empty so auto-discovery selects the devices. - VLLM_RDMA_PATCH="$(dirname "$0")/patches/vllm-mooncake-rdma-device-filter.patch" + VLLM_RDMA_PATCH="$(cd "$(dirname "$0")" && pwd)/patches/vllm-mooncake-rdma-device-filter.patch" VLLM_SITE_DIR="$(python3 -c 'import pathlib, vllm; print(pathlib.Path(vllm.__file__).resolve().parent.parent)')" if git -C "$VLLM_SITE_DIR" apply --reverse --check "$VLLM_RDMA_PATCH" >/dev/null 2>&1; then echo "vLLM RDMA device-filter patch already applied." From 19ed8dcf5160062cd5f94e65d834306eda63270b Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Tue, 7 Jul 2026 10:24:48 -0700 Subject: [PATCH 23/24] update --- .../single_node/agentic/kimik2.5_fp4_b300.sh | 13 - .../vllm-mooncake-rdma-device-filter.patch | 334 ------------------ configs/nvidia-master.yaml | 8 +- 3 files changed, 4 insertions(+), 351 deletions(-) delete mode 100644 benchmarks/single_node/agentic/patches/vllm-mooncake-rdma-device-filter.patch diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index 54395ab37..5f24421b3 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -64,7 +64,6 @@ if agentic_kv_offload_enabled; then OFFLOAD_ARGS=( --kv_offloading_backend native --kv_offloading_size "$TOTAL_CPU_DRAM_GB" - --disable-hybrid-kv-cache-manager ) ;; mooncake) @@ -77,18 +76,6 @@ if agentic_kv_offload_enabled; then agentic_pip_install --quiet --no-cache-dir --no-deps \ --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" - # Filter Mooncake RDMA auto-discovery to routable RoCE NICs; B300 - # hosts expose IB-mode fabric ports and IP-less NICs that crash the - # server when left in the device pool. With the filter applied, - # device_name stays empty so auto-discovery selects the devices. - VLLM_RDMA_PATCH="$(cd "$(dirname "$0")" && pwd)/patches/vllm-mooncake-rdma-device-filter.patch" - VLLM_SITE_DIR="$(python3 -c 'import pathlib, vllm; print(pathlib.Path(vllm.__file__).resolve().parent.parent)')" - if git -C "$VLLM_SITE_DIR" apply --reverse --check "$VLLM_RDMA_PATCH" >/dev/null 2>&1; then - echo "vLLM RDMA device-filter patch already applied." - else - git -C "$VLLM_SITE_DIR" apply --verbose "$VLLM_RDMA_PATCH" - fi - MOONCAKE_MASTER_PORT=$((PORT + 12000)) MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" cat > "$MOONCAKE_CONFIG_PATH" < str | None: - if not isinstance(value, str): -@@ -44,3 +52,283 @@ def get_configured_preferred_segment( - ) - return env_value - return None -+ -+ -+def _normalize_explicit_worker_rnics(device_list: str) -> str: -+ entries = [entry.strip() for entry in device_list.split(",")] -+ if any(not entry for entry in entries): -+ raise ValueError( -+ "Mooncake worker device_name contains an empty RDMA device entry" -+ ) -+ return ",".join(entries) -+ -+ -+def _read_sysfs_text(path: Path) -> str | None: -+ try: -+ return path.read_text().strip() -+ except OSError: -+ return None -+ -+ -+def _get_mooncake_pkey_index() -> int: -+ raw_index = os.environ.get("MC_PKEY_INDEX") -+ if raw_index is None: -+ return _DEFAULT_MOONCAKE_PKEY_INDEX -+ try: -+ pkey_index = int(raw_index) -+ except ValueError: -+ return _DEFAULT_MOONCAKE_PKEY_INDEX -+ if not 0 <= pkey_index <= 65535: -+ return _DEFAULT_MOONCAKE_PKEY_INDEX -+ return pkey_index -+ -+ -+def _has_compatible_pkey(port_path: Path, link_layer: str) -> bool: -+ if link_layer != "InfiniBand": -+ return True -+ -+ pkey = _read_sysfs_text(port_path / "pkeys" / str(_get_mooncake_pkey_index())) -+ if pkey is None: -+ return False -+ try: -+ pkey_value = int(pkey, 16) -+ except ValueError: -+ return False -+ -+ # 0x7FFF: check bits 0-14 for valid partition number -+ # 0x8000: check bit 15 for membership type (full vs limited) -+ # Limited P_Keys can fail peer-to-peer transfers -+ return bool(pkey_value & 0x8000) and bool(pkey_value & 0x7FFF) -+ -+ -+def _is_compatible_port(port_path: Path) -> bool: -+ link_layer = _read_sysfs_text(port_path / "link_layer") -+ state = _read_sysfs_text(port_path / "state") -+ -+ if link_layer not in {"InfiniBand", "Ethernet"}: -+ return False -+ -+ if state != "4: ACTIVE": -+ return False -+ return _has_compatible_pkey(port_path, link_layer) -+ -+ -+def _is_compatible_rdma_device(device_path: Path) -> bool: -+ # node_type must be CA (Channel Adapter) -+ if _read_sysfs_text(device_path / "node_type") != "1: CA": -+ return False -+ -+ # Inspect every port: a device is usable if any of its ports is ACTIVE with -+ # a compatible link layer and P_Key. -+ try: -+ port_paths = sorted( -+ path for path in (device_path / "ports").iterdir() if path.is_dir() -+ ) -+ except OSError: -+ return False -+ -+ return any(_is_compatible_port(port_path) for port_path in port_paths) -+ -+ -+def _ipv4_from_gid(gid: str) -> ipaddress.IPv4Address | None: -+ """Extract IPv4 from an IPv4-mapped RoCE GID.""" -+ try: -+ ip = ipaddress.IPv6Address(gid) -+ except ValueError: -+ return None -+ return ip.ipv4_mapped -+ -+ -+def _first_netdev_name(device_path: Path) -> str | None: -+ """Return the first Linux netdev backing an RDMA device.""" -+ try: -+ netdevs = sorted(path.name for path in (device_path / "device/net").iterdir()) -+ except OSError: -+ return None -+ return netdevs[0] if netdevs else None -+ -+ -+def _first_roce_v2_ipv4_gid(device_path: Path) -> ipaddress.IPv4Address | None: -+ """Return the first non-link-local RoCE v2 IPv4 GID for an RDMA device.""" -+ try: -+ port_paths = sorted( -+ path for path in (device_path / "ports").iterdir() if path.is_dir() -+ ) -+ except OSError: -+ return None -+ -+ for port_path in port_paths: -+ try: -+ gid_paths = sorted( -+ path for path in (port_path / "gids").iterdir() if path.name.isdigit() -+ ) -+ except OSError: -+ continue -+ for gid_path in gid_paths: -+ gid_type = _read_sysfs_text( -+ port_path / "gid_attrs" / "types" / gid_path.name -+ ) -+ if gid_type != "RoCE v2": -+ continue -+ gid_ip = _ipv4_from_gid(_read_sysfs_text(gid_path) or "") -+ if gid_ip is not None and not gid_ip.is_link_local: -+ return gid_ip -+ return None -+ -+ -+def _ipv4_from_proc_route_hex(value: str) -> ipaddress.IPv4Address | None: -+ """Decode /proc/net/route IPv4 hex, e.g. 000010AC -> 172.16.0.0.""" -+ try: -+ return ipaddress.IPv4Address(int.from_bytes(bytes.fromhex(value), "little")) -+ except ValueError: -+ return None -+ -+ -+def _read_ipv4_routes() -> dict[str, list[ipaddress.IPv4Network]]: -+ """Read non-default IPv4 routes from /proc/net/route grouped by netdev.""" -+ try: -+ lines = _PROC_NET_ROUTE.read_text().splitlines() -+ except OSError: -+ return {} -+ -+ # Map each Linux netdev to its non-default IPv4 routes, e.g. -+ # {"spx_p1n1": [IPv4Network("172.16.0.0/13")]}. -+ routes: dict[str, list[ipaddress.IPv4Network]] = defaultdict(list) -+ for line in lines[1:]: -+ fields = line.split() -+ if len(fields) < 8: -+ continue -+ iface = fields[0] -+ dest_hex = fields[1] -+ flags_hex = fields[3] -+ mask_hex = fields[7] -+ try: -+ flags = int(flags_hex, 16) -+ except ValueError: -+ continue -+ # ensure the route is active -+ if not flags & 0x1: -+ continue -+ dest = _ipv4_from_proc_route_hex(dest_hex) -+ mask = _ipv4_from_proc_route_hex(mask_hex) -+ if dest is None or mask is None: -+ continue -+ try: -+ network = ipaddress.IPv4Network(f"{dest}/{mask}", strict=False) -+ except ValueError: -+ continue -+ if network.prefixlen == 0: -+ continue -+ routes[iface].append(network) -+ return routes -+ -+ -+def _route_group_for_device( -+ device_path: Path, -+ routes_by_netdev: Mapping[str, list[ipaddress.IPv4Network]], -+) -> str | None: -+ """Return the broadest route containing the device's RoCE v2 GID IP.""" -+ netdev = _first_netdev_name(device_path) -+ gid_ip = _first_roce_v2_ipv4_gid(device_path) -+ if netdev is None or gid_ip is None: -+ return None -+ -+ candidates = [ -+ route for route in routes_by_netdev.get(netdev, []) if gid_ip in route -+ ] -+ if not candidates: -+ return None -+ -+ # The least-specific non-default route captures the routed rail family. On -+ # split-rail RoCE hosts, direct host/link routes differ per NIC while the -+ # broad rail route is shared by mutually reachable NICs. -+ return str(min(candidates, key=lambda route: route.prefixlen)) -+ -+ -+def _filter_to_largest_routable_rnic_group( -+ device_names: list[str], -+ route_groups: Mapping[str, str], -+) -> list[str]: -+ """Choose the largest routable RNIC group for RDMA device auto discovery.""" -+ if not device_names: -+ return [] -+ -+ # Group devices by routed rail. -+ by_route_group: dict[str, list[str]] = defaultdict(list) -+ for name in device_names: -+ route_group = route_groups.get(name) -+ if route_group is not None: -+ by_route_group[route_group].append(name) -+ if not by_route_group: -+ return sorted(device_names) -+ -+ # Pick the largest routable rail group, with a deterministic tie-break. -+ selected = min( -+ by_route_group.values(), -+ key=lambda names: (-len(names), sorted(names)[0]), -+ ) -+ return sorted(selected) -+ -+ -+def _get_auto_discovered_worker_rnics() -> str: -+ try: -+ device_paths = sorted( -+ path -+ for path in _SYSFS_INFINIBAND_ROOT.iterdir() -+ if path.is_dir() or path.is_symlink() -+ ) -+ except OSError: -+ return "" -+ -+ compatible = [path for path in device_paths if _is_compatible_rdma_device(path)] -+ device_names = [path.name for path in compatible] -+ routes_by_netdev = _read_ipv4_routes() -+ route_groups: dict[str, str] = {} -+ for path in compatible: -+ route_group = _route_group_for_device(path, routes_by_netdev) -+ if route_group is not None: -+ route_groups[path.name] = route_group -+ -+ selected = _filter_to_largest_routable_rnic_group(device_names, route_groups) -+ return ",".join(selected) -+ -+ -+def get_configured_worker_rnic( -+ *, -+ protocol: str, -+ configured_device: str, -+) -> str: -+ normalized_device = normalize_string_override(configured_device) -+ if normalized_device is not None: -+ return _normalize_explicit_worker_rnics(normalized_device) -+ -+ if protocol not in {"rdma", "efa"}: -+ return "" -+ -+ if protocol == "rdma": -+ logger.info( -+ "No RDMA devices specified for Mooncake backend (protocol=%s). " -+ "Running automatic device selection.", -+ protocol, -+ ) -+ discovered_devices = _get_auto_discovered_worker_rnics() -+ if discovered_devices: -+ logger.info( -+ "Mooncake auto-selected compatible RDMA devices for protocol=%s: %s", -+ protocol, -+ discovered_devices, -+ ) -+ return discovered_devices -+ -+ logger.warning( -+ "No compatible RDMA devices were discovered in /sys/class/infiniband. " -+ "Falling back to Mooncake's built-in auto-selection.", -+ ) -+ return "" -+ -+ logger.warning( -+ "No devices specified for Mooncake backend (protocol=%s); falling back " -+ "to Mooncake's built-in auto-selection.", -+ protocol, -+ ) -+ return "" -diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py -index 18cae18ee9..0e4053c0da 100644 ---- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py -+++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py -@@ -989,6 +989,10 @@ class MooncakeStoreWorker: - if vllm_config.kv_transfer_config - else {} - ) -+ store_config.device_name = rdma_utils.get_configured_worker_rnic( -+ protocol=store_config.protocol, -+ configured_device=store_config.device_name, -+ ) - self.store = MooncakeDistributedStore() - local_ip = get_ip() - local_hostname = rdma_utils.get_requester_local_hostname(local_ip) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 094a4ebe7..93f94fdae 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -2970,10 +2970,10 @@ kimik2.5-fp4-b300-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - # - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4] } - # - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80] } - # - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [64, 72, 80] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80] } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4] } + - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [64, 72, 80] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [64, 72, 80] } dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 From 46258df94abd935b896086751f3676c2848a838f Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Tue, 7 Jul 2026 10:25:28 -0700 Subject: [PATCH 24/24] update --- benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh index 5f24421b3..9cd13bd09 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh @@ -86,7 +86,7 @@ if agentic_kv_offload_enabled; then "global_segment_size": "${PER_RANK_GB}GB", "local_buffer_size": "4GB", "protocol": "rdma", - "device_name": "" + "device_name": "mlx5_0,mlx5_2,mlx5_4,mlx5_8" } EOF export MOONCAKE_CONFIG_PATH