From 551d81e9509b60c3e943f6af44a889b792be6a78 Mon Sep 17 00:00:00 2001 From: Kimbo Chen Date: Mon, 15 Sep 2025 18:32:27 +0000 Subject: [PATCH 1/2] Added gpt-oss for H200 TRT. --- .github/workflows/gptoss-tmpl.yml | 17 +++++ .github/workflows/runner-sweep-test.yml | 1 + .github/workflows/runner-test.yml | 1 + benchmarks/gptoss_fp4_h200_trt_slurm.sh | 85 ++++++++++++++++++++++++- 4 files changed, 103 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gptoss-tmpl.yml b/.github/workflows/gptoss-tmpl.yml index b7128dc54d..c65dcd7547 100644 --- a/.github/workflows/gptoss-tmpl.yml +++ b/.github/workflows/gptoss-tmpl.yml @@ -39,6 +39,23 @@ on: required: true jobs: + bmk-h200-trt: + if: ${{ inputs.use_h200 }} + uses: ./.github/workflows/benchmark-tmpl.yml + secrets: inherit + with: + exp-name: ${{ inputs.exp-name }} + isl: ${{ inputs.isl }} + osl: ${{ inputs.osl }} + max-model-len: ${{ inputs.max-model-len }} + random-range-ratio: ${{ inputs.random-range-ratio }} + runner: h200-trt + image: 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post1' + model: 'openai/gpt-oss-120b' + tp-list: '[1, 2, 4, 8]' + framework: 'trt' + precision: 'fp4' + bmk-mi300x: if: ${{ inputs.use_mi300x }} uses: ./.github/workflows/benchmark-tmpl.yml diff --git a/.github/workflows/runner-sweep-test.yml b/.github/workflows/runner-sweep-test.yml index 3ad159c852..aa1054ebd0 100644 --- a/.github/workflows/runner-sweep-test.yml +++ b/.github/workflows/runner-sweep-test.yml @@ -31,6 +31,7 @@ on: - 'lmsysorg/sglang:v0.5.0rc1-cu128-b200' - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_sgl-dev-v0.5.2rc2-mi30x_rc1' - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2' + - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post1' model: description: 'Model' diff --git a/.github/workflows/runner-test.yml b/.github/workflows/runner-test.yml index 7f4e31343a..fbe73d2461 100644 --- a/.github/workflows/runner-test.yml +++ b/.github/workflows/runner-test.yml @@ -53,6 +53,7 @@ on: - 'lmsysorg/sglang:v0.5.0rc1-cu128-b200' - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_sgl-dev-v0.5.2rc2-mi30x_rc1' - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2' + - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post1' model: description: 'Model' diff --git a/benchmarks/gptoss_fp4_h200_trt_slurm.sh b/benchmarks/gptoss_fp4_h200_trt_slurm.sh index 4c6b9e9f57..ecc94f3cb9 100644 --- a/benchmarks/gptoss_fp4_h200_trt_slurm.sh +++ b/benchmarks/gptoss_fp4_h200_trt_slurm.sh @@ -1 +1,84 @@ -echo "dummy script" +#!/usr/bin/env bash + +# === Required Env Vars === +# HF_TOKEN +# HF_HUB_CACHE +# IMAGE +# MODEL +# ISL +# OSL +# MAX_MODEL_LEN +# RANDOM_RANGE_RATIO +# TP +# CONC +# RESULT_FILENAME +# PORT_OFFSET + +echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" + +hf download $MODEL +SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log) +PORT=$(( 8888 + $PORT_OFFSET )) + + +set +x + +git clone https://github.com/triton-lang/triton.git +cd triton +# Specific commit verified with TensorRT-LLM +git checkout f3067cd3bd0c29065fa4ecdb724b6f29cbabea5f +pip install -r python/requirements.txt # build-time dependencies +pip install wheel build +python3 setup.py bdist_wheel +pip install ./dist/*.whl +export TRITON_ROOT=/workspace/triton +export ENABLE_PDL=1 + +set -x +cat > gptoss-config.yml << EOF +cuda_graph_config: + enable_padding: true + max_batch_size: $CONC +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +EOF + + +#mpirun -n 1 --oversubscribe --allow-run-as-root trtllm-serve $MODEL --tp_size $TP --trust_remote_code --max_seq_len $MAX_MODEL_LEN --max_num_tokens $MAX_MODEL_LEN --num_postprocess_workers 2 --extra_llm_api_options llama-config.yml --port $PORT > $SERVER_LOG 2>&1 & +mpirun -n 1 --oversubscribe --allow-run-as-root trtllm-serve $MODEL --max_batch_size $CONC --max_num_tokens 20000 --backend pytorch --extra_llm_api_options gptoss-config.yml --ep_size 1 --trust_remote_code --gpus_per_node 8 --host 0.0.0.0 --port $PORT --tp_size=$TP --pp_size=1 > $SERVER_LOG 2>&1 & + + +set +x +while IFS= read -r line; do + printf '%s\n' "$line" + if [[ "$line" =~ [Ee][Rr][Rr][Oo][Rr] ]]; then + sleep 5 + tail -n100 $SERVER_LOG + echo "JOB $SLURM_JOB_ID ran on NODE $SLURMD_NODENAME" + exit 1 + fi + if [[ "$line" == *"Application startup complete"* ]]; then + break + fi +done < <(tail -F -n0 "$SERVER_LOG") + +set -x +git clone https://github.com/kimbochen/bench_serving.git +python3 bench_serving/benchmark_serving.py \ +--model $MODEL --backend openai \ +--base-url http://0.0.0.0:$PORT \ +--dataset-name random \ +--random-input-len $ISL --random-output-len $OSL --random-range-ratio $RANDOM_RANGE_RATIO \ +--num-prompts $(( $CONC * 10 )) --max-concurrency $CONC \ +--request-rate inf --ignore-eos \ +--save-result --percentile-metrics 'ttft,tpot,itl,e2el' \ +--result-dir /workspace/ \ +--result-filename $RESULT_FILENAME.json From a9d41b52b227d028859752ffc78a3f7538b8add2 Mon Sep 17 00:00:00 2001 From: Kimbo Chen Date: Mon, 15 Sep 2025 18:50:22 +0000 Subject: [PATCH 2/2] Updated collect results running logic. --- .github/workflows/workflow-schedule-tmpl.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/workflow-schedule-tmpl.yml b/.github/workflows/workflow-schedule-tmpl.yml index de86f5b95e..250cda4bb1 100644 --- a/.github/workflows/workflow-schedule-tmpl.yml +++ b/.github/workflows/workflow-schedule-tmpl.yml @@ -52,7 +52,7 @@ jobs: collect-70b-1k1k-results: needs: _70b-1k1k - if: ${{ always() && !cancelled() }} + if: ${{ inputs.run_1k1k && (success() || failure()) }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: @@ -76,7 +76,7 @@ jobs: collect-dsr1-1k1k-results: needs: dsr1-1k1k - if: ${{ always() && !cancelled() }} + if: ${{ inputs.run_1k1k && (success() || failure()) }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: @@ -101,7 +101,7 @@ jobs: collect-gptoss-1k1k-results: needs: gptoss-1k1k - if: ${{ always() && !cancelled() }} + if: ${{ inputs.run_1k1k && (success() || failure()) }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: @@ -126,7 +126,7 @@ jobs: collect-70b-8k1k-results: needs: _70b-8k1k - if: ${{ always() && !cancelled() }} + if: ${{ inputs.run_8k1k && (success() || failure()) }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: @@ -150,7 +150,7 @@ jobs: collect-dsr1-8k1k-results: needs: dsr1-8k1k - if: ${{ always() && !cancelled() }} + if: ${{ inputs.run_8k1k && (success() || failure()) }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: @@ -175,7 +175,7 @@ jobs: collect-gptoss-8k1k-results: needs: gptoss-8k1k - if: ${{ always() && !cancelled() }} + if: ${{ inputs.run_8k1k && (success() || failure()) }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: @@ -200,7 +200,7 @@ jobs: collect-70b-1k8k-results: needs: _70b-1k8k - if: ${{ always() && !cancelled() }} + if: ${{ inputs.run_1k8k && (success() || failure()) }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: @@ -224,7 +224,7 @@ jobs: collect-dsr1-1k8k-results: needs: dsr1-1k8k - if: ${{ always() && !cancelled() }} + if: ${{ inputs.run_1k8k && (success() || failure()) }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: @@ -249,7 +249,7 @@ jobs: collect-gptoss-1k8k-results: needs: gptoss-1k8k - if: ${{ always() && !cancelled() }} + if: ${{ inputs.run_1k8k && (success() || failure()) }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: