[NVIDIA] Added gpt-oss for H200 TRT.

.github/workflows/gptoss-tmpl.yml

@@ -39,6 +39,23 @@ on:
         required: true
 
 jobs:
+  bmk-h200-trt:
+    if: ${{ inputs.use_h200 }}
+    uses: ./.github/workflows/benchmark-tmpl.yml
+    secrets: inherit
+    with:
+      exp-name: ${{ inputs.exp-name }}
+      isl: ${{ inputs.isl }}
+      osl: ${{ inputs.osl }}
+      max-model-len: ${{ inputs.max-model-len }}
+      random-range-ratio: ${{ inputs.random-range-ratio }}
+      runner: h200-trt
+      image: 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post1'
+      model: 'openai/gpt-oss-120b'
+      tp-list: '[1, 2, 4, 8]'
+      framework: 'trt'
+      precision: 'fp4'
+
   bmk-mi300x:
     if: ${{ inputs.use_mi300x }}
     uses: ./.github/workflows/benchmark-tmpl.yml

.github/workflows/runner-sweep-test.yml

@@ -31,6 +31,7 @@ on:
           - 'lmsysorg/sglang:v0.5.0rc1-cu128-b200'
           - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_sgl-dev-v0.5.2rc2-mi30x_rc1'
           - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2'
+          - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post1'
 
       model:
         description: 'Model'

.github/workflows/runner-test.yml

@@ -53,6 +53,7 @@ on:
           - 'lmsysorg/sglang:v0.5.0rc1-cu128-b200'
           - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_sgl-dev-v0.5.2rc2-mi30x_rc1'
           - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2'
+          - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post1'
 
       model:
         description: 'Model'

.github/workflows/workflow-schedule-tmpl.yml

@@ -52,7 +52,7 @@ jobs:
 
   collect-70b-1k1k-results:
     needs: _70b-1k1k
-    if: ${{ always() && !cancelled() }}
+    if: ${{ inputs.run_1k1k && (success() || failure()) }}
     uses: ./.github/workflows/collect-results.yml
     secrets: inherit
     with:
@@ -76,7 +76,7 @@ jobs:
 
   collect-dsr1-1k1k-results:
     needs: dsr1-1k1k
-    if: ${{ always() && !cancelled() }}
+    if: ${{ inputs.run_1k1k && (success() || failure()) }}
     uses: ./.github/workflows/collect-results.yml
     secrets: inherit
     with:
@@ -101,7 +101,7 @@ jobs:
 
   collect-gptoss-1k1k-results:
     needs: gptoss-1k1k
-    if: ${{ always() && !cancelled() }}
+    if: ${{ inputs.run_1k1k && (success() || failure()) }}
     uses: ./.github/workflows/collect-results.yml
     secrets: inherit
     with:
@@ -126,7 +126,7 @@ jobs:
 
   collect-70b-8k1k-results:
     needs: _70b-8k1k
-    if: ${{ always() && !cancelled() }}
+    if: ${{ inputs.run_8k1k && (success() || failure()) }}
     uses: ./.github/workflows/collect-results.yml
     secrets: inherit
     with:
@@ -150,7 +150,7 @@ jobs:
 
   collect-dsr1-8k1k-results:
     needs: dsr1-8k1k
-    if: ${{ always() && !cancelled() }}
+    if: ${{ inputs.run_8k1k && (success() || failure()) }}
     uses: ./.github/workflows/collect-results.yml
     secrets: inherit
     with:
@@ -175,7 +175,7 @@ jobs:
 
   collect-gptoss-8k1k-results:
     needs: gptoss-8k1k
-    if: ${{ always() && !cancelled() }}
+    if: ${{ inputs.run_8k1k && (success() || failure()) }}
     uses: ./.github/workflows/collect-results.yml
     secrets: inherit
     with:
@@ -200,7 +200,7 @@ jobs:
 
   collect-70b-1k8k-results:
     needs: _70b-1k8k
-    if: ${{ always() && !cancelled() }}
+    if: ${{ inputs.run_1k8k && (success() || failure()) }}
     uses: ./.github/workflows/collect-results.yml
     secrets: inherit
     with:
@@ -224,7 +224,7 @@ jobs:
 
   collect-dsr1-1k8k-results:
     needs: dsr1-1k8k
-    if: ${{ always() && !cancelled() }}
+    if: ${{ inputs.run_1k8k && (success() || failure()) }}
     uses: ./.github/workflows/collect-results.yml
     secrets: inherit
     with:
@@ -249,7 +249,7 @@ jobs:
 
   collect-gptoss-1k8k-results:
     needs: gptoss-1k8k
-    if: ${{ always() && !cancelled() }}
+    if: ${{ inputs.run_1k8k && (success() || failure()) }}
     uses: ./.github/workflows/collect-results.yml
     secrets: inherit
     with:

benchmarks/gptoss_fp4_h200_trt_slurm.sh

@@ -1 +1,84 @@
-echo "dummy script"
+#!/usr/bin/env bash
+
+# === Required Env Vars === 
+# HF_TOKEN
+# HF_HUB_CACHE
+# IMAGE
+# MODEL
+# ISL
+# OSL
+# MAX_MODEL_LEN
+# RANDOM_RANGE_RATIO
+# TP
+# CONC
+# RESULT_FILENAME
+# PORT_OFFSET
+
+echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+
+hf download $MODEL
+SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log)
+PORT=$(( 8888 + $PORT_OFFSET ))
+
+
+set +x
+
+git clone https://github.com/triton-lang/triton.git
+cd triton
+# Specific commit verified with TensorRT-LLM
+git checkout f3067cd3bd0c29065fa4ecdb724b6f29cbabea5f
+pip install -r python/requirements.txt          # build-time dependencies
+pip install wheel build
+python3 setup.py bdist_wheel
+pip install ./dist/*.whl
+export TRITON_ROOT=/workspace/triton
+export ENABLE_PDL=1 
+
+set -x
+cat > gptoss-config.yml << EOF
+cuda_graph_config:
+  enable_padding: true
+  max_batch_size: $CONC
+enable_attention_dp: false
+kv_cache_config:
+  dtype: auto
+  enable_block_reuse: false
+  free_gpu_memory_fraction: 0.85
+moe_config:
+  backend: TRITON
+num_postprocess_workers: 4
+print_iter_log: true
+stream_interval: 20 
+EOF
+
+
+#mpirun -n 1 --oversubscribe --allow-run-as-root trtllm-serve $MODEL --tp_size $TP --trust_remote_code --max_seq_len $MAX_MODEL_LEN --max_num_tokens $MAX_MODEL_LEN --num_postprocess_workers 2 --extra_llm_api_options llama-config.yml --port $PORT > $SERVER_LOG 2>&1 &
+mpirun -n 1 --oversubscribe --allow-run-as-root trtllm-serve $MODEL --max_batch_size $CONC --max_num_tokens 20000 --backend pytorch --extra_llm_api_options gptoss-config.yml  --ep_size 1 --trust_remote_code --gpus_per_node 8 --host 0.0.0.0 --port $PORT --tp_size=$TP --pp_size=1 > $SERVER_LOG 2>&1 &
+
+
+set +x
+while IFS= read -r line; do
+    printf '%s\n' "$line"
+    if [[ "$line" =~ [Ee][Rr][Rr][Oo][Rr] ]]; then
+        sleep 5
+        tail -n100 $SERVER_LOG
+        echo "JOB $SLURM_JOB_ID ran on NODE $SLURMD_NODENAME"
+        exit 1
+    fi
+    if [[ "$line" == *"Application startup complete"* ]]; then
+        break
+    fi
+done < <(tail -F -n0 "$SERVER_LOG")
+
+set -x
+git clone https://github.com/kimbochen/bench_serving.git
+python3 bench_serving/benchmark_serving.py \
+--model $MODEL --backend openai \
+--base-url http://0.0.0.0:$PORT \
+--dataset-name random \
+--random-input-len $ISL --random-output-len $OSL --random-range-ratio $RANDOM_RANGE_RATIO \
+--num-prompts $(( $CONC * 10 )) --max-concurrency $CONC \
+--request-rate inf --ignore-eos \
+--save-result --percentile-metrics 'ttft,tpot,itl,e2el' \
+--result-dir /workspace/ \
+--result-filename $RESULT_FILENAME.json