diff --git a/.github/workflows/70b-tmpl.yml b/.github/workflows/70b-tmpl.yml index 6e3c741fff..5211a294c8 100644 --- a/.github/workflows/70b-tmpl.yml +++ b/.github/workflows/70b-tmpl.yml @@ -62,7 +62,7 @@ jobs: secrets: inherit with: runner: h200 - image: 'kedarpotdar147/vllm0.1:latest' + image: 'vllm/vllm-openai:v0.10.2' model: 'nvidia/Llama-3.3-70B-Instruct-FP8' framework: 'vllm' precision: 'fp8' diff --git a/.github/workflows/runner-sweep-test.yml b/.github/workflows/runner-sweep-test.yml index aa1054ebd0..d120d6c448 100644 --- a/.github/workflows/runner-sweep-test.yml +++ b/.github/workflows/runner-sweep-test.yml @@ -24,6 +24,7 @@ on: options: - 'kedarpotdar147/vllm0.1:latest' - 'kedarpotdar147/vllm:05' + - 'vllm/vllm-openai:v0.10.2' - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_vllm_0.10.1_instinct_rc1' - 'rocm/vllm-dev:open-mi300-08052025' - 'rocm/vllm-dev:open-mi355-08052025' @@ -32,6 +33,7 @@ on: - 'rocm/7.0-preview:rocm7.0_preview_ubuntu_22.04_sgl-dev-v0.5.2rc2-mi30x_rc1' - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2' - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post1' + - 'nvcr.io#nvidia/tensorrt-llm/release:1.1.0rc2.post2' model: description: 'Model' diff --git a/benchmarks/70b_fp8_h200_slurm.sh b/benchmarks/70b_fp8_h200_slurm.sh index b60c938259..a88a6d0013 100644 --- a/benchmarks/70b_fp8_h200_slurm.sh +++ b/benchmarks/70b_fp8_h200_slurm.sh @@ -18,21 +18,31 @@ echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" set -x hf download $MODEL +pip install datasets pandas + +# Create config.yaml +cat > config.yaml << EOF +kv-cache-dtype: fp8 +async-scheduling: true +no-enable-prefix-caching: true +max-num-batched-tokens: 8192 +max-model-len: 10240 +EOF + SERVER_LOG=$(mktemp /tmp/server-XXXXXX.log) PORT=$(( 8888 + $PORT_OFFSET )) export TORCH_CUDA_ARCH_LIST="9.0" -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code --quantization modelopt --gpu-memory-utilization 0.9 \ ---pipeline-parallel-size 1 --tensor-parallel-size $TP --max-num-seqs $CONC --max-num-batched-tokens 8192 --max-model-len $MAX_MODEL_LEN \ ---enable-chunked-prefill --async-scheduling --no-enable-prefix-caching \ ---compilation-config '{"pass_config": {"enable_fi_allreduce_fusion": true}, "custom_ops": ["+rms_norm"], "level": 3}' \ ---disable-log-requests > $SERVER_LOG 2>&1 & + +PYTHONNOUSERSITE=1 vllm serve $MODEL --host 0.0.0.0 --port $PORT --config config.yaml \ + --gpu-memory-utilization 0.9 --tensor-parallel-size $TP --max-num-seqs $CONC \ + --disable-log-requests > $SERVER_LOG 2>&1 & set +x while IFS= read -r line; do printf '%s\n' "$line" - if [[ "$line" =~ [Ee][Rr][Rr][Oo][Rr] ]]; then + # Ignore intel_extension_for_pytorch import errors + if [[ "$line" =~ [Ee][Rr][Rr][Oo][Rr] ]] && [[ ! "$line" =~ "intel_extension_for_pytorch" ]]; then sleep 5 tail -n100 $SERVER_LOG echo "JOB $SLURM_JOB_ID ran on NODE $SLURMD_NODENAME" diff --git a/runners/launch_h200-cw.sh b/runners/launch_h200-cw.sh index 16a5fbda0d..548e733b94 100644 --- a/runners/launch_h200-cw.sh +++ b/runners/launch_h200-cw.sh @@ -7,7 +7,7 @@ MODEL_CODE="${EXP_NAME%%_*}" FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '') PARTITION="h200" -SQUASH_FILE="/mnt/vast/squash/image_${MODEL_CODE}_h200${FRAMEWORK_SUFFIX}.sqsh" +SQUASH_FILE="/mnt/vast/squash/image_${MODEL_CODE}_h200${FRAMEWORK_SUFFIX}-0915.sqsh" salloc --partition=$PARTITION --gres=gpu:$TP --exclusive --time=180 --no-shell JOB_ID=$(squeue -u $USER -h -o %A | head -n1) diff --git a/runners/launch_h200-nb.sh b/runners/launch_h200-nb.sh index a603d9b221..dede752763 100644 --- a/runners/launch_h200-nb.sh +++ b/runners/launch_h200-nb.sh @@ -7,7 +7,7 @@ MODEL_CODE="${EXP_NAME%%_*}" FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '') PARTITION="main" -SQUASH_FILE="/home/squash/image_${MODEL_CODE}_h200${FRAMEWORK_SUFFIX}.sqsh" +SQUASH_FILE="/home/squash/image_${MODEL_CODE}_h200${FRAMEWORK_SUFFIX}-0915.sqsh" salloc --partition=$PARTITION --gres=gpu:$TP --exclusive --time=180 --no-shell JOB_ID=$(squeue -u $USER -h -o %A | head -n1) diff --git a/runners/launch_h200-nv.sh b/runners/launch_h200-nv.sh index d58ffe76f5..cad91b0441 100644 --- a/runners/launch_h200-nv.sh +++ b/runners/launch_h200-nv.sh @@ -7,7 +7,7 @@ MODEL_CODE="${EXP_NAME%%_*}" FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '') PARTITION="dgx-h200" -SQUASH_FILE="/raid/image_${MODEL_CODE}_h200${FRAMEWORK_SUFFIX}.sqsh" +SQUASH_FILE="/raid/image_${MODEL_CODE}_h200${FRAMEWORK_SUFFIX}-0915.sqsh" salloc --partition=$PARTITION --gres=gpu:$TP --exclusive --time=180 --no-shell JOB_ID=$(squeue -u $USER -h -o %A | head -n1)