deploy qwen3.5 and nemotron
This commit is contained in:
13
vllm-qwen3.5-27b.sh
Normal file
13
vllm-qwen3.5-27b.sh
Normal file
@@ -0,0 +1,13 @@
|
||||
#! /bin/bash
|
||||
|
||||
export CUDA_VISIBLE_DEVICES="4,5"
|
||||
#export PYTHONWARNINGS="ignore::UserWarning:vllm.model_executor.layers.fla"
|
||||
MODEL="/home/kongcunliang/workspace/pretrained-models/Qwen/Qwen3.5-27B-FP8"
|
||||
|
||||
vllm serve $MODEL \
|
||||
--served-model-name "Qwen3.5-27B" \
|
||||
--port 8000 \
|
||||
--tensor-parallel-size 2 \
|
||||
--max-model-len 262144 \
|
||||
--reasoning-parser qwen3
|
||||
# --speculative-config '{"method":"mtp","num_speculative_tokens":1}'
|
||||
Reference in New Issue
Block a user