Docs · vLLM fork

tcclaviger/vllm

ROCm/RDNA4 (gfx1201) vLLM: native HIP kernels, baked-in quantizers and tuner - vLLM 0.29 tree, torch 2.11, ROCm 10.0, Python 3.14.

Qwen3.8 Flash Next

TP1 - MXFP4-FP8-AA, 64 GB host

services:
  qfn-aa:
    image: tcclaviger/vllm:dev
    container_name: vllm-QFN-AA-tp1
    restart: "no"
    network_mode: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    ulimits:
      memlock: -1
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
      - <yourpath>/cache/triton:/cache/triton
      - <yourpath>/cache/vllm:/cache/vllm
      - <yourpath>/cache/inductor:/cache/inductor
    environment:
      - TRITON_CACHE_DIR=/cache/triton
      - VLLM_CACHE_ROOT=/cache/vllm
      - TORCHINDUCTOR_CACHE_DIR=/cache/inductor
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=1
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=0
    command: >
      /app/models
      --tensor-parallel-size 1
      --tool-call-parser qwen3_coder
      --enable-auto-tool-choice
      --max-num-seqs 8
      --max-num-batched-tokens 1024
      --gpu-memory-utilization 0.97
      --host 0.0.0.0
      --port 8080
      --kv-cache-dtype fp8
      --served-model-name Qwen3.8-Flash-Next
      --max-model-len 131072
      --reasoning-parser qwen3
      --override-generation-config '{"max_tokens": 65536, "temperature": 1.0, "top_p": 0.95, "top_k":20}'

TP2 - MXFP4-FP8-AA

services:
  qfn-aa:
    image: tcclaviger/vllm:dev
    container_name: vllm-QFN-AA-tp2
    restart: "no"
    network_mode: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    ulimits:
      memlock: -1
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
      - <yourpath>/cache/triton:/cache/triton
      - <yourpath>/cache/vllm:/cache/vllm
      - <yourpath>/cache/inductor:/cache/inductor
    environment:
      - TRITON_CACHE_DIR=/cache/triton
      - VLLM_CACHE_ROOT=/cache/vllm
      - TORCHINDUCTOR_CACHE_DIR=/cache/inductor
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=1
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=0,1
    command: >
      /app/models
      --tensor-parallel-size 2
      --tool-call-parser qwen3_coder
      --enable-auto-tool-choice
      --max-num-seqs 8
      --max-num-batched-tokens 4096
      --gpu-memory-utilization 0.965
      --host 0.0.0.0
      --port 8080
      --served-model-name Qwen3.8-Flash-Next
      --max-model-len 200000
      --kv-cache-dtype fp8
      --reasoning-parser qwen3
      --override-generation-config '{"max_tokens": 65536, "temperature": 1.0, "top_p": 0.95, "top_k":20}'
      --compilation-config '{"cudagraph_capture_sizes": [4,8,16,32], "max_cudagraph_capture_size": 32}'
      --speculative-config '{"method": "mtp", "num_speculative_tokens": 3, "index_share_for_mtp_iteration": true}'

TP2 - Quark MXFP4 checkpoint

services:
  qfn-quark:
    image: tcclaviger/vllm:dev
    container_name: vllm-QFN-quark
    restart: "no"
    network_mode: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    ulimits:
      memlock: -1
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
      - <yourpath>/cache/triton:/cache/triton
      - <yourpath>/cache/vllm:/cache/vllm
      - <yourpath>/cache/inductor:/cache/inductor
    environment:
      - TRITON_CACHE_DIR=/cache/triton
      - VLLM_CACHE_ROOT=/cache/vllm
      - TORCHINDUCTOR_CACHE_DIR=/cache/inductor
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=1
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=0,1
    command: >
      /app/models
      --tensor-parallel-size 2
      --tool-call-parser qwen3_coder
      --enable-auto-tool-choice
      --max-num-seqs 8
      --max-num-batched-tokens 4096
      --gpu-memory-utilization 0.965
      --host 0.0.0.0
      --port 8080
      --kv-cache-dtype fp8
      --served-model-name Qwen3.8-Flash-Next-Quark-mxfp4
      --max-model-len 200000
      --reasoning-parser qwen3
      --override-generation-config '{"max_tokens": 65536, "temperature": 1.0, "top_p": 0.95, "top_k":20}'
      --compilation-config '{"cudagraph_capture_sizes": [4,8,16,32], "max_cudagraph_capture_size": 32}'
      --speculative-config '{"method": "mtp", "num_speculative_tokens": 3, "index_share_for_mtp_iteration": true}'

TP3 - MXFP4-FP8-AA, 524K

services:
  qfn-aa:
    image: tcclaviger/vllm:dev
    container_name: vllm-QFN-AA-tp3
    restart: "no"
    network_mode: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    ulimits:
      memlock: -1
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
      - <yourpath>/cache/triton:/cache/triton
      - <yourpath>/cache/vllm:/cache/vllm
      - <yourpath>/cache/inductor:/cache/inductor
    environment:
      - TRITON_CACHE_DIR=/cache/triton
      - VLLM_CACHE_ROOT=/cache/vllm
      - TORCHINDUCTOR_CACHE_DIR=/cache/inductor
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=1
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=1,2,3
    command: >
      /app/models
      --tensor-parallel-size 3
      --tool-call-parser qwen3_coder
      --enable-auto-tool-choice
      --max-num-seqs 16
      --max-num-batched-tokens 4096
      --gpu-memory-utilization 0.97
      --host 0.0.0.0
      --port 8080
      --served-model-name Qwen3.8-Flash-Next
      --max-model-len 524288
      --kv-cache-dtype fp8
      --reasoning-parser qwen3
      --override-generation-config '{"max_tokens": 65536, "temperature": 1.0, "top_p": 0.95, "top_k":20}'
      --hf-overrides '{"text_config": {"max_position_embeddings": 524288, "rope_parameters": {"rope_type": "yarn", "factor": 2.0, "original_max_position_embeddings": 262144, "mrope_section": [11, 11, 10], "mrope_interleaved": true, "partial_rotary_factor": 0.25, "rope_theta": 10000000}}}'
      --compilation-config '{"cudagraph_capture_sizes": [4], "max_cudagraph_capture_size": 4}'
      --speculative-config '{"method": "mtp", "num_speculative_tokens": 3, "draft_sample_method": "probabilistic", "index_share_for_mtp_iteration": true}'

TP4 - MXFP4-FP8-AA, turbo, wht6 wire, 524K

services:
  qfn-aa:
    image: tcclaviger/vllm:dev
    container_name: vllm-QFN-AA-tp4
    restart: "no"
    network_mode: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    ulimits:
      memlock: -1
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
      - <yourpath>/cache/triton:/cache/triton
      - <yourpath>/cache/vllm:/cache/vllm
      - <yourpath>/cache/inductor:/cache/inductor
    environment:
      - TRITON_CACHE_DIR=/cache/triton
      - VLLM_CACHE_ROOT=/cache/vllm
      - TORCHINDUCTOR_CACHE_DIR=/cache/inductor
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=1
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=0,1,2,3
      - CLAV_Q4_TURBO=1
      - R4D_AR_WIRE=wht6
    command: >
      /app/models
      --tensor-parallel-size 4
      --tool-call-parser qwen3_coder
      --enable-auto-tool-choice
      --max-num-seqs 16
      --max-num-batched-tokens 4096
      --gpu-memory-utilization 0.975
      --host 0.0.0.0
      --port 8080
      --served-model-name Qwen3.8-Flash-Next
      --max-model-len 524288
      --kv-cache-dtype fp8
      --reasoning-parser qwen3
      --override-generation-config '{"max_tokens": 65536, "temperature": 1.0, "top_p": 0.95, "top_k":20}'
      --hf-overrides '{"text_config": {"max_position_embeddings": 524288, "rope_parameters": {"rope_type": "yarn", "factor": 2.0, "original_max_position_embeddings": 262144, "mrope_section": [11, 11, 10], "mrope_interleaved": true, "partial_rotary_factor": 0.25, "rope_theta": 10000000}}}'
      --compilation-config '{"cudagraph_capture_sizes": [4,8,16], "max_cudagraph_capture_size": 16}'
      --speculative-config '{"method": "mtp", "num_speculative_tokens": 3, "draft_sample_method": "probabilistic", "index_share_for_mtp_iteration": true}'

TP4 - W4A16 AutoRound checkpoint, 524K

services:
  qfn-autoround:
    image: tcclaviger/vllm:dev
    container_name: vllm-QFN-AutoRound-tp4
    restart: "no"
    network_mode: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    ulimits:
      memlock: -1
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
      - <yourpath>/cache/triton:/cache/triton
      - <yourpath>/cache/vllm:/cache/vllm
      - <yourpath>/cache/inductor:/cache/inductor
    environment:
      - TRITON_CACHE_DIR=/cache/triton
      - VLLM_CACHE_ROOT=/cache/vllm
      - TORCHINDUCTOR_CACHE_DIR=/cache/inductor
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=1
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=0,1,2,3
    command: >
      /app/models
      --tensor-parallel-size 4
      --tool-call-parser qwen3_coder
      --enable-auto-tool-choice
      --max-num-seqs 16
      --max-num-batched-tokens 4096
      --gpu-memory-utilization 0.94
      --host 0.0.0.0
      --port 8080
      --kv-cache-dtype fp8
      --served-model-name Qwen3.8-Flash-Next-AutoRound
      --max-model-len 524288
      --reasoning-parser qwen3
      --override-generation-config '{"max_tokens": 65536, "temperature": 0.6, "top_p": 0.95, "top_k":20}'
      --compilation-config '{"cudagraph_capture_sizes": [4,8,12,16,20,24,28,32,36,40,44,48], "max_cudagraph_capture_size": 48}'
      --speculative-config '{"method": "mtp", "num_speculative_tokens": 3, "draft_sample_method": "probabilistic"}'
      --hf-overrides '{"text_config": {"max_position_embeddings": 524288, "rope_parameters": {"rope_type": "yarn", "factor": 2.0, "original_max_position_embeddings": 262144, "mrope_section": [11, 11, 10], "mrope_interleaved": true, "partial_rotary_factor": 0.25, "rope_theta": 10000000}}}'

Qwen3.x 27B

TP2 - FP8, DFlash 7

services:
  vllm-qwen38-27b-fp8:
    image: tcclaviger/vllm:dev
    container_name: vllm-q3827bfp8
    restart: "no"
    network_mode: host
    ipc: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
      - <draftpath>:/app/draft
      - <yourpath>/cache/triton:/cache/triton
      - <yourpath>/cache/vllm:/cache/vllm
      - <yourpath>/cache/inductor:/cache/inductor
    environment:
      - TRITON_CACHE_DIR=/cache/triton
      - VLLM_CACHE_ROOT=/cache/vllm
      - TORCHINDUCTOR_CACHE_DIR=/cache/inductor
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=1
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - VLLM_CLAV_ATTN=0
      - HSA_NO_SCRATCH_RECLAIM=0
      - ROCR_VISIBLE_DEVICES=0,1
    command: >
      /app/models
      --tensor-parallel-size 2
      --tool-call-parser qwen3_coder
      --enable-auto-tool-choice
      --max-num-seqs 8
      --max-num-batched-tokens 4096
      --gpu-memory-utilization 0.95
      --host 0.0.0.0
      --port 8080
      --kv-cache-dtype fp8
      --served-model-name Qwen3.8-27B
      --max-model-len 262144
      --reasoning-parser qwen3
      --limit-mm-per-prompt.video 0
      --limit-mm-per-prompt.image 2
      --compilation-config '{"cudagraph_capture_sizes": [8, 16], "max_cudagraph_capture_size": 16}'
      --speculative-config '{"method": "dflash", "model": "/app/draft", "num_speculative_tokens": 7}'

TP2 - FP8, native MTP 6

      --speculative-config '{"method": "qwen3_next_mtp", "num_speculative_tokens": 6}'

TP4 - RFI8-AA, DFlash 7

services:
  vllm-qwen38-27b-rfi8:
    image: tcclaviger/vllm:dev
    container_name: vllm-q3827b-rfi8
    restart: "no"
    network_mode: host
    ipc: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
      - <draftpath>:/app/draft
    environment:
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=1
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=0,1,2,3
      - CLAV_TUNABLEOP_SWEEP=1
    command: >
      /app/models
      --tensor-parallel-size 4
      --tool-call-parser qwen3_coder
      --enable-auto-tool-choice
      --max-num-seqs 16
      --max-num-batched-tokens 4096
      --gpu-memory-utilization 0.92
      --host 0.0.0.0
      --port 8080
      --served-model-name Qwen3.8-27B
      --max-model-len 262144
      --kv-cache-dtype fp8
      --reasoning-parser qwen3
      --default-chat-template-kwargs '{"preserve_thinking": true}'
      --limit-mm-per-prompt.video 1
      --limit-mm-per-prompt.image 20
      --mm-processor-cache-gb .5
      --override-generation-config '{"max_tokens": 65536, "temperature": 0.8, "top_p": 0.95, "top_k": 20, "presence_penalty": 1}'
      --compilation-config '{"cudagraph_capture_sizes": [8, 16, 24, 32, 40, 48, 56, 64], "max_cudagraph_capture_size": 64}'
      --speculative-config '{"method": "dflash", "model": "/app/draft", "num_speculative_tokens": 7}'

TP4 - ThinkingCap PARO5 FP8-AA, DFlash 7, 524K

services:
  tc27b-paro5-aa:
    image: tcclaviger/vllm:dev
    container_name: vllm-TC27B-PARO5-AA
    restart: "no"
    network_mode: host
    ipc: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    ulimits:
      memlock: -1
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
      - <draftpath>:/app/draft:ro
      - <yourpath>/cache/triton:/cache/triton
      - <yourpath>/cache/vllm:/cache/vllm
      - <yourpath>/cache/inductor:/cache/inductor
    environment:
      - TRITON_CACHE_DIR=/cache/triton
      - VLLM_CACHE_ROOT=/cache/vllm
      - TORCHINDUCTOR_CACHE_DIR=/cache/inductor
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=2
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=0,1,2,3
      - CLAV_TUNABLEOP_SWEEP=1
      - CLAV_Q4_TURBO=1
      - CLAV_MTP_REDUCED_HEAD=0
      - CLAV_MTP_W4_REPACK=0
      - CLAV_MTP_W4_REFIT=0
      - CLAV_MTP_MXFP4_VERIFY=0
    command: >
      /app/models
      --tensor-parallel-size 4
      --tool-call-parser qwen3_coder
      --enable-auto-tool-choice
      --max-num-seqs 16
      --max-num-batched-tokens 4096
      --gpu-memory-utilization 0.95
      --host 0.0.0.0
      --port 8080
      --kv-cache-dtype fp8
      --served-model-name ThinkingCap-Qwen3.8-27B-PARO5-FP8-AA
      --max-model-len 524288
      --reasoning-parser qwen3
      --override-generation-config '{"max_tokens": 65536, "temperature": 0.6, "top_p": 0.95, "top_k": 20}'
      --compilation-config '{"cudagraph_mode": "PIECEWISE", "cudagraph_capture_sizes": [1, 8, 16, 24, 32, 40, 48, 56, 64], "max_cudagraph_capture_size": 64}'
      --speculative-config '{"method": "dflash", "model": "/app/draft", "num_speculative_tokens": 7}'
      --hf-overrides '{"text_config": {"max_position_embeddings": 524288, "rope_parameters": {"rope_type": "yarn", "factor": 2.0, "original_max_position_embeddings": 262144, "mrope_interleaved": true, "mrope_section": [11, 11, 10], "partial_rotary_factor": 0.25, "rope_theta": 10000000}}}'

TP4 - ThinkingCap PARO5 FP8-AA, MTP 3

      --max-model-len 262144
      --speculative-config '{"method": "mtp", "num_speculative_tokens": 3}'

Muse Glimmer

TP2 - FP8, DFlash 15, 262K

services:
  vllm-muse-glimmer-30b:
    image: tcclaviger/vllm:dev
    container_name: vllmmusefp8
    restart: "no"
    network_mode: host
    ipc: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
      - <draftpath>:/app/draft
    environment:
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=1
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=0,1
    command: >
      /app/models
      --tensor-parallel-size 2
      --tool-call-parser muse_glimmer
      --enable-auto-tool-choice
      --reasoning-parser muse_glimmer
      --max-num-seqs 8
      --max-num-batched-tokens 8192
      --gpu-memory-utilization 0.95
      --host 0.0.0.0
      --port 8080
      --served-model-name Muse-Glimmer-30B
      --max-model-len 262144
      --hf-overrides '{"text_config": {"max_position_embeddings": 262144}}'
      --limit-mm-per-prompt.video 1
      --limit-mm-per-prompt.image 20
      --mm-processor-cache-gb .5
      --override-generation-config '{"max_tokens": 65536, "temperature": 1.0, "top_p": 0.95, "top_k": 64}'
      --compilation-config '{"cudagraph_capture_sizes": [1, 16, 32, 48, 64, 80, 96, 112, 128], "max_cudagraph_capture_size": 128}'
      --speculative-config '{"method": "dflash", "model": "/app/draft", "num_speculative_tokens": 15}'

TP4 - FP8, DFlash 15, 1M

      --tensor-parallel-size 4
      --max-model-len 1048576
      --hf-overrides '{"text_config": {"max_position_embeddings": 1048576}}'

Qwen3.x 35B

TP4 - bf16, MTP 5

services:
  vllm-qwen36-35b:
    image: tcclaviger/vllm:dev
    container_name: vllmq36
    restart: "no"
    network_mode: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
    environment:
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=1
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=0,1,2,3
    command: >
      /app/models
      --tensor-parallel-size 4
      --tool-call-parser qwen3_coder
      --enable-auto-tool-choice
      --max-num-seqs 16
      --max-num-batched-tokens 4096
      --gpu-memory-utilization 0.85
      --host 0.0.0.0
      --port 8080
      --served-model-name Qwen3.6-35B
      --max-model-len 262144
      --reasoning-parser qwen3
      --limit-mm-per-prompt.video 1
      --limit-mm-per-prompt.image 20
      --mm-processor-cache-gb .5
      --override-generation-config '{"max_tokens": 65536, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "presence_penalty": 1.2}'
      --compilation-config '{"cudagraph_capture_sizes": [6,12,18,24,30,36,42,48], "max_cudagraph_capture_size": 48}'
      --speculative-config '{"method": "mtp", "num_speculative_tokens": 5}'

TP4 - bf16, MTP 5, 524K

      --max-model-len 524288
      --hf-overrides '{"text_config": {"rope_parameters": {"rope_type": "yarn", "factor": 2.0, "original_max_position_embeddings": 262144, "mrope_interleaved": true, "mrope_section": [11, 11, 10], "partial_rotary_factor": 0.25, "rope_theta": 10000000}}}'

GLM 5.3 Flash

TP4 - W4A16 AutoRound, MTP 3, expert offload

services:
  glm53-flash:
    image: tcclaviger/vllm:dev
    container_name: vllm-GLM53F
    restart: "no"
    network_mode: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    ulimits:
      memlock: -1
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models:ro
      - <yourpath>/cache/triton:/cache/triton
      - <yourpath>/cache/vllm:/cache/vllm
      - <yourpath>/cache/inductor:/cache/inductor
    environment:
      - TRITON_CACHE_DIR=/cache/triton
      - VLLM_CACHE_ROOT=/cache/vllm
      - TORCHINDUCTOR_CACHE_DIR=/cache/inductor
      - OMP_NUM_THREADS=8
      - CLAV_ATTN_AUTOTUNE=0
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=2
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=0,1,2,3
    command: >
      /app/models
      --tensor-parallel-size 4
      --tool-call-parser glm47
      --enable-auto-tool-choice
      --reasoning-parser glm45
      --max-num-seqs 16
      --max-num-batched-tokens 2048
      --gpu-memory-utilization 0.95
      --host 0.0.0.0
      --port 8080
      --kv-cache-dtype auto
      --served-model-name GLM-5.3-Flash-W4A16
      --max-model-len 262144
      --compilation-config '{"cudagraph_capture_sizes": [4], "max_cudagraph_capture_size": 4}'
      --limit-mm-per-prompt.video 0
      --limit-mm-per-prompt.image 2
      --mm-processor-cache-gb 0.2
      --speculative-config '{"method": "mtp", "num_speculative_tokens": 3}'

Gemma4 31B

TP4 - FP8, EAGLE3 3

services:
  vllm-gemma4-31b:
    image: tcclaviger/vllm:dev
    container_name: gemma4-31bfp8
    restart: "no"
    network_mode: host
    ipc: host
    cap_add:
      - SYS_PTRACE
    security_opt:
      - seccomp=unconfined
    group_add:
      - "1101"
    shm_size: 32gb
    devices:
      - /dev/kfd
      - /dev/dri
    volumes:
      - <yourpath>:/app/models
      - <draftpath>:/app/draft
    environment:
      - OMP_NUM_THREADS=8
      - VLLM_ROCM_USE_AITER=0
      - GPU_MAX_HW_QUEUES=1
      - HSA_ENABLE_INTERRUPT=1
      - HSA_ENABLE_MWAITX=1
      - ROCR_VISIBLE_DEVICES=0,1,2,3
    command: >
      /app/models
      --tensor-parallel-size 4
      --max-num-seqs 16
      --max-num-batched-tokens 4096
      --gpu-memory-utilization 0.85
      --host 0.0.0.0
      --port 8080
      --served-model-name Gemma4-31B
      --max-model-len 262144
      --enable-auto-tool-choice
      --tool-call-parser gemma4
      --reasoning-parser gemma4
      --default-chat-template-kwargs '{"enable_thinking": true, "preserve_thinking": true}'
      --limit-mm-per-prompt.video 1
      --limit-mm-per-prompt.image 20
      --mm-processor-cache-gb .5
      --override-generation-config '{"max_tokens": 65536, "temperature": 1.0, "top_p": 0.95, "top_k": 64}'
      --compilation-config '{"cudagraph_capture_sizes": [1, 2, 3, 4, 5, 6, 7, 8], "max_cudagraph_capture_size": 8}'
      --speculative-config '{"method": "eagle3", "model": "/app/draft", "num_speculative_tokens": 3}'

TP4 - FP8, no drafter

      drop the /app/draft volume and the --speculative-config line