>_ŁukaszGawenda
Configs

Run them yourself

The exact launch each model was measured with. Copy it into a compose.yaml and run docker compose up -d; the first start downloads the weights at the measured revision.

Before you start

  1. Docker with the NVIDIA Container Toolkit, and a GPU with enough memory for the model.
  2. Save the model's file below as compose.yaml in an empty folder.
  3. Optional: a .env file next to it. Docker Compose reads it on its own.
.env 
# Gated models only: your Hugging Face token
HF_TOKEN=hf_your_token_here
# Where the weights go (default ~/.cache/huggingface)
HF_HOME=/path/to/huggingface
# Which GPU to use (default: 0)
CUDA_VISIBLE_DEVICES=0

The .env holds your token: keep it private, never commit or share it.

Then docker compose up -d and wait for the first download. The server answers on http://127.0.0.1:8000/v1 (an OpenAI-compatible API). Every other setting in the file has the measured value as its default: leave it as it is to run it the way I did.

ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF

Strata · strata:0.1.45 · the video

compose.yaml Download 
# ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF
# docker compose up -d   (the first start builds Strata 0.1.45 from GitHub, then downloads the weights and sets up)
# As measured: add "answer_room": false to /data/config/strata-coder-iq1_m.json after the first start, then docker compose restart
name: bench_qwen38_flash_next_coder
services:
  engine:
    build:
      context: https://github.com/Niko1221/Strata.git#v0.1.45
    image: strata:0.1.45
    container_name: bench-model
    gpus: all
    ipc: host
    ports:
    - 127.0.0.1:8000:8000
    ulimits:
      memlock: -1
    stop_grace_period: 60s
    environment:
      CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0}
      FAMILY: coder
      MODEL: IQ1_M
      PORT: '8000'
      CONTEXT: '262144'
      VISION: gpu
      KV: int8
    volumes:
    - strata-data:/data
    healthcheck:
      test:
      - CMD
      - curl
      - -fsS
      - http://127.0.0.1:8000/health
      interval: 15s
      timeout: 5s
      retries: 60
      start_period: 7200s
    restart: 'no'
volumes:
  strata-data: {}

RadixArk/Qwen3.8-Flash-Next-NVFP4

SGLang · lmsysorg/sglang:dev-qwen38-next-local · the video

compose.yaml Download 
# RadixArk/Qwen3.8-Flash-Next-NVFP4
# docker compose up -d   (the first start downloads the weights)
name: best_sglang_official_qwen38_flash_next
services:
  flashnext-official:
    image: lmsysorg/sglang:dev-qwen38-next-local
    container_name: flashnext-official
    gpus: all
    ipc: host
    shm_size: 32gb
    mem_limit: 86g
    memswap_limit: 86g
    ports:
    - 127.0.0.1:8000:8000
    ulimits:
      memlock: -1
      stack: 67108864
      nofile:
        soft: 1048576
        hard: 1048576
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
      CUDA_VISIBLE_DEVICES: '0'
      PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
      SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1'
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    entrypoint:
    - sglang
    command:
    - serve
    - --model-path
    - RadixArk/Qwen3.8-Flash-Next-NVFP4
    - --revision
    - 7b719225242aacd3dbd3f9407468c2ee9a9d2594
    - --served-model-name
    - Qwen3.8-Flash-Next
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --tp
    - '1'
    - --quantization
    - modelopt_fp4
    - --fp4-gemm-backend
    - flashinfer_cutlass
    - --moe-runner-backend
    - flashinfer_cutlass
    - --page-size
    - '64'
    - --mamba-track-interval
    - '64'
    - --chunked-prefill-size
    - '4096'
    - --context-length
    - '262144'
    - --speculative-algorithm
    - NEXTN
    - --speculative-num-steps
    - '3'
    - --speculative-eagle-topk
    - '1'
    - --speculative-num-draft-tokens
    - '4'
    - --mamba-radix-cache-strategy
    - extra_buffer_lazy
    - --max-running-requests
    - '1'
    - --max-mamba-cache-size
    - '5'
    - --mamba-ssm-dtype
    - bfloat16
    - --mem-fraction-static
    - ${SGLANG_MEM_FRACTION_STATIC:-0.96}
    - --cuda-graph-max-bs-decode
    - '1'
    - --ple-offload-embedding
    - --reasoning-parser
    - qwen3
    - --tool-call-parser
    - qwen3_coder
    - --enable-metrics
    healthcheck:
      test:
      - CMD
      - curl
      - -fsS
      - http://127.0.0.1:8000/health
      interval: 15s
      timeout: 5s
      retries: 40
      start_period: 3600s
    restart: 'no'

bottlecapai/ThinkingCap-Qwen3.8-27B-GGUF Q4_K_M

llama.cpp · ghcr.io/ggml-org/llama.cpp:server-cuda13 · the video

compose.yaml Download 
# bottlecapai/ThinkingCap-Qwen3.8-27B-GGUF Q4_K_M
# docker compose up -d   (the first start downloads the weights)
name: bench_thinkingcap_q4km
services:
  download:
    image: python:3.12-slim
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - sh
    - -c
    - pip install -q huggingface_hub && python -c "from huggingface_hub import hf_hub_download as d; d('bottlecapai/ThinkingCap-Qwen3.8-27B-GGUF', 'ThinkingCap-Qwen3.8-27B-Q4_K_M.gguf', revision='dc3aa8563d5d48d1d83637eb67147e1b52215cf7'); d('bottlecapai/ThinkingCap-Qwen3.8-27B-GGUF', 'mmproj-ThinkingCap-Qwen3.8-27B-f16.gguf', revision='dc3aa8563d5d48d1d83637eb67147e1b52215cf7'); d('incoai/Qwen3.8-27B-DFlash2-GGUF', 'Qwen3.8-27B-DFlash2-BF16.gguf', revision='51962825493a48b846b40126d35c799ac4093ad0')"
  engine:
    image: ghcr.io/ggml-org/llama.cpp:server-cuda13
    container_name: bench-model
    gpus: all
    ipc: host
    ports:
    - 127.0.0.1:8000:8000
    ulimits:
      memlock: -1
      stack: 67108864
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
      CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - -m
    - /hf/hub/models--bottlecapai--ThinkingCap-Qwen3.8-27B-GGUF/snapshots/dc3aa8563d5d48d1d83637eb67147e1b52215cf7/ThinkingCap-Qwen3.8-27B-Q4_K_M.gguf
    - --mmproj
    - /hf/hub/models--bottlecapai--ThinkingCap-Qwen3.8-27B-GGUF/snapshots/dc3aa8563d5d48d1d83637eb67147e1b52215cf7/mmproj-ThinkingCap-Qwen3.8-27B-f16.gguf
    - --alias
    - ThinkingCap-Qwen3.8-27B-Q4_K_M
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - -c
    - '262144'
    - --parallel
    - '4'
    - --kv-unified
    - -ngl
    - '999'
    - --flash-attn
    - 'on'
    - --cache-type-k
    - q8_0
    - --cache-type-v
    - q8_0
    - --cache-ram
    - '0'
    - --slot-save-path
    - /tmp
    - --jinja
    - --model-draft
    - /hf/hub/models--incoai--Qwen3.8-27B-DFlash2-GGUF/snapshots/51962825493a48b846b40126d35c799ac4093ad0/Qwen3.8-27B-DFlash2-BF16.gguf
    - --spec-type
    - draft-dflash
    - --spec-draft-n-max
    - '7'
    - --metrics
    - --slots
    healthcheck:
      test:
      - CMD
      - curl
      - -fsS
      - http://127.0.0.1:8000/health
      interval: 15s
      timeout: 5s
      retries: 60
      start_period: 120s
    restart: 'no'
    depends_on:
      download:
        condition: service_completed_successfully

ukisai/Swift-1.5-Qwen3.8-27B-GGUF Q4_K_M

llama.cpp · ghcr.io/ggml-org/llama.cpp:server-cuda13 · the video

compose.yaml Download 
# ukisai/Swift-1.5-Qwen3.8-27B-GGUF Q4_K_M
# docker compose up -d   (the first start downloads the weights)
name: bench_swift15_q4km
services:
  download:
    image: python:3.12-slim
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - sh
    - -c
    - pip install -q huggingface_hub && python -c "from huggingface_hub import hf_hub_download as d; d('ukisai/Swift-1.5-Qwen3.8-27B-GGUF', 'Swift-1.5-Qwen3.8-27B-Q4_K_M.gguf', revision='14bfe4b42be4a925d98816db830155f476c605e7'); d('ukisai/Swift-1.5-Qwen3.8-27B-GGUF', 'mmproj-Swift-1.5-Qwen3.8-27B-F16.gguf', revision='14bfe4b42be4a925d98816db830155f476c605e7'); d('incoai/Qwen3.8-27B-DFlash2-GGUF', 'Qwen3.8-27B-DFlash2-BF16.gguf', revision='51962825493a48b846b40126d35c799ac4093ad0')"
  engine:
    image: ghcr.io/ggml-org/llama.cpp:server-cuda13
    container_name: bench-model
    gpus: all
    ipc: host
    ports:
    - 127.0.0.1:8000:8000
    ulimits:
      memlock: -1
      stack: 67108864
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
      CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - -m
    - /hf/hub/models--ukisai--Swift-1.5-Qwen3.8-27B-GGUF/snapshots/14bfe4b42be4a925d98816db830155f476c605e7/Swift-1.5-Qwen3.8-27B-Q4_K_M.gguf
    - --mmproj
    - /hf/hub/models--ukisai--Swift-1.5-Qwen3.8-27B-GGUF/snapshots/14bfe4b42be4a925d98816db830155f476c605e7/mmproj-Swift-1.5-Qwen3.8-27B-F16.gguf
    - --alias
    - Swift-1.5-Qwen3.8-27B-Q4_K_M
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - -c
    - '262144'
    - --parallel
    - '4'
    - --kv-unified
    - -ngl
    - '999'
    - --flash-attn
    - 'on'
    - --cache-type-k
    - q8_0
    - --cache-type-v
    - q8_0
    - --cache-ram
    - '0'
    - --slot-save-path
    - /tmp
    - --jinja
    - --model-draft
    - /hf/hub/models--incoai--Qwen3.8-27B-DFlash2-GGUF/snapshots/51962825493a48b846b40126d35c799ac4093ad0/Qwen3.8-27B-DFlash2-BF16.gguf
    - --spec-type
    - draft-dflash
    - --spec-draft-n-max
    - '7'
    - --metrics
    - --slots
    healthcheck:
      test:
      - CMD
      - curl
      - -fsS
      - http://127.0.0.1:8000/health
      interval: 15s
      timeout: 5s
      retries: 60
      start_period: 120s
    restart: 'no'
    depends_on:
      download:
        condition: service_completed_successfully

ukisai/Swift-1.5-Qwen3.8-27b-NVFP4

SGLang · lmsysorg/sglang:v0.5.20 · the video

compose.yaml Download 
# ukisai/Swift-1.5-Qwen3.8-27b-NVFP4
# docker compose up -d   (the first start downloads the weights)
name: sglang_qwen38_27b
services:
  sglang:
    image: ${SGLANG_IMAGE:-lmsysorg/sglang:v0.5.20}
    container_name: sglang27b
    gpus: all
    ipc: host
    shm_size: 32gb
    mem_limit: ${ENGINE_MEM_LIMIT:-86g}
    memswap_limit: ${ENGINE_MEM_LIMIT:-86g}
    ports:
    - 127.0.0.1:${SGLANG_HOST_PORT:-8000}:8000
    ulimits:
      memlock: -1
      stack: 67108864
      nofile:
        soft: 1048576
        hard: 1048576
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
      CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - python3
    - -m
    - sglang.launch_server
    - --model-path
    - ukisai/Swift-1.5-Qwen3.8-27b-NVFP4
    - --revision
    - 25482027debd5485e8108897ba9fed8d3ba16595
    - --served-model-name
    - Swift-1.5-Qwen3.8-27B
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --tp-size
    - '1'
    - --quantization
    - modelopt_fp4
    - --kv-cache-dtype
    - ${KV_CACHE_DTYPE:-fp8_e4m3}
    - --attention-backend
    - ${SGLANG_ATTENTION_BACKEND:-flashinfer}
    - --chunked-prefill-size
    - ${PREFILL_BUDGET:-2048}
    - --context-length
    - ${ENGINE_CTX:-262144}
    - --max-running-requests
    - ${MAX_RUNNING_REQUESTS:-4}
    - --max-mamba-cache-size
    - ${SGLANG_MAX_MAMBA_CACHE_SIZE:-20}
    - --mamba-radix-cache-strategy
    - ${SGLANG_MAMBA_RADIX_CACHE_STRATEGY:-extra_buffer}
    - --mamba-full-memory-ratio
    - '9.94'
    - --mamba-ssm-dtype
    - ${SGLANG_MAMBA_SSM_DTYPE:-float32}
    - --mem-fraction-static
    - ${SGLANG_MEM_FRACTION_STATIC:-0.85}
    - --disable-radix-cache
    - --reasoning-parser
    - qwen3
    - --tool-call-parser
    - qwen3_coder
    - --enable-metrics
    - --speculative-algorithm
    - DFLASH
    - --speculative-draft-model-path
    - incoai/Qwen3.8-27B-DFlash2
    - --speculative-draft-model-revision
    - 015e795645c74b1a0eeef3b570031fb62e769bc5
    - --speculative-num-draft-tokens
    - '8'
    - --speculative-draft-model-quantization
    - unquant
    healthcheck:
      test:
      - CMD
      - curl
      - -fsS
      - http://127.0.0.1:8000/health
      interval: 15s
      timeout: 5s
      retries: 60
      start_period: 3600s
    restart: 'no'

RadixArk/Qwen3.8-27B-NVFP4

SGLang · lmsysorg/sglang:v0.5.20 · the video

compose.yaml Download 
# RadixArk/Qwen3.8-27B-NVFP4
# docker compose up -d   (the first start downloads the weights)
name: sglang_qwen38_27b
services:
  sglang:
    image: ${SGLANG_IMAGE:-lmsysorg/sglang:v0.5.20}
    container_name: sglang27b
    gpus: all
    ipc: host
    shm_size: 32gb
    mem_limit: ${ENGINE_MEM_LIMIT:-86g}
    memswap_limit: ${ENGINE_MEM_LIMIT:-86g}
    ports:
    - 127.0.0.1:${SGLANG_HOST_PORT:-8000}:8000
    ulimits:
      memlock: -1
      stack: 67108864
      nofile:
        soft: 1048576
        hard: 1048576
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
      CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - python3
    - -m
    - sglang.launch_server
    - --model-path
    - RadixArk/Qwen3.8-27B-NVFP4
    - --revision
    - 319f741cce68d7914884900c138a1fbb70a42f30
    - --served-model-name
    - Qwen3.8-27B
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --tp-size
    - '1'
    - --quantization
    - modelopt_fp4
    - --kv-cache-dtype
    - ${KV_CACHE_DTYPE:-fp8_e4m3}
    - --attention-backend
    - ${SGLANG_ATTENTION_BACKEND:-flashinfer}
    - --chunked-prefill-size
    - ${PREFILL_BUDGET:-2048}
    - --context-length
    - ${ENGINE_CTX:-262144}
    - --max-running-requests
    - ${MAX_RUNNING_REQUESTS:-4}
    - --max-mamba-cache-size
    - ${SGLANG_MAX_MAMBA_CACHE_SIZE:-20}
    - --mamba-radix-cache-strategy
    - ${SGLANG_MAMBA_RADIX_CACHE_STRATEGY:-extra_buffer}
    - --mamba-full-memory-ratio
    - '9.94'
    - --mamba-ssm-dtype
    - ${SGLANG_MAMBA_SSM_DTYPE:-float32}
    - --mem-fraction-static
    - ${SGLANG_MEM_FRACTION_STATIC:-0.85}
    - --disable-radix-cache
    - --reasoning-parser
    - qwen3
    - --tool-call-parser
    - qwen3_coder
    - --enable-metrics
    - --speculative-algorithm
    - DFLASH
    - --speculative-draft-model-path
    - incoai/Qwen3.8-27B-DFlash2
    - --speculative-draft-model-revision
    - 015e795645c74b1a0eeef3b570031fb62e769bc5
    - --speculative-num-draft-tokens
    - '8'
    - --speculative-draft-model-quantization
    - unquant
    healthcheck:
      test:
      - CMD
      - curl
      - -fsS
      - http://127.0.0.1:8000/health
      interval: 15s
      timeout: 5s
      retries: 60
      start_period: 3600s
    restart: 'no'

orcarouter/Qwen3.8-27B-Uncensored-NVFP4

SGLang · lmsysorg/sglang:v0.5.20 · the video

compose.yaml Download 
# orcarouter/Qwen3.8-27B-Uncensored-NVFP4
# docker compose up -d   (the first start downloads the weights)
name: sglang_qwen38_27b
services:
  sglang:
    image: ${SGLANG_IMAGE:-lmsysorg/sglang:v0.5.20}
    container_name: sglang27b
    gpus: all
    ipc: host
    shm_size: 32gb
    mem_limit: ${ENGINE_MEM_LIMIT:-86g}
    memswap_limit: ${ENGINE_MEM_LIMIT:-86g}
    ports:
    - 127.0.0.1:${SGLANG_HOST_PORT:-8000}:8000
    ulimits:
      memlock: -1
      stack: 67108864
      nofile:
        soft: 1048576
        hard: 1048576
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
      CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - python3
    - -m
    - sglang.launch_server
    - --model-path
    - orcarouter/Qwen3.8-27B-Uncensored-NVFP4
    - --revision
    - 96d4d0b66d94314909d1af73f955e94bf68335ac
    - --served-model-name
    - Qwen3.8-27B-Uncensored
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --tp-size
    - '1'
    - --quantization
    - compressed-tensors
    - --kv-cache-dtype
    - ${KV_CACHE_DTYPE:-fp8_e4m3}
    - --attention-backend
    - ${SGLANG_ATTENTION_BACKEND:-flashinfer}
    - --chunked-prefill-size
    - ${PREFILL_BUDGET:-2048}
    - --context-length
    - ${ENGINE_CTX:-262144}
    - --max-running-requests
    - ${MAX_RUNNING_REQUESTS:-4}
    - --max-mamba-cache-size
    - ${SGLANG_MAX_MAMBA_CACHE_SIZE:-20}
    - --mamba-radix-cache-strategy
    - ${SGLANG_MAMBA_RADIX_CACHE_STRATEGY:-extra_buffer}
    - --mamba-full-memory-ratio
    - '9.94'
    - --mamba-ssm-dtype
    - ${SGLANG_MAMBA_SSM_DTYPE:-float32}
    - --mem-fraction-static
    - ${SGLANG_MEM_FRACTION_STATIC:-0.85}
    - --disable-radix-cache
    - --reasoning-parser
    - qwen3
    - --tool-call-parser
    - qwen3_coder
    - --enable-metrics
    - --speculative-algorithm
    - DFLASH
    - --speculative-draft-model-path
    - incoai/Qwen3.8-27B-DFlash2
    - --speculative-draft-model-revision
    - 015e795645c74b1a0eeef3b570031fb62e769bc5
    - --speculative-num-draft-tokens
    - '8'
    - --speculative-draft-model-quantization
    - unquant
    healthcheck:
      test:
      - CMD
      - curl
      - -fsS
      - http://127.0.0.1:8000/health
      interval: 15s
      timeout: 5s
      retries: 60
      start_period: 3600s
    restart: 'no'