# RadixArk/Qwen3.8-27B-NVFP4
# docker compose up -d   (the first start downloads the weights)
name: sglang_qwen38_27b
services:
  sglang:
    image: ${SGLANG_IMAGE:-lmsysorg/sglang:v0.5.20}
    container_name: sglang27b
    gpus: all
    ipc: host
    shm_size: 32gb
    mem_limit: ${ENGINE_MEM_LIMIT:-86g}
    memswap_limit: ${ENGINE_MEM_LIMIT:-86g}
    ports:
    - 127.0.0.1:${SGLANG_HOST_PORT:-8000}:8000
    ulimits:
      memlock: -1
      stack: 67108864
      nofile:
        soft: 1048576
        hard: 1048576
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
      CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - python3
    - -m
    - sglang.launch_server
    - --model-path
    - RadixArk/Qwen3.8-27B-NVFP4
    - --revision
    - 319f741cce68d7914884900c138a1fbb70a42f30
    - --served-model-name
    - Qwen3.8-27B
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --tp-size
    - '1'
    - --quantization
    - modelopt_fp4
    - --kv-cache-dtype
    - ${KV_CACHE_DTYPE:-fp8_e4m3}
    - --attention-backend
    - ${SGLANG_ATTENTION_BACKEND:-flashinfer}
    - --chunked-prefill-size
    - ${PREFILL_BUDGET:-2048}
    - --context-length
    - ${ENGINE_CTX:-262144}
    - --max-running-requests
    - ${MAX_RUNNING_REQUESTS:-4}
    - --max-mamba-cache-size
    - ${SGLANG_MAX_MAMBA_CACHE_SIZE:-20}
    - --mamba-radix-cache-strategy
    - ${SGLANG_MAMBA_RADIX_CACHE_STRATEGY:-extra_buffer}
    - --mamba-full-memory-ratio
    - '9.94'
    - --mamba-ssm-dtype
    - ${SGLANG_MAMBA_SSM_DTYPE:-float32}
    - --mem-fraction-static
    - ${SGLANG_MEM_FRACTION_STATIC:-0.85}
    - --disable-radix-cache
    - --reasoning-parser
    - qwen3
    - --tool-call-parser
    - qwen3_coder
    - --enable-metrics
    - --speculative-algorithm
    - DFLASH
    - --speculative-draft-model-path
    - incoai/Qwen3.8-27B-DFlash2
    - --speculative-draft-model-revision
    - 015e795645c74b1a0eeef3b570031fb62e769bc5
    - --speculative-num-draft-tokens
    - '8'
    - --speculative-draft-model-quantization
    - unquant
    healthcheck:
      test:
      - CMD
      - curl
      - -fsS
      - http://127.0.0.1:8000/health
      interval: 15s
      timeout: 5s
      retries: 60
      start_period: 3600s
    restart: 'no'