# ukisai/Swift-1.5-Qwen3.8-27B-GGUF Q4_K_M
# docker compose up -d   (the first start downloads the weights)
name: bench_swift15_q4km
services:
  download:
    image: python:3.12-slim
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - sh
    - -c
    - pip install -q huggingface_hub && python -c "from huggingface_hub import hf_hub_download as d; d('ukisai/Swift-1.5-Qwen3.8-27B-GGUF', 'Swift-1.5-Qwen3.8-27B-Q4_K_M.gguf', revision='14bfe4b42be4a925d98816db830155f476c605e7'); d('ukisai/Swift-1.5-Qwen3.8-27B-GGUF', 'mmproj-Swift-1.5-Qwen3.8-27B-F16.gguf', revision='14bfe4b42be4a925d98816db830155f476c605e7'); d('incoai/Qwen3.8-27B-DFlash2-GGUF', 'Qwen3.8-27B-DFlash2-BF16.gguf', revision='51962825493a48b846b40126d35c799ac4093ad0')"
  engine:
    image: ghcr.io/ggml-org/llama.cpp:server-cuda13
    container_name: bench-model
    gpus: all
    ipc: host
    ports:
    - 127.0.0.1:8000:8000
    ulimits:
      memlock: -1
      stack: 67108864
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
      CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - -m
    - /hf/hub/models--ukisai--Swift-1.5-Qwen3.8-27B-GGUF/snapshots/14bfe4b42be4a925d98816db830155f476c605e7/Swift-1.5-Qwen3.8-27B-Q4_K_M.gguf
    - --mmproj
    - /hf/hub/models--ukisai--Swift-1.5-Qwen3.8-27B-GGUF/snapshots/14bfe4b42be4a925d98816db830155f476c605e7/mmproj-Swift-1.5-Qwen3.8-27B-F16.gguf
    - --alias
    - Swift-1.5-Qwen3.8-27B-Q4_K_M
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - -c
    - '262144'
    - --parallel
    - '4'
    - --kv-unified
    - -ngl
    - '999'
    - --flash-attn
    - 'on'
    - --cache-type-k
    - q8_0
    - --cache-type-v
    - q8_0
    - --cache-ram
    - '0'
    - --slot-save-path
    - /tmp
    - --jinja
    - --model-draft
    - /hf/hub/models--incoai--Qwen3.8-27B-DFlash2-GGUF/snapshots/51962825493a48b846b40126d35c799ac4093ad0/Qwen3.8-27B-DFlash2-BF16.gguf
    - --spec-type
    - draft-dflash
    - --spec-draft-n-max
    - '7'
    - --metrics
    - --slots
    healthcheck:
      test:
      - CMD
      - curl
      - -fsS
      - http://127.0.0.1:8000/health
      interval: 15s
      timeout: 5s
      retries: 60
      start_period: 120s
    restart: 'no'
    depends_on:
      download:
        condition: service_completed_successfully