# bottlecapai/ThinkingCap-Qwen3.8-27B-GGUF Q4_K_M
# docker compose up -d   (the first start downloads the weights)
name: bench_thinkingcap_q4km
services:
  download:
    image: python:3.12-slim
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - sh
    - -c
    - pip install -q huggingface_hub && python -c "from huggingface_hub import hf_hub_download as d; d('bottlecapai/ThinkingCap-Qwen3.8-27B-GGUF', 'ThinkingCap-Qwen3.8-27B-Q4_K_M.gguf', revision='dc3aa8563d5d48d1d83637eb67147e1b52215cf7'); d('bottlecapai/ThinkingCap-Qwen3.8-27B-GGUF', 'mmproj-ThinkingCap-Qwen3.8-27B-f16.gguf', revision='dc3aa8563d5d48d1d83637eb67147e1b52215cf7'); d('incoai/Qwen3.8-27B-DFlash2-GGUF', 'Qwen3.8-27B-DFlash2-BF16.gguf', revision='51962825493a48b846b40126d35c799ac4093ad0')"
  engine:
    image: ghcr.io/ggml-org/llama.cpp:server-cuda13
    container_name: bench-model
    gpus: all
    ipc: host
    ports:
    - 127.0.0.1:8000:8000
    ulimits:
      memlock: -1
      stack: 67108864
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
      CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - -m
    - /hf/hub/models--bottlecapai--ThinkingCap-Qwen3.8-27B-GGUF/snapshots/dc3aa8563d5d48d1d83637eb67147e1b52215cf7/ThinkingCap-Qwen3.8-27B-Q4_K_M.gguf
    - --mmproj
    - /hf/hub/models--bottlecapai--ThinkingCap-Qwen3.8-27B-GGUF/snapshots/dc3aa8563d5d48d1d83637eb67147e1b52215cf7/mmproj-ThinkingCap-Qwen3.8-27B-f16.gguf
    - --alias
    - ThinkingCap-Qwen3.8-27B-Q4_K_M
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - -c
    - '262144'
    - --parallel
    - '4'
    - --kv-unified
    - -ngl
    - '999'
    - --flash-attn
    - 'on'
    - --cache-type-k
    - q8_0
    - --cache-type-v
    - q8_0
    - --cache-ram
    - '0'
    - --slot-save-path
    - /tmp
    - --jinja
    - --model-draft
    - /hf/hub/models--incoai--Qwen3.8-27B-DFlash2-GGUF/snapshots/51962825493a48b846b40126d35c799ac4093ad0/Qwen3.8-27B-DFlash2-BF16.gguf
    - --spec-type
    - draft-dflash
    - --spec-draft-n-max
    - '7'
    - --metrics
    - --slots
    healthcheck:
      test:
      - CMD
      - curl
      - -fsS
      - http://127.0.0.1:8000/health
      interval: 15s
      timeout: 5s
      retries: 60
      start_period: 120s
    restart: 'no'
    depends_on:
      download:
        condition: service_completed_successfully