# unsloth/DeepSeek-V4.1-Flash-1017-GGUF UD-IQ2_XXS
# docker compose up -d   (the first start downloads the weights)
# The engine image is a local build: llama.cpp from ggml-org/llama.cpp#30817 (DeepSeek V4.1 support, not merged yet) at commit b72050d03a. Build it and tag it llamacpp-deepseek41:local first.
name: bench_deepseek41_flash_iq2xxs
services:
  download:
    image: python:3.12-slim
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - sh
    - -c
    - pip install -q huggingface_hub && python -c "from huggingface_hub import hf_hub_download as d; d('unsloth/DeepSeek-V4.1-Flash-1017-GGUF', 'UD-IQ2_XXS/DeepSeek-V4.1-Flash-1017-UD-IQ2_XXS-00001-of-00006.gguf', revision='60ad3651641b5c97679c807e064ab1d7a3ca1455'); d('unsloth/DeepSeek-V4.1-Flash-1017-GGUF', 'UD-IQ2_XXS/DeepSeek-V4.1-Flash-1017-UD-IQ2_XXS-00002-of-00006.gguf', revision='60ad3651641b5c97679c807e064ab1d7a3ca1455'); d('unsloth/DeepSeek-V4.1-Flash-1017-GGUF', 'UD-IQ2_XXS/DeepSeek-V4.1-Flash-1017-UD-IQ2_XXS-00003-of-00006.gguf', revision='60ad3651641b5c97679c807e064ab1d7a3ca1455'); d('unsloth/DeepSeek-V4.1-Flash-1017-GGUF', 'UD-IQ2_XXS/DeepSeek-V4.1-Flash-1017-UD-IQ2_XXS-00004-of-00006.gguf', revision='60ad3651641b5c97679c807e064ab1d7a3ca1455'); d('unsloth/DeepSeek-V4.1-Flash-1017-GGUF', 'UD-IQ2_XXS/DeepSeek-V4.1-Flash-1017-UD-IQ2_XXS-00005-of-00006.gguf', revision='60ad3651641b5c97679c807e064ab1d7a3ca1455'); d('unsloth/DeepSeek-V4.1-Flash-1017-GGUF', 'UD-IQ2_XXS/DeepSeek-V4.1-Flash-1017-UD-IQ2_XXS-00006-of-00006.gguf',
      revision='60ad3651641b5c97679c807e064ab1d7a3ca1455'); d('unsloth/DeepSeek-V4.1-Flash-1017-GGUF', 'mmproj-F16.gguf', revision='60ad3651641b5c97679c807e064ab1d7a3ca1455')"
  engine:
    image: llamacpp-deepseek41:local
    container_name: bench-model
    gpus: all
    ipc: host
    ports:
    - 127.0.0.1:8000:8000
    ulimits:
      memlock: -1
      stack: 67108864
    environment:
      HF_HOME: /hf
      HF_TOKEN: ${HF_TOKEN:-}
      CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-0}
    volumes:
    - type: bind
      source: ${HF_HOME:-${HOME}/.cache/huggingface}
      target: /hf
      read_only: false
      bind:
        create_host_path: true
    command:
    - -m
    - /hf/hub/models--unsloth--DeepSeek-V4.1-Flash-1017-GGUF/snapshots/60ad3651641b5c97679c807e064ab1d7a3ca1455/UD-IQ2_XXS/DeepSeek-V4.1-Flash-1017-UD-IQ2_XXS-00001-of-00006.gguf
    - --mmproj
    - /hf/hub/models--unsloth--DeepSeek-V4.1-Flash-1017-GGUF/snapshots/60ad3651641b5c97679c807e064ab1d7a3ca1455/mmproj-F16.gguf
    - --alias
    - DeepSeek-V4.1-Flash-1017-UD-IQ2_XXS
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - -c
    - '262144'
    - --parallel
    - '4'
    - --kv-unified
    - -ngl
    - '999'
    - --n-cpu-moe
    - '22'
    - --fit
    - 'off'
    - --override-tensor
    - per_layer_token_embd.weight=CPU
    - --load-mode
    - none
    - --lazy-mode
    - 'on'
    - --threads
    - '16'
    - --ubatch-size
    - '1024'
    - --batch-size
    - '2048'
    - --flash-attn
    - 'on'
    - --cache-type-k
    - q8_0
    - --cache-type-v
    - q8_0
    - --cache-ram
    - '0'
    - --jinja
    - --metrics
    - --slots
    - --slot-save-path
    - /tmp
    healthcheck:
      test:
      - CMD
      - curl
      - -fsS
      - http://127.0.0.1:8000/health
      interval: 15s
      timeout: 5s
      retries: 60
      start_period: 900s
    restart: 'no'
    depends_on:
      download:
        condition: service_completed_successfully