26 lines
1.0 KiB
Bash
26 lines
1.0 KiB
Bash
# qwen3.5-122b (bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4, text-only) — ana-ml2
|
|
# GPU 0 tunables. Real .env lives at /opt/docker/compose/qwen3.5-122b/.env.
|
|
|
|
# vllm/vllm-openai:latest per the repo (the text-only patch targets latest).
|
|
# MUTABLE tag — pin a digest once a known-good version is established.
|
|
QWEN35_IMAGE=vllm/vllm-openai:latest
|
|
QWEN35_CONTAINER_NAME=vllm-qwen35-122b
|
|
|
|
# Own port (8010=mistral heretic [downed], 8007=qwen36, 8011=selene — 8013 free).
|
|
QWEN35_PORT=8013
|
|
QWEN35_GPU_ID=0
|
|
|
|
# vLLM served-model-name; litellm fans out qwen3.5-122-a10b[-reasoning] + aliases
|
|
# (qwen-large, gen) onto this, differentiated by chat_template_kwargs.enable_thinking.
|
|
QWEN35_SERVED_NAME=qwen3.5-122-a10b
|
|
|
|
QWEN35_MAX_MODEL_LEN=131072
|
|
QWEN35_MAX_NUM_SEQS=8
|
|
# Single-tenant 96GB card; 0.95 is safe-aggressive (leaves ~5GB for activation
|
|
# spikes + fragmentation, esp. with 8 concurrent seqs). 0.97+ risks OOM under load.
|
|
QWEN35_GPU_MEM_UTIL=0.95
|
|
QWEN35_MAX_NUM_BATCHED_TOKENS=32768
|
|
|
|
# Optional upstream vLLM API key (empty = no auth; internal net only).
|
|
API_KEY=
|