vh
7f066a4b79
Same model revision (df3ce67, blob sha matches fv-ml1), vLLM v0.24.0 and flags; 0.33 of the Ada (~5.4 GB, fv-ml1's absolute budget). Healthy, KV 79,808 tokens. Parity (teacher-forced true code, 40 FIM prompts, 2,215 tokens, cache-salted): each host is bit-exact with itself; cross-host |dlogprob| median 0.049, top-1 agreement 0.966 (Blackwell vs Ada + fp8 KV); against ground truth no quality difference (mean logprob diff +0.008 +/- 0.019 SE). Speed on-box: ~5x slower (64-tok FIM p50 ~1.0 s vs ~0.2 s; 63 vs 338 tok/s). Gateway coder-fast left on fv-ml1 pending Prime's call.
72 lines
2.2 KiB
YAML
72 lines
2.2 KiB
YAML
# coder-seat — the fleet's small code-completion model, on nh3-ml1 (CT 109 on
|
|
# nh3-pve, RTX 2000E Ada, 16 GB).
|
|
#
|
|
# Moved here from fv-ml1's `vllm` stack on 2026-09-25 (Prime: utility seats off
|
|
# the Blackwells). Same model revision, same vLLM version and flags; only the
|
|
# GPU, the host and the memory fraction changed (the fraction is of a 16 GB card
|
|
# now, sized to the same ~5.4 GB absolute budget).
|
|
#
|
|
# vllm-coder Qwen/Qwen2.5-Coder-1.5B (BASE, FIM) → /v1/completions :8020
|
|
#
|
|
# Consumers: the gateway alias `coder-fast` (ana-docker:4000,
|
|
# stacks/litellm/conf/config.yaml) and Zed edit-predictions (see README for how
|
|
# Zed reaches it).
|
|
#
|
|
# Shares the card with TEI embed/rerank (stacks/embed-rerank, ~2.6 GB).
|
|
name: coder-seat
|
|
services:
|
|
vllm-coder:
|
|
image: vllm/vllm-openai:${VLLM_VERSION}
|
|
container_name: vllm-coder
|
|
restart: unless-stopped
|
|
ipc: host
|
|
ports:
|
|
- "${CODER_PORT}:8000"
|
|
volumes:
|
|
- /opt/aimodels/huggingface:/hfcache
|
|
environment:
|
|
- HF_HOME=/hfcache
|
|
- HF_HUB_CACHE=/hfcache/hub
|
|
- VLLM_API_KEY=${API_KEY:-}
|
|
command:
|
|
- ${CODER_MODEL}
|
|
# Pinned: the revision fv-ml1 served (and HF main as of 2026-09-25).
|
|
- --revision
|
|
- ${CODER_REVISION}
|
|
- --served-model-name
|
|
- ${CODER_SERVED_NAME}
|
|
- --host
|
|
- 0.0.0.0
|
|
- --port
|
|
- "8000"
|
|
- --gpu-memory-utilization
|
|
- ${CODER_GPU_MEM_UTIL}
|
|
- --max-model-len
|
|
- ${CODER_MAX_MODEL_LEN}
|
|
- --max-num-seqs
|
|
- ${CODER_MAX_NUM_SEQS}
|
|
- --dtype
|
|
- auto
|
|
- --kv-cache-dtype
|
|
- ${CODER_KV_CACHE_DTYPE}
|
|
- --enable-prefix-caching
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
device_ids: ["0"]
|
|
capabilities: [gpu]
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 300s
|
|
labels:
|
|
- homepage.group=AI - Inference
|
|
- homepage.name=vLLM Qwen2.5-Coder 1.5B (FIM, nh3-ml1)
|
|
- homepage.icon=mdi-code-braces
|
|
- homepage.description=Qwen2.5-Coder-1.5B FIM code-completion (gateway coder-fast, Zed edit-predictions)
|
|
- homepage.href=http://10.100.50.80:${CODER_PORT}/docs
|