vh
6972e7ef7f
- services/erp-seat-quant/run_quant_erp_v7.sh: v6 runner retargeted; dry-run gate passed identically (11,725 targets, 11,520 experts = 30x128x3, routers+vision BF16) - 49 GiB bf16 relayed gx10 -> ana-ml2 (no key path either way; nh3-dev relays), checksums verified against source; quant 49 -> 16 GiB, all post-steps clean - stacks/erp-seat: .env-driven swap to erp-tune-v7-nvfp4a16, served under its TRUE name; homepage labels + README updated, v6 rollback path recorded - stacks/litellm: trial -> erp-tune-v7-nvfp4a16 (config-file alias; /model/update refuses a config model, so this is an edit + restart)
88 lines
3.6 KiB
YAML
88 lines
3.6 KiB
YAML
# erp-seat — the ERP-tune seat on ana-ml2 GPU1: NVFP4A16 quant of the latest gated ERP LoRA merge
|
|
# (run 7 = jenerallee78 ARA-abliterated Gemma-4-26B-A4B + R47 SFT + the opening-split slot), served under its TRUE name.
|
|
# Routing aliases (e.g. LiteLLM `trial`) are the operator's call and live in the gateway, not here.
|
|
#
|
|
# Serve recipe copied from stacks/gemma4-charrp (same architecture + quant format, proven on this
|
|
# box): gemma4 tool + reasoning parsers, enable_thinking pinned false, model's own stock template.
|
|
# GPU1 is SHARED (charrp-MoE moved? no — scriberr, embed, rerank, coder, reward live there):
|
|
# ~47 GiB was free on 2026-09-08; 0.35 x 97.9 GiB = 34 GiB keeps ~13 GiB of real margin.
|
|
# Quant pipeline: services/erp-seat-quant/. Tunables in .env.
|
|
|
|
name: erp-seat
|
|
|
|
services:
|
|
vllm-erp-seat:
|
|
image: ${ERP_IMAGE:-vllm/vllm-openai:nightly-311b3513af33bc29b4acb2fde2e9313e5e9966a0}
|
|
container_name: ${ERP_CONTAINER:-vllm-erp-seat}
|
|
restart: unless-stopped
|
|
ipc: host
|
|
ports:
|
|
- "${ERP_PORT:-8021}:8000"
|
|
volumes:
|
|
- /tank/aimodels:/tank/aimodels
|
|
environment:
|
|
- VLLM_API_KEY=${API_KEY:-}
|
|
command:
|
|
- ${ERP_MODEL:-/tank/aimodels/erp-tune-v7-nvfp4a16}
|
|
- --quantization
|
|
- compressed-tensors
|
|
- --served-model-name
|
|
- ${ERP_SERVED_NAME:-erp-tune-v7-nvfp4a16}
|
|
- --tool-call-parser
|
|
- gemma4
|
|
- --enable-auto-tool-choice
|
|
# --reasoning-parser gemma4 + the enable_thinking pin are BOTH required (measured 2026-09-08):
|
|
# without them the post-tool-response turn leaks a literal "<|channel>thought\n<channel|>"
|
|
# prefix into content (3/3), exactly as stacks/gemma4-charrp/README.md warns. They do NOT
|
|
# fix forced (named) tool_choice — that failure is independent of the reasoning parser
|
|
# (measured 1/9 with it, 3/9 without): the model free-writes Gemma native call syntax
|
|
# instead of grammar-constrained JSON. See README for the forced-tool_choice status.
|
|
- --reasoning-parser
|
|
- gemma4
|
|
- --default-chat-template-kwargs
|
|
- '{"enable_thinking": false}'
|
|
# tool_choice:"none" trap (measured 2026-09-08): without this flag vLLM still renders the
|
|
# tools into the prompt, the model emits a tool call anyway, and because tool parsing is
|
|
# off for tool_choice=none the reply comes back with content=null AND tool_calls=null —
|
|
# an empty turn. The flag drops the tools from the prompt so the model answers in prose.
|
|
- --exclude-tools-when-tool-choice-none
|
|
- --chat-template
|
|
- ${ERP_CHAT_TEMPLATE:-/tank/aimodels/erp-tune-v7-nvfp4a16/chat_template.jinja}
|
|
- --max-model-len
|
|
- "${ERP_MAX_MODEL_LEN:-32768}"
|
|
- --max-num-seqs
|
|
- "${ERP_MAX_NUM_SEQS:-8}"
|
|
- --gpu-memory-utilization
|
|
- "${ERP_GPU_MEM_UTIL:-0.35}"
|
|
- --kv-cache-dtype
|
|
- fp8
|
|
- --trust-remote-code
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
device_ids:
|
|
- "${ERP_GPU_ID:-1}"
|
|
capabilities:
|
|
- gpu
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 600s
|
|
networks:
|
|
- tnet
|
|
labels:
|
|
- homepage.group=AI - Inference
|
|
- homepage.name=erp-tune-v7 (Gemma-4 26B-A4B ARA, NVFP4A16)
|
|
- homepage.icon=mdi-fire
|
|
- homepage.description=ERP-seat run-7 LoRA merge on the jenerallee78 abliteration, NVFP4A16 MoE (ana-ml2 GPU1)
|
|
- homepage.href=http://10.250.50.54:${ERP_PORT:-8021}/docs
|
|
|
|
networks:
|
|
tnet:
|
|
name: traefik-net
|
|
external: true
|