vh
c08992f95f
The :8033 seat now serves h2oai/h2o-lightning-4b @ b43f759b on vLLM 0.30.0 with byte-pinned KV (4.48 GiB, 135,558 tokens) behind services/h2o-decision-front: 32k refuse-not-trim admission, the seat's bearer token, and /decide + /decide/shared translated onto /v1/systemone for albok-nemi's judge. URL, port, token and stack name are unchanged. INC down 13:08:18, h2o healthy 13:11:27. Post-checks on the live seat: /health at the pinned revision; JevBench public 205/231 (= bench); 422 over the cap; 16q at the cap warm 2.0-2.16 s; seat peak 16,164 MiB under 4 concurrent 16q x 32k calls against a 16,847 budget. compose.inc.yaml + rollback-to-inc.sh restore Intern-Decision-4B in one step. The temporary GPU 3 eval seat (stacks/h2o-eval) is down.
71 lines
3.1 KiB
YAML
71 lines
3.1 KiB
YAML
# The decision seat on fv-ml1 GPU 1, address intern-decision.fv.internal:8033 (name kept for continuity, brokkr rule 8).
|
|
# Since 2026-10-12: h2o-lightning-4b @ b43f759b (Prime via brokkr; albok-dev signed off Nemi's judge at 0.5 through
|
|
# /decide/shared). Stock vLLM 0.30.0 + the vendor's UNMODIFIED shim + services/h2o-decision-front (copied to
|
|
# conf/front): 32,000-token REFUSE-not-trim admission, the seat's bearer token (the same INTERN_DECISION_API_TOKEN),
|
|
# /decide + /decide/shared translated onto /v1/systemone, /health model {name, revision}.
|
|
# Rollback to Intern-Decision-4B: compose.inc.yaml (the previous compose, same .env): see the stack README.
|
|
# GPU 1 is shared (meromero-rp, gen-small): KV is pinned in BYTES (4.48 GiB = 135,070 tokens, the benched size);
|
|
# util only gates the boot check. Gate on scripts/oneoff/fv-ml1-gpucheck.sh before any (re)create.
|
|
name: intern-decision
|
|
services:
|
|
vllm:
|
|
image: vllm/vllm-openai:v0.30.0
|
|
container_name: intern-decision-vllm
|
|
restart: unless-stopped
|
|
ports:
|
|
- "127.0.0.1:18299:8000"
|
|
environment:
|
|
VLLM_USE_FLASHINFER_SAMPLER: "0"
|
|
volumes:
|
|
- /tank/aimodels/h2o-lightning-4b-b43f759b:/model:ro
|
|
- vllm-cache:/root/.cache/vllm
|
|
command: ["--model", "/model", "--served-model-name", "h2oai/h2o-lightning-4b",
|
|
"--max-model-len", "40960", "--max-num-seqs", "16",
|
|
"--gpu-memory-utilization", "0.1506", "--kv-cache-memory", "4810363371",
|
|
"--limit-mm-per-prompt", "{\"image\": 4, \"video\": 0}",
|
|
"--mm-processor-kwargs", "{\"max_pixels\": 1605632}"]
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
device_ids: ["${GPU_ID:-1}"]
|
|
capabilities: [gpu]
|
|
front:
|
|
image: python:3.12-slim
|
|
container_name: intern-decision
|
|
restart: unless-stopped
|
|
network_mode: host
|
|
depends_on: [vllm]
|
|
environment:
|
|
FRONT_TOKEN: ${INTERN_DECISION_API_TOKEN:?set INTERN_DECISION_API_TOKEN}
|
|
SHIM_VLLM: http://127.0.0.1:18299
|
|
SHIM_PORT: "18291"
|
|
SHIM_NOUL_FLOOR: "0.801"
|
|
FRONT_SHIM: http://127.0.0.1:18291
|
|
FRONT_VLLM: http://127.0.0.1:18299
|
|
FRONT_BIND: 0.0.0.0
|
|
FRONT_PORT: ${PORT:-8033}
|
|
FRONT_CAP: "32000"
|
|
FRONT_SHIM_FILE: /model/h2o_lightning_shim.py
|
|
FRONT_SERVE_CONFIG: /model/serve_config.json
|
|
FRONT_MODEL_REVISION: b43f759b6385d21182f87c3829931ec3d4dfa98a
|
|
volumes:
|
|
- /tank/aimodels/h2o-lightning-4b-b43f759b:/model:ro
|
|
- /opt/docker/conf/intern-decision/front:/front:ro
|
|
command: ["bash", "/front/run.sh"]
|
|
healthcheck:
|
|
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8033/health', timeout=5)"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 300s
|
|
labels:
|
|
- homepage.group=AI - Eval & Retrieval
|
|
- homepage.name=Decision seat (h2o-lightning-4b)
|
|
- homepage.icon=mdi-scale-balance
|
|
- homepage.description=Typed decisions, 32k refusing (h2o-lightning-4b on vLLM, fv-ml1 GPU1; /decide/shared kept)
|
|
- homepage.href=http://${HOST_IP:-10.251.50.54}:${PORT:-8033}/health
|
|
volumes:
|
|
vllm-cache:
|