feat(intern-decision): h2o-lightning-4b is the production decision seat; fix forward, no INC fallback
Prime (18:05, via albok-dev): go to h2o and fix forward, do not fall back to INC. albok-dev removed the chapter margin first (the two published nonfiction titles declared visibility: public). switch-to-h2o.sh ran 18:05:13, h2o healthy 18:07:40; /health at the pin, /decide/shared 200, 422 over the cap, inline image answers, image URL refused by the shim and by vLLM's media-domain backstop. rollback-to-inc.sh stays on the host as an artifact only.
This commit is contained in:
@@ -1,46 +1,33 @@
|
||||
# intern-decision: Intern-Decision-4B (internlm, Apache-2.0) behind intern-decision-serve, on
|
||||
# fv-ml1 GPU 1 (the utility card, beside vllm-coder and the erp/meromero seats; scriberr moved to
|
||||
# GPU 3 on 2026-09-30 1322, Prime, to free this card's headroom for 32k-token calls).
|
||||
# Replaces semif (Prime, 2026-09-30: "replace semif with intern-decision now").
|
||||
#
|
||||
# One forward pass per call, scored by the checkpoint's OWN inference.py (sha256-pinned); the
|
||||
# service keeps semif-serve's HTTP surface (/decide, /decide/shared, /health). Service code +
|
||||
# contract: services/intern-decision-serve/ (intern-decision-serve.contract.md). Image built on
|
||||
# fv-ml1 from that dir.
|
||||
#
|
||||
# ⚠ VRAM_CAP_GIB is a HARD cap on torch's allocator (per-process memory fraction), set so the
|
||||
# container's WHOLE nvidia-smi footprint, CUDA context included, fits GPU 1's free memory beside
|
||||
# the static vLLM seats (infra-ops budget, 2026-09-30); MAX_TOKENS keeps every accepted call under the cap. A request that needs more
|
||||
# gets 503 out_of_memory and the service stays up. See the README before changing it.
|
||||
#
|
||||
# .env (tunables): IMAGE, PORT, GPU_ID, VRAM_CAP_GIB, MAX_TOKENS, HOST_IP, INTERN_DECISION_API_TOKEN
|
||||
# (vault intern-decision/api-token, >= 32 chars).
|
||||
|
||||
# STAGED, NOT LIVE (since the 2026-10-12 1403 rollback): the h2o-lightning-4b decision seat, which switch-to-h2o.sh
|
||||
# copies over compose.yaml. It served 1311-1403 on 2026-10-12 (Prime via brokkr) and was rolled back when Nemi's judge
|
||||
# flagged 242 texts adult against INC's 88 at fleet scale. Address intern-decision.fv.internal:8033 (brokkr rule 8).
|
||||
# h2o-lightning-4b @ b43f759b. Stock vLLM 0.30.0 + the vendor's UNMODIFIED shim + services/h2o-decision-front (copied to
|
||||
# conf/front): 32,000-token REFUSE-not-trim admission, the seat's bearer token (the same INTERN_DECISION_API_TOKEN),
|
||||
# /decide + /decide/shared translated onto /v1/systemone, /health model {name, revision}.
|
||||
# Rollback to Intern-Decision-4B: compose.inc.yaml (the previous compose, same .env): see the stack README.
|
||||
# GPU 1 is shared (meromero-rp, gen-small): KV is pinned in BYTES (4.48 GiB = 135,070 tokens, the benched size);
|
||||
# util only gates the boot check. Gate on scripts/oneoff/fv-ml1-gpucheck.sh before any (re)create.
|
||||
# Images: INLINE ONLY (Prime 2026-10-12). LIGHTNING_IMAGE_URLS=off makes the shim refuse every image URL; vLLM's
|
||||
# --allowed-media-domains media.invalid (a reserved name that never resolves) is the backstop if the shim ever changes.
|
||||
name: intern-decision
|
||||
|
||||
services:
|
||||
intern-decision:
|
||||
image: ${IMAGE:?set IMAGE}
|
||||
container_name: intern-decision
|
||||
vllm:
|
||||
image: vllm/vllm-openai:v0.30.0
|
||||
container_name: intern-decision-vllm
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${PORT:-8033}:8000"
|
||||
- "127.0.0.1:18299:8000"
|
||||
environment:
|
||||
INTERN_DECISION_API_TOKEN: ${INTERN_DECISION_API_TOKEN:?set INTERN_DECISION_API_TOKEN}
|
||||
INTERN_DECISION_DEVICE: cuda
|
||||
INTERN_DECISION_VRAM_CAP_GIB: ${VRAM_CAP_GIB:?set VRAM_CAP_GIB}
|
||||
# Coupled to VRAM_CAP_GIB: the largest call measured to fit under the cap (README "VRAM").
|
||||
INTERN_DECISION_MAX_TOKENS: ${MAX_TOKENS:?set MAX_TOKENS}
|
||||
INTERN_DECISION_MAX_DECISIONS: ${MAX_DECISIONS:-64}
|
||||
# POSTs in progress (queued + scoring) before new ones get 429 busy.
|
||||
INTERN_DECISION_MAX_QUEUE: ${MAX_QUEUE:-32}
|
||||
VLLM_USE_FLASHINFER_SAMPLER: "0"
|
||||
volumes:
|
||||
# Pinned weights AND the checkpoint's inference.py, read offline. Never downloads.
|
||||
- /tank/aimodels/huggingface:/hf:ro
|
||||
# Triton/fla autotune cache (README "Cold-shape latency"): the ~6.5 s first-call-per-bucket
|
||||
# autotune results survive RECREATES (deploys, image upgrades, .env edits), not just restarts.
|
||||
# The mount point exists in the image owned by 10001, so the fresh volume is intern-writable.
|
||||
- triton-cache:/tmp/triton-cache
|
||||
- /tank/aimodels/h2o-lightning-4b-b43f759b:/model:ro
|
||||
- vllm-cache:/root/.cache/vllm
|
||||
command: ["--model", "/model", "--served-model-name", "h2oai/h2o-lightning-4b",
|
||||
"--max-model-len", "40960", "--max-num-seqs", "16",
|
||||
"--gpu-memory-utilization", "0.1506", "--kv-cache-memory", "4810363371",
|
||||
"--limit-mm-per-prompt", "{\"image\": 4, \"video\": 0}",
|
||||
"--mm-processor-kwargs", "{\"max_pixels\": 1605632}",
|
||||
"--allowed-media-domains", "media.invalid"]
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
@@ -48,27 +35,41 @@ services:
|
||||
- driver: nvidia
|
||||
device_ids: ["${GPU_ID:-1}"]
|
||||
capabilities: [gpu]
|
||||
front:
|
||||
image: python:3.12-slim
|
||||
container_name: intern-decision
|
||||
restart: unless-stopped
|
||||
network_mode: host
|
||||
depends_on: [vllm]
|
||||
environment:
|
||||
FRONT_TOKEN: ${INTERN_DECISION_API_TOKEN:?set INTERN_DECISION_API_TOKEN}
|
||||
SHIM_VLLM: http://127.0.0.1:18299
|
||||
SHIM_PORT: "18291"
|
||||
SHIM_NOUL_FLOOR: "0.801"
|
||||
LIGHTNING_IMAGE_URLS: "off" # Prime 2026-10-12: inline images only, the seat never fetches an image URL
|
||||
FRONT_SHIM: http://127.0.0.1:18291
|
||||
FRONT_VLLM: http://127.0.0.1:18299
|
||||
FRONT_BIND: 0.0.0.0
|
||||
FRONT_PORT: ${PORT:-8033}
|
||||
FRONT_CAP: "32000"
|
||||
FRONT_SHIM_FILE: /model/h2o_lightning_shim.py
|
||||
FRONT_SERVE_CONFIG: /model/serve_config.json
|
||||
FRONT_MODEL_REVISION: b43f759b6385d21182f87c3829931ec3d4dfa98a
|
||||
volumes:
|
||||
- /tank/aimodels/h2o-lightning-4b-b43f759b:/model:ro
|
||||
- /opt/docker/conf/intern-decision/front:/front:ro
|
||||
command: ["bash", "/front/run.sh"]
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
|
||||
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8033/health', timeout=5)"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
# Startup loads ~9 GB of weights and scores the warm-up three times before it serves.
|
||||
start_period: 300s
|
||||
networks:
|
||||
- tnet
|
||||
labels:
|
||||
- homepage.group=AI - Eval & Retrieval
|
||||
- homepage.name=Intern-Decision — typed decisions
|
||||
- homepage.name=Decision seat (h2o-lightning-4b)
|
||||
- homepage.icon=mdi-scale-balance
|
||||
- homepage.description=Typed decisions from one forward pass (Intern-Decision-4B, fv-ml1 GPU1)
|
||||
- homepage.description=Typed decisions, 32k refusing (h2o-lightning-4b on vLLM, fv-ml1 GPU1; /decide/shared kept)
|
||||
- homepage.href=http://${HOST_IP:-10.251.50.54}:${PORT:-8033}/health
|
||||
|
||||
networks:
|
||||
tnet:
|
||||
name: traefik-net
|
||||
external: true
|
||||
|
||||
volumes:
|
||||
# -> intern-decision_triton-cache on the host
|
||||
triton-cache:
|
||||
vllm-cache:
|
||||
|
||||
Reference in New Issue
Block a user