feat(intern-decision-serve): Intern-Decision-4B behind semif-serve's HTTP surface
Contract, service and tests (fake engine, no GPU). Scores through the checkpoint's own inference.py (DecisionEngine.predict, sha256-pinned); maps semif decisions onto Jev choice questions, packs /decide/shared into calls of at most 16, runs orderings in waves, and keeps semif's error mapping, admission, body limit and hard VRAM cap. Deltas from semif-serve are listed in the contract.
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
"""Settings for intern-decision-serve. Contract: intern-decision-serve.contract.md § Configuration.
|
||||
|
||||
Every value is validated at startup and a bad one is refused with a ValueError naming the
|
||||
variable: a service that starts and then rejects every request (or runs uncapped) is worse than
|
||||
one that does not start.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
PREFIX = "INTERN_DECISION_"
|
||||
MIN_TOKEN_CHARS = 32
|
||||
MODEL_ID = "internlm/Intern-Decision-4B"
|
||||
REVISION = "0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd"
|
||||
# The checkpoint's own inference.py is EXECUTED from the (data) mount, so its content is pinned.
|
||||
INFERENCE_PY_SHA256 = "c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863"
|
||||
DEFAULT_CHECKPOINT = f"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/{REVISION}"
|
||||
MAX_QUESTIONS_PER_CALL = 16 # the model's own limit (inference.validate_request)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
api_token: str
|
||||
checkpoint: str = DEFAULT_CHECKPOINT
|
||||
device: str = "cuda"
|
||||
vram_cap_gib: float | None = None
|
||||
max_tokens: int = 8192
|
||||
max_decisions: int = 64
|
||||
max_body_bytes: int = 1024 * 1024
|
||||
max_queue: int = 32
|
||||
release_slack_mib: int = 512
|
||||
keep_vision: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, env: Mapping[str, str]) -> "Settings":
|
||||
token = env.get(f"{PREFIX}API_TOKEN", "")
|
||||
# INV-6: visible ASCII only. A CR, LF or NUL can never arrive in a header, so a token
|
||||
# carrying one would lock every caller out while /health still said ok.
|
||||
if len(token) < MIN_TOKEN_CHARS or not all(33 <= ord(c) <= 126 for c in token):
|
||||
raise ValueError(f"{PREFIX}API_TOKEN must be at least {MIN_TOKEN_CHARS} visible ASCII characters")
|
||||
device = env.get(f"{PREFIX}DEVICE", "cuda")
|
||||
if device not in ("cuda", "cpu"):
|
||||
raise ValueError(f"{PREFIX}DEVICE must be cuda or cpu, not {device!r}")
|
||||
keep_vision = env.get(f"{PREFIX}KEEP_VISION", "0")
|
||||
if keep_vision not in ("0", "1"):
|
||||
raise ValueError(f"{PREFIX}KEEP_VISION must be 0 or 1, not {keep_vision!r}")
|
||||
return cls(
|
||||
api_token=token,
|
||||
checkpoint=env.get(f"{PREFIX}CHECKPOINT", DEFAULT_CHECKPOINT),
|
||||
device=device,
|
||||
vram_cap_gib=_positive_float(env, "VRAM_CAP_GIB"),
|
||||
max_tokens=_int(env, "MAX_TOKENS", 8192),
|
||||
max_decisions=_int(env, "MAX_DECISIONS", 64),
|
||||
max_body_bytes=_int(env, "MAX_BODY_BYTES", 1024 * 1024),
|
||||
max_queue=_int(env, "MAX_QUEUE", 32),
|
||||
release_slack_mib=_int(env, "RELEASE_SLACK_MIB", 512, minimum=0),
|
||||
keep_vision=keep_vision == "1",
|
||||
)
|
||||
|
||||
|
||||
def _int(env: Mapping[str, str], name: str, default: int, minimum: int = 1) -> int:
|
||||
raw = env.get(PREFIX + name)
|
||||
if raw is None:
|
||||
return default
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
raise ValueError(f"{PREFIX}{name} must be an integer, got {raw!r}") from None
|
||||
if value < minimum:
|
||||
raise ValueError(f"{PREFIX}{name} must be >= {minimum}, got {value}")
|
||||
return value
|
||||
|
||||
|
||||
def _positive_float(env: Mapping[str, str], name: str) -> float | None:
|
||||
"""Unset or empty means no cap. When set it must be finite and > 0."""
|
||||
raw = env.get(PREFIX + name)
|
||||
if raw is None or raw == "":
|
||||
return None
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
raise ValueError(f"{PREFIX}{name} must be a number, got {raw!r}") from None
|
||||
if not math.isfinite(value) or value <= 0:
|
||||
raise ValueError(f"{PREFIX}{name} must be a finite number > 0, got {raw!r}")
|
||||
return value
|
||||
Reference in New Issue
Block a user