vh
5bbf0aaeba
Contract, service and tests (fake engine, no GPU). Scores through the checkpoint's own inference.py (DecisionEngine.predict, sha256-pinned); maps semif decisions onto Jev choice questions, packs /decide/shared into calls of at most 16, runs orderings in waves, and keeps semif's error mapping, admission, body limit and hard VRAM cap. Deltas from semif-serve are listed in the contract.
88 lines
3.6 KiB
Python
88 lines
3.6 KiB
Python
"""Settings for intern-decision-serve. Contract: intern-decision-serve.contract.md § Configuration.
|
|
|
|
Every value is validated at startup and a bad one is refused with a ValueError naming the
|
|
variable: a service that starts and then rejects every request (or runs uncapped) is worse than
|
|
one that does not start.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from collections.abc import Mapping
|
|
from dataclasses import dataclass
|
|
|
|
PREFIX = "INTERN_DECISION_"
|
|
MIN_TOKEN_CHARS = 32
|
|
MODEL_ID = "internlm/Intern-Decision-4B"
|
|
REVISION = "0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd"
|
|
# The checkpoint's own inference.py is EXECUTED from the (data) mount, so its content is pinned.
|
|
INFERENCE_PY_SHA256 = "c904e2c67ca0775621a22375ee373d2ba30b52117cda870c6c9ef74143b29863"
|
|
DEFAULT_CHECKPOINT = f"/hf/hub/models--internlm--Intern-Decision-4B/snapshots/{REVISION}"
|
|
MAX_QUESTIONS_PER_CALL = 16 # the model's own limit (inference.validate_request)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Settings:
|
|
api_token: str
|
|
checkpoint: str = DEFAULT_CHECKPOINT
|
|
device: str = "cuda"
|
|
vram_cap_gib: float | None = None
|
|
max_tokens: int = 8192
|
|
max_decisions: int = 64
|
|
max_body_bytes: int = 1024 * 1024
|
|
max_queue: int = 32
|
|
release_slack_mib: int = 512
|
|
keep_vision: bool = False
|
|
|
|
@classmethod
|
|
def from_env(cls, env: Mapping[str, str]) -> "Settings":
|
|
token = env.get(f"{PREFIX}API_TOKEN", "")
|
|
# INV-6: visible ASCII only. A CR, LF or NUL can never arrive in a header, so a token
|
|
# carrying one would lock every caller out while /health still said ok.
|
|
if len(token) < MIN_TOKEN_CHARS or not all(33 <= ord(c) <= 126 for c in token):
|
|
raise ValueError(f"{PREFIX}API_TOKEN must be at least {MIN_TOKEN_CHARS} visible ASCII characters")
|
|
device = env.get(f"{PREFIX}DEVICE", "cuda")
|
|
if device not in ("cuda", "cpu"):
|
|
raise ValueError(f"{PREFIX}DEVICE must be cuda or cpu, not {device!r}")
|
|
keep_vision = env.get(f"{PREFIX}KEEP_VISION", "0")
|
|
if keep_vision not in ("0", "1"):
|
|
raise ValueError(f"{PREFIX}KEEP_VISION must be 0 or 1, not {keep_vision!r}")
|
|
return cls(
|
|
api_token=token,
|
|
checkpoint=env.get(f"{PREFIX}CHECKPOINT", DEFAULT_CHECKPOINT),
|
|
device=device,
|
|
vram_cap_gib=_positive_float(env, "VRAM_CAP_GIB"),
|
|
max_tokens=_int(env, "MAX_TOKENS", 8192),
|
|
max_decisions=_int(env, "MAX_DECISIONS", 64),
|
|
max_body_bytes=_int(env, "MAX_BODY_BYTES", 1024 * 1024),
|
|
max_queue=_int(env, "MAX_QUEUE", 32),
|
|
release_slack_mib=_int(env, "RELEASE_SLACK_MIB", 512, minimum=0),
|
|
keep_vision=keep_vision == "1",
|
|
)
|
|
|
|
|
|
def _int(env: Mapping[str, str], name: str, default: int, minimum: int = 1) -> int:
|
|
raw = env.get(PREFIX + name)
|
|
if raw is None:
|
|
return default
|
|
try:
|
|
value = int(raw)
|
|
except ValueError:
|
|
raise ValueError(f"{PREFIX}{name} must be an integer, got {raw!r}") from None
|
|
if value < minimum:
|
|
raise ValueError(f"{PREFIX}{name} must be >= {minimum}, got {value}")
|
|
return value
|
|
|
|
|
|
def _positive_float(env: Mapping[str, str], name: str) -> float | None:
|
|
"""Unset or empty means no cap. When set it must be finite and > 0."""
|
|
raw = env.get(PREFIX + name)
|
|
if raw is None or raw == "":
|
|
return None
|
|
try:
|
|
value = float(raw)
|
|
except ValueError:
|
|
raise ValueError(f"{PREFIX}{name} must be a number, got {raw!r}") from None
|
|
if not math.isfinite(value) or value <= 0:
|
|
raise ValueError(f"{PREFIX}{name} must be a finite number > 0, got {raw!r}")
|
|
return value
|