feat(intern-decision-serve): Intern-Decision-4B behind semif-serve's HTTP surface
Contract, service and tests (fake engine, no GPU). Scores through the checkpoint's own inference.py (DecisionEngine.predict, sha256-pinned); maps semif decisions onto Jev choice questions, packs /decide/shared into calls of at most 16, runs orderings in waves, and keeps semif's error mapping, admission, body limit and hard VRAM cap. Deltas from semif-serve are listed in the contract.
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
@@ -0,0 +1,55 @@
|
||||
"""A torch-free stand-in for the real engine: answers Jev requests in the shape
|
||||
DecisionEngine.predict() returns, and records every call it was given."""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
|
||||
CALIBRATION = {"method": "temperature-scaling", "temperature": 1.99241824}
|
||||
|
||||
|
||||
def softmax(xs: list[float]) -> list[float]:
|
||||
top = max(xs)
|
||||
e = [math.exp(x - top) for x in xs]
|
||||
return [v / sum(e) for v in e]
|
||||
|
||||
|
||||
def score_by_description(field: str, question: dict) -> list[float]:
|
||||
"""Default scorer: the option whose description is longest wins; a small first-position bias."""
|
||||
descs = list(question["criteria"].values())
|
||||
return [len(d) + (0.5 if i == 0 else 0.0) for i, d in enumerate(descs)]
|
||||
|
||||
|
||||
class FakeEngine:
|
||||
def __init__(self, scorer=score_by_description, tokens_per_question: int = 100):
|
||||
self.scorer = scorer
|
||||
self.tokens_per_question = tokens_per_question
|
||||
self.calls: list[dict] = []
|
||||
|
||||
metadata = {"name": "Intern-Decision-4B", "revision": "0" * 40}
|
||||
|
||||
def health(self) -> dict:
|
||||
return {**self.metadata, "reserved_gib": 9.1}
|
||||
|
||||
def predict(self, request: dict) -> tuple[dict, str]:
|
||||
self.calls.append(copy.deepcopy(request))
|
||||
answers = {}
|
||||
for field, question in request["questions"].items():
|
||||
ids = list(question["criteria"])
|
||||
probs = dict(zip(ids, softmax(self.scorer(field, question))))
|
||||
best = min(ids, key=lambda i: (-probs[i], i))
|
||||
answers[field] = {"type": "choice", "probabilities": probs, "confidence": probs[best],
|
||||
"choice": best, "source": "local", "decision": best}
|
||||
response = {"answers": answers,
|
||||
"usage": {"input_tokens": self.tokens_per_question * len(answers),
|
||||
"output_tokens": len(answers), "decision_count": len(answers)},
|
||||
"timing": {"inference_ms": 12.5}, "calibration": dict(CALIBRATION),
|
||||
"model": "Intern-Decision-4B", "backend": "hf"}
|
||||
return response, request_sha(request)
|
||||
|
||||
|
||||
def request_sha(request: dict) -> str:
|
||||
"""Stands in for the real prompt hash: the same call always hashes the same."""
|
||||
return hashlib.sha256(json.dumps(request, sort_keys=True).encode()).hexdigest()
|
||||
@@ -0,0 +1,434 @@
|
||||
"""intern-decision-serve HTTP behaviour against a fake engine (no torch, no model).
|
||||
Contract: services/intern-decision-serve/intern-decision-serve.contract.md"""
|
||||
import json
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from fake_engine import CALIBRATION, FakeEngine, request_sha
|
||||
from intern_decision_serve.app import create_app
|
||||
from intern_decision_serve.config import Settings
|
||||
|
||||
TOKEN = "t" * 40
|
||||
AUTH = {"Authorization": f"Bearer {TOKEN}"}
|
||||
OPTIONS = [{"id": "yes", "description": "It passed."}, {"id": "no", "description": "It did not pass at all."}]
|
||||
ROW = {"id": "r1", "state": "The deploy passed.", "question": "Did it pass?", "options": OPTIONS}
|
||||
|
||||
|
||||
def make_client(engine=None, **overrides):
|
||||
return TestClient(create_app(Settings(api_token=TOKEN, **overrides), engine or FakeEngine()))
|
||||
|
||||
|
||||
def test_decide_asks_one_question_named_q_and_maps_the_answer_back_in_option_order():
|
||||
engine = FakeEngine()
|
||||
response = make_client(engine).post("/decide", json=ROW, headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
assert engine.calls == [{"state": "The deploy passed.", "questions": {"q": {
|
||||
"type": "choice", "instructions": "Did it pass?",
|
||||
"criteria": {"yes": "It passed.", "no": "It did not pass at all."}}}}]
|
||||
out = response.json()
|
||||
native, _ = FakeEngine().predict(engine.calls[0])
|
||||
answer = native["answers"]["q"]
|
||||
assert out["id"] == "r1"
|
||||
assert out["option_ids"] == ["yes", "no"]
|
||||
assert out["probabilities"] == [answer["probabilities"]["yes"], answer["probabilities"]["no"]]
|
||||
assert out["top"] == "no" == answer["choice"]
|
||||
assert out["confidence"] == answer["confidence"]
|
||||
assert out["calibration"] == CALIBRATION
|
||||
assert out["native"] == answer
|
||||
assert out["input_tokens"] == 100
|
||||
assert out["prompt_sha256"] == request_sha(engine.calls[0])
|
||||
assert out["call"] == {"index": 0, "field": "q", "questions": 1}
|
||||
assert out["forward_seconds"] == 0.0125
|
||||
assert out["total_seconds"] >= 0
|
||||
assert out["model"] == FakeEngine.metadata
|
||||
for key in ("prompt_version", "readout", "probability_status"):
|
||||
assert out[key]
|
||||
|
||||
|
||||
import pytest # noqa: E402
|
||||
|
||||
|
||||
@pytest.mark.parametrize("headers", [{}, {"Authorization": "Bearer wrong"}, {"Authorization": TOKEN}])
|
||||
def test_posts_without_the_right_bearer_are_401_and_never_reach_the_engine(headers):
|
||||
engine = FakeEngine()
|
||||
client = make_client(engine)
|
||||
for path, body in (("/decide", ROW), ("/decide/shared", {"state": "s", "decisions": [ROW]})):
|
||||
response = client.post(path, json=body, headers=headers)
|
||||
assert response.status_code == 401
|
||||
assert response.json()["error"]["code"] == "unauthorized"
|
||||
assert engine.calls == []
|
||||
|
||||
|
||||
def test_health_needs_no_auth_and_reports_the_limits_and_the_chunking_rule():
|
||||
response = make_client(vram_cap_gib=11.0).get("/health")
|
||||
assert response.status_code == 200
|
||||
out = response.json()
|
||||
assert out["status"] == "ok"
|
||||
assert out["model"] == FakeEngine().health()
|
||||
assert (out["vram_cap_gib"], out["max_tokens"], out["max_decisions"]) == (11.0, 8192, 64)
|
||||
assert out["max_questions_per_call"] == 16
|
||||
assert "16" in out["chunking"]
|
||||
assert out["workloads"] == []
|
||||
|
||||
|
||||
def decisions(n, options=OPTIONS):
|
||||
return [{"id": f"decision-{i}", "question": f"Question {i}?", "options": options} for i in range(n)]
|
||||
|
||||
|
||||
def test_shared_asks_q1_to_qn_over_the_shared_state_in_one_call():
|
||||
engine = FakeEngine()
|
||||
body = {"state": {"deploy": "passed"}, "decisions": decisions(3)}
|
||||
response = make_client(engine).post("/decide/shared", json=body, headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
assert engine.calls == [{"state": {"deploy": "passed"}, "questions": {
|
||||
f"q{i + 1}": {"type": "choice", "instructions": f"Question {i}?",
|
||||
"criteria": {"yes": "It passed.", "no": "It did not pass at all."}} for i in range(3)}}]
|
||||
out = response.json()
|
||||
assert [r["id"] for r in out["results"]] == ["decision-0", "decision-1", "decision-2"]
|
||||
assert [r["call"] for r in out["results"]] == [{"index": 0, "field": f"q{i}", "questions": 3} for i in (1, 2, 3)]
|
||||
assert out["timing"]["calls"] == 1
|
||||
assert out["timing"]["batch_size"] == 3
|
||||
assert out["timing"]["questions_per_call"] == [3]
|
||||
assert out["timing"]["input_tokens"] == [300]
|
||||
assert out["timing"]["inference_seconds"] == 0.0125
|
||||
assert out["timing"]["total_seconds"] >= 0
|
||||
|
||||
|
||||
def test_one_shared_decision_is_the_same_call_as_decide():
|
||||
engine = FakeEngine()
|
||||
client = make_client(engine)
|
||||
client.post("/decide", json=ROW, headers=AUTH)
|
||||
client.post("/decide/shared", json={"state": ROW["state"], "decisions": [ROW]}, headers=AUTH)
|
||||
assert engine.calls[0] == engine.calls[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n, sizes", [(16, [16]), (17, [16, 1]), (40, [16, 16, 8])])
|
||||
def test_more_than_16_questions_are_split_greedily_in_request_order(n, sizes):
|
||||
engine = FakeEngine()
|
||||
out = make_client(engine).post("/decide/shared", json={"state": "s", "decisions": decisions(n)},
|
||||
headers=AUTH).json()
|
||||
assert [len(c["questions"]) for c in engine.calls] == sizes
|
||||
asked = [q["instructions"] for c in engine.calls for q in c["questions"].values()]
|
||||
assert asked == [f"Question {i}?" for i in range(n)]
|
||||
assert [r["id"] for r in out["results"]] == [f"decision-{i}" for i in range(n)]
|
||||
for i, result in enumerate(out["results"]):
|
||||
call = i // 16
|
||||
size = sizes[call]
|
||||
field = "q" if size == 1 else f"q{i % 16 + 1}"
|
||||
assert result["call"] == {"index": call, "field": field, "questions": size}
|
||||
assert result["prompt_sha256"] == request_sha(engine.calls[call])
|
||||
assert out["timing"]["questions_per_call"] == sizes
|
||||
assert out["timing"]["calls"] == len(sizes)
|
||||
|
||||
|
||||
def test_decision_ids_never_reach_the_model():
|
||||
engine = FakeEngine()
|
||||
make_client(engine).post("/decide/shared", json={"state": "s", "decisions": decisions(20)}, headers=AUTH)
|
||||
make_client(engine).post("/decide", json=ROW, headers=AUTH)
|
||||
assert "decision-" not in repr(engine.calls) and "r1" not in repr(engine.calls)
|
||||
|
||||
|
||||
from intern_decision_serve.errors import OutOfMemory, ScoringFailed # noqa: E402
|
||||
|
||||
|
||||
class RaisingEngine(FakeEngine):
|
||||
def __init__(self, exc, after=0):
|
||||
super().__init__()
|
||||
self.exc, self.after = exc, after
|
||||
|
||||
def predict(self, request):
|
||||
if len(self.calls) >= self.after:
|
||||
self.calls.append(request)
|
||||
raise self.exc
|
||||
return super().predict(request)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exc, status, code", [
|
||||
(ValueError("Example has 9000 tokens, above 8192; truncation is forbidden"), 422, "invalid_request"),
|
||||
(OutOfMemory("CUDA out of memory. Tried to allocate 2.00 GiB."), 503, "out_of_memory"),
|
||||
(ScoringFailed("RuntimeError: boom"), 500, "scoring_failed"),
|
||||
(ArithmeticError("Floating-point temperature scaling changed argmax"), 500, "scoring_failed"),
|
||||
])
|
||||
def test_engine_failures_map_to_the_contract_codes_on_both_endpoints(exc, status, code):
|
||||
for path, body in (("/decide", ROW), ("/decide/shared", {"state": "s", "decisions": decisions(2)})):
|
||||
response = make_client(RaisingEngine(exc)).post(path, json=body, headers=AUTH)
|
||||
assert response.status_code == status
|
||||
assert response.json()["error"]["code"] == code
|
||||
assert str(exc) in response.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_an_oom_in_a_later_chunk_fails_the_whole_request_with_503():
|
||||
engine = RaisingEngine(OutOfMemory("CUDA out of memory."), after=1)
|
||||
response = make_client(engine).post("/decide/shared", json={"state": "s", "decisions": decisions(20)},
|
||||
headers=AUTH)
|
||||
assert response.status_code == 503
|
||||
assert len(engine.calls) == 2
|
||||
|
||||
|
||||
class DroppingEngine(FakeEngine):
|
||||
"""Returns an answer that lacks one of the asked option ids."""
|
||||
def predict(self, request):
|
||||
response, sha = super().predict(request)
|
||||
for answer in response["answers"].values():
|
||||
answer["probabilities"].pop("no")
|
||||
return response, sha
|
||||
|
||||
|
||||
def test_an_answer_missing_an_option_id_is_a_500_not_a_guess():
|
||||
response = make_client(DroppingEngine()).post("/decide", json=ROW, headers=AUTH)
|
||||
assert response.status_code == 500
|
||||
assert response.json()["error"]["code"] == "scoring_failed"
|
||||
assert "no" in response.json()["error"]["message"]
|
||||
|
||||
|
||||
def opts(n):
|
||||
return [{"id": f"o{i}", "description": f"Option {i}"} for i in range(n)]
|
||||
|
||||
|
||||
BAD_DECIDE = [
|
||||
{**ROW, "options": opts(1)},
|
||||
{**ROW, "options": opts(17)},
|
||||
{**ROW, "options": [{"id": "a", "description": "x"}, {"id": "a", "description": "y"}]},
|
||||
{**ROW, "options": [{"id": "a"}, {"id": "b", "description": "y"}]},
|
||||
{**ROW, "options": [{"id": 1, "description": "x"}, {"id": "b", "description": "y"}]},
|
||||
{**ROW, "id": ""},
|
||||
{**ROW, "question": ""},
|
||||
{**ROW, "state": ""},
|
||||
{**ROW, "state": {}},
|
||||
{**ROW, "state": []},
|
||||
{**ROW, "state": 7},
|
||||
{**ROW, "workload": "triage"},
|
||||
{k: v for k, v in ROW.items() if k != "question"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", BAD_DECIDE)
|
||||
def test_semif_rule_violations_are_422_before_the_engine_runs(body):
|
||||
engine = FakeEngine()
|
||||
client = make_client(engine)
|
||||
response = client.post("/decide", json=body, headers=AUTH)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["error"]["code"] == "invalid_request"
|
||||
decision = {k: v for k, v in body.items() if k not in ("state", "workload")}
|
||||
shared = {"state": body.get("state", "s"), "decisions": [decision],
|
||||
**({"workload": body["workload"]} if "workload" in body else {})}
|
||||
assert client.post("/decide/shared", json=shared, headers=AUTH).status_code == 422
|
||||
assert engine.calls == []
|
||||
|
||||
|
||||
def test_a_non_finite_state_is_422():
|
||||
engine = FakeEngine()
|
||||
raw = '{"id": "r", "state": {"x": NaN}, "question": "q?", "options": [{"id": "a", "description": "A"}, {"id": "b", "description": "B"}]}'
|
||||
response = make_client(engine).post("/decide", content=raw, headers={**AUTH, "Content-Type": "application/json"})
|
||||
assert response.status_code == 422
|
||||
assert engine.calls == []
|
||||
|
||||
|
||||
def test_duplicate_decision_ids_are_422():
|
||||
body = {"state": "s", "decisions": [{**decisions(1)[0]}, {**decisions(1)[0]}]}
|
||||
assert make_client().post("/decide/shared", json=body, headers=AUTH).status_code == 422
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", ["{not json", "[]", '{"state": "s"}', '{"state": "s", "decisions": "x"}'])
|
||||
def test_malformed_bodies_are_422(raw):
|
||||
client = make_client()
|
||||
for path in ("/decide", "/decide/shared"):
|
||||
response = client.post(path, content=raw, headers={**AUTH, "Content-Type": "application/json"})
|
||||
assert response.status_code == 422
|
||||
assert response.json()["error"]["code"] == "invalid_request"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [0, 5])
|
||||
def test_a_row_count_outside_1_to_max_decisions_is_422(n):
|
||||
engine = FakeEngine()
|
||||
response = make_client(engine, max_decisions=4).post(
|
||||
"/decide/shared", json={"state": "s", "decisions": decisions(n)}, headers=AUTH)
|
||||
assert response.status_code == 422
|
||||
assert engine.calls == []
|
||||
|
||||
|
||||
import math # noqa: E402
|
||||
|
||||
|
||||
def position_bias(field, question):
|
||||
"""Pure position bias: the first-listed option gets +2, whatever it says."""
|
||||
return [2.0 if i == 0 else 0.0 for i in range(len(question["criteria"]))]
|
||||
|
||||
|
||||
def test_rotations_ask_each_ordering_in_its_own_wave_and_cancel_a_position_bias_exactly():
|
||||
engine = FakeEngine(scorer=position_bias)
|
||||
body = {**ROW, "options": opts(3), "orderings": "rotations"}
|
||||
out = make_client(engine).post("/decide", json=body, headers=AUTH).json()
|
||||
assert [list(c["questions"]) for c in engine.calls] == [["q"], ["q"], ["q"]]
|
||||
assert [list(c["questions"]["q"]["criteria"]) for c in engine.calls] == [
|
||||
["o0", "o1", "o2"], ["o1", "o2", "o0"], ["o2", "o0", "o1"]]
|
||||
assert out["id"] == "r1" and out["option_ids"] == ["o0", "o1", "o2"]
|
||||
c = out["combined"]
|
||||
assert (c["method"], c["orderings"]) == ("rotations", 3)
|
||||
assert c["probabilities"] == pytest.approx([1 / 3] * 3)
|
||||
assert c["top"] == "o0" # first maximum in the caller's order
|
||||
assert c["agreement"] == pytest.approx(1 / 3)
|
||||
assert [r["id"] for r in out["orderings"]] == ["r1#o0", "r1#o1", "r1#o2"]
|
||||
assert [r["option_ids"] for r in out["orderings"]] == [["o0", "o1", "o2"], ["o1", "o2", "o0"], ["o2", "o0", "o1"]]
|
||||
high, low = softmax_pair = (math.exp(2) / (math.exp(2) + 2), 1 / (math.exp(2) + 2))
|
||||
assert c["spread"] == {i: [pytest.approx(low), pytest.approx(high)] for i in ("o0", "o1", "o2")}
|
||||
del softmax_pair
|
||||
|
||||
|
||||
def test_all_asks_every_permutation_and_is_422_above_4_options():
|
||||
engine = FakeEngine()
|
||||
client = make_client(engine)
|
||||
out = client.post("/decide", json={**ROW, "options": opts(3), "orderings": "all"}, headers=AUTH).json()
|
||||
assert len(engine.calls) == 6 and out["combined"]["orderings"] == 6
|
||||
assert len({tuple(c["questions"]["q"]["criteria"]) for c in engine.calls}) == 6
|
||||
assert list(engine.calls[0]["questions"]["q"]["criteria"]) == ["o0", "o1", "o2"]
|
||||
response = client.post("/decide", json={**ROW, "options": opts(5), "orderings": "all"}, headers=AUTH)
|
||||
assert response.status_code == 422 and len(engine.calls) == 6
|
||||
|
||||
|
||||
def test_a_mixed_shared_request_runs_in_waves_and_wave_0_is_the_request_as_written():
|
||||
engine = FakeEngine()
|
||||
plain = {"state": "s", "decisions": [
|
||||
{"id": "a", "question": "A?", "options": OPTIONS},
|
||||
{"id": "b", "question": "B?", "options": opts(3)},
|
||||
{"id": "c", "question": "C?", "options": OPTIONS}]}
|
||||
client = make_client(engine)
|
||||
plain_out = client.post("/decide/shared", json=plain, headers=AUTH).json()
|
||||
plain_call = engine.calls.pop()
|
||||
mixed = {**plain, "decisions": [plain["decisions"][0], {**plain["decisions"][1], "orderings": "rotations"},
|
||||
plain["decisions"][2]]}
|
||||
out = client.post("/decide/shared", json=mixed, headers=AUTH).json()
|
||||
assert engine.calls[0] == plain_call # wave 0
|
||||
assert [list(c["questions"]) for c in engine.calls] == [["q1", "q2", "q3"], ["q"], ["q"]]
|
||||
assert [c["questions"]["q"]["instructions"] for c in engine.calls[1:]] == ["B?", "B?"]
|
||||
assert out["results"][0] == plain_out["results"][0] and out["results"][2] == plain_out["results"][2]
|
||||
assert out["results"][1]["combined"]["orderings"] == 3
|
||||
assert out["timing"]["batch_size"] == 5 and out["timing"]["calls"] == 3
|
||||
|
||||
|
||||
def test_waves_never_put_two_orderings_of_one_decision_in_one_call_and_are_packed_at_16():
|
||||
engine = FakeEngine()
|
||||
body = {"state": "s", "decisions": [{**d, "orderings": "rotations"} for d in decisions(17)]}
|
||||
out = make_client(engine).post("/decide/shared", json=body, headers=AUTH).json()
|
||||
assert [len(c["questions"]) for c in engine.calls] == [16, 1, 16, 1]
|
||||
for call in engine.calls:
|
||||
asked = [q["instructions"] for q in call["questions"].values()]
|
||||
assert len(asked) == len(set(asked))
|
||||
assert [r["id"] for r in out["results"]] == [f"decision-{i}" for i in range(17)]
|
||||
assert out["timing"]["batch_size"] == 34
|
||||
|
||||
|
||||
def test_orderings_count_toward_the_row_cap():
|
||||
engine = FakeEngine()
|
||||
body = {"state": "s", "decisions": [{**d, "orderings": "rotations"} for d in decisions(3)]}
|
||||
assert make_client(engine, max_decisions=5).post("/decide/shared", json=body, headers=AUTH).status_code == 422
|
||||
assert engine.calls == []
|
||||
|
||||
|
||||
def test_a_zero_probability_is_floored_before_the_log():
|
||||
class ZeroEngine(FakeEngine):
|
||||
def predict(self, request):
|
||||
response, sha = super().predict(request)
|
||||
for answer in response["answers"].values():
|
||||
first = next(iter(answer["probabilities"]))
|
||||
answer["probabilities"] = {k: (0.0 if k == first else 1.0 / (len(answer["probabilities"]) - 1))
|
||||
for k in answer["probabilities"]}
|
||||
return response, sha
|
||||
out = make_client(ZeroEngine()).post("/decide", json={**ROW, "orderings": "rotations"}, headers=AUTH)
|
||||
assert out.status_code == 200
|
||||
assert sum(out.json()["combined"]["probabilities"]) == pytest.approx(1.0)
|
||||
|
||||
|
||||
import threading # noqa: E402
|
||||
import time # noqa: E402
|
||||
from concurrent.futures import ThreadPoolExecutor # noqa: E402
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunked", [False, True])
|
||||
def test_a_body_over_the_limit_is_413_whether_or_not_it_declares_its_length(chunked):
|
||||
engine = FakeEngine()
|
||||
client = make_client(engine, max_body_bytes=200)
|
||||
body = ('{"id": "r1", "state": "' + "x" * 500 + '", "question": "Q?", "options": []}').encode()
|
||||
content = (chunk for chunk in [body[:100], body[100:]]) if chunked else body
|
||||
response = client.post("/decide", content=content, headers={**AUTH, "content-type": "application/json"})
|
||||
assert response.status_code == 413
|
||||
assert response.json()["error"]["code"] == "request_too_large"
|
||||
assert engine.calls == []
|
||||
|
||||
|
||||
def test_a_body_of_exactly_the_limit_is_accepted():
|
||||
body = json.dumps(ROW).encode()
|
||||
client = make_client(max_body_bytes=len(body))
|
||||
assert client.post("/decide", content=body, headers={**AUTH, "content-type": "application/json"}).status_code == 200
|
||||
|
||||
|
||||
def test_read_limited_stops_at_the_crossing_chunk_and_ignores_a_non_ascii_length():
|
||||
import asyncio
|
||||
from intern_decision_serve.app import ApiError, read_limited
|
||||
consumed = []
|
||||
|
||||
async def chunks():
|
||||
for i in range(10):
|
||||
consumed.append(i)
|
||||
yield b"x" * 100
|
||||
|
||||
with pytest.raises(ApiError) as info:
|
||||
asyncio.run(read_limited(chunks(), None, 250))
|
||||
assert info.value.status == 413 and consumed == [0, 1, 2]
|
||||
|
||||
async def small():
|
||||
yield b"{}"
|
||||
|
||||
assert asyncio.run(read_limited(small(), "²", 100)) == b"{}" # int("²") would raise
|
||||
|
||||
|
||||
class SlowEngine(FakeEngine):
|
||||
"""Holds each call until released; records the peak number of calls inside at once and the
|
||||
order in which requests' calls ran."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.inside = self.peak = 0
|
||||
self.guard = threading.Lock()
|
||||
self.release = threading.Event()
|
||||
self.entered = threading.Event()
|
||||
|
||||
def predict(self, request):
|
||||
with self.guard:
|
||||
self.inside += 1
|
||||
self.peak = max(self.peak, self.inside)
|
||||
self.entered.set()
|
||||
self.release.wait(5)
|
||||
with self.guard:
|
||||
self.inside -= 1
|
||||
return super().predict(request)
|
||||
|
||||
|
||||
def test_concurrent_requests_never_overlap_and_a_requests_chunks_are_not_interleaved():
|
||||
engine = SlowEngine()
|
||||
with make_client(engine) as client, ThreadPoolExecutor(4) as pool:
|
||||
futures = [pool.submit(client.post, "/decide/shared",
|
||||
json={"state": f"state {i}", "decisions": decisions(20)}, headers=AUTH)
|
||||
for i in range(3)]
|
||||
assert engine.entered.wait(5)
|
||||
started = time.monotonic()
|
||||
assert client.get("/health").status_code == 200
|
||||
assert time.monotonic() - started < 1.0 # answered while a call is held
|
||||
engine.release.set()
|
||||
assert [f.result().status_code for f in futures] == [200] * 3
|
||||
assert engine.peak == 1
|
||||
states = [c["state"] for c in engine.calls]
|
||||
assert all(states[k] == states[k + 1] for k in range(0, 6, 2)) # each request's 2 chunks back to back
|
||||
|
||||
|
||||
def test_more_than_max_queue_requests_in_progress_get_429_busy_before_the_body_is_read():
|
||||
engine = SlowEngine()
|
||||
with make_client(engine, max_queue=2) as client, ThreadPoolExecutor(3) as pool:
|
||||
held = [pool.submit(client.post, "/decide", json={**ROW, "id": f"r{i}"}, headers=AUTH) for i in range(2)]
|
||||
assert engine.entered.wait(5)
|
||||
time.sleep(0.2) # let the second request reach the queue
|
||||
extra = client.post("/decide", content=b"{not even json", headers={**AUTH, "content-type": "application/json"})
|
||||
assert extra.status_code == 429 and extra.json()["error"]["code"] == "busy"
|
||||
engine.release.set()
|
||||
assert [f.result().status_code for f in held] == [200, 200]
|
||||
assert make_client(FakeEngine(), max_queue=2).post("/decide", json=ROW, headers=AUTH).status_code == 200
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Settings.from_env: every value validated at startup, a bad one refused naming the variable.
|
||||
Contract: intern-decision-serve.contract.md § Configuration."""
|
||||
import pytest
|
||||
|
||||
from intern_decision_serve.config import DEFAULT_CHECKPOINT, Settings
|
||||
|
||||
TOKEN = "t" * 40
|
||||
P = "INTERN_DECISION_"
|
||||
|
||||
|
||||
def env(**kw):
|
||||
return {f"{P}API_TOKEN": TOKEN, **{f"{P}{k}": v for k, v in kw.items()}}
|
||||
|
||||
|
||||
def test_defaults():
|
||||
s = Settings.from_env(env())
|
||||
assert (s.api_token, s.checkpoint, s.device, s.vram_cap_gib) == (TOKEN, DEFAULT_CHECKPOINT, "cuda", None)
|
||||
assert (s.max_tokens, s.max_decisions, s.max_body_bytes, s.max_queue) == (8192, 64, 1024 * 1024, 32)
|
||||
assert (s.release_slack_mib, s.keep_vision) == (512, False)
|
||||
|
||||
|
||||
def test_every_value_is_read():
|
||||
s = Settings.from_env(env(CHECKPOINT="/x", DEVICE="cpu", VRAM_CAP_GIB="10.5", MAX_TOKENS="6000",
|
||||
MAX_DECISIONS="32", MAX_BODY_BYTES="2048", MAX_QUEUE="4", RELEASE_SLACK_MIB="0",
|
||||
KEEP_VISION="1"))
|
||||
assert (s.checkpoint, s.device, s.vram_cap_gib, s.max_tokens, s.max_decisions) == ("/x", "cpu", 10.5, 6000, 32)
|
||||
assert (s.max_body_bytes, s.max_queue, s.release_slack_mib, s.keep_vision) == (2048, 4, 0, True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token", ["", "short", "x" * 31, "x" * 40 + " ", "x" * 40 + "\n", "x" * 39 + "é"])
|
||||
def test_a_short_or_non_visible_ascii_token_is_refused(token):
|
||||
with pytest.raises(ValueError, match=f"{P}API_TOKEN"):
|
||||
Settings.from_env({f"{P}API_TOKEN": token})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name, value", [
|
||||
("VRAM_CAP_GIB", "0"), ("VRAM_CAP_GIB", "-1"), ("VRAM_CAP_GIB", "nan"), ("VRAM_CAP_GIB", "inf"),
|
||||
("VRAM_CAP_GIB", "lots"), ("MAX_TOKENS", "0"), ("MAX_TOKENS", "1.5"), ("MAX_DECISIONS", "0"),
|
||||
("MAX_BODY_BYTES", "x"), ("MAX_QUEUE", "-2"), ("RELEASE_SLACK_MIB", "-1"), ("KEEP_VISION", "yes"),
|
||||
("DEVICE", "mps"),
|
||||
])
|
||||
def test_a_bad_value_is_refused_naming_the_variable(name, value):
|
||||
with pytest.raises(ValueError, match=f"{P}{name}"):
|
||||
Settings.from_env(env(**{name: value}))
|
||||
|
||||
|
||||
def test_an_empty_cap_means_uncapped():
|
||||
assert Settings.from_env(env(VRAM_CAP_GIB="")).vram_cap_gib is None
|
||||
@@ -0,0 +1,321 @@
|
||||
"""TorchEngine against a fake torch and a fake checkpoint (no GPU, no model).
|
||||
Contract: intern-decision-serve.contract.md INV-3, INV-4, INV-7, INV-8."""
|
||||
import hashlib
|
||||
import logging
|
||||
import textwrap
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from intern_decision_serve.config import REVISION, Settings
|
||||
from intern_decision_serve.engine import TorchEngine
|
||||
from intern_decision_serve.errors import OutOfMemory, ScoringFailed
|
||||
|
||||
TOKEN = "t" * 40
|
||||
REQUEST = {"state": "s", "questions": {"q": {"type": "choice", "instructions": "Q?", "criteria": {"a": "A", "b": "B"}}}}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# A fake torch: records what was called, in order.
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
class FakeTorch:
|
||||
__version__ = "2.10.0+fake"
|
||||
events: list = []
|
||||
|
||||
class nn:
|
||||
class Module:
|
||||
def __call__(self, *a, **k):
|
||||
return self.forward(*a, **k)
|
||||
|
||||
class cuda:
|
||||
class OutOfMemoryError(RuntimeError):
|
||||
pass
|
||||
|
||||
available = True
|
||||
capability = (12, 0)
|
||||
arch_list = ["sm_90", "sm_120"]
|
||||
reserved = 0
|
||||
watched: list = []
|
||||
empties: list = []
|
||||
|
||||
@classmethod
|
||||
def is_available(cls):
|
||||
return cls.available
|
||||
|
||||
@classmethod
|
||||
def get_device_capability(cls, _i):
|
||||
return cls.capability
|
||||
|
||||
@classmethod
|
||||
def get_arch_list(cls):
|
||||
return cls.arch_list
|
||||
|
||||
@classmethod
|
||||
def get_device_properties(cls, _i):
|
||||
return type("P", (), {"total_memory": 96 * 2**30})
|
||||
|
||||
@classmethod
|
||||
def set_per_process_memory_fraction(cls, fraction, device):
|
||||
FakeTorch.events.append(("cap", round(fraction, 6), device))
|
||||
|
||||
@classmethod
|
||||
def memory_reserved(cls, _i):
|
||||
return cls.reserved
|
||||
|
||||
@classmethod
|
||||
def memory_allocated(cls, _i):
|
||||
return cls.reserved
|
||||
|
||||
@classmethod
|
||||
def empty_cache(cls):
|
||||
cls.empties.append(all(ref() is None for ref in cls.watched))
|
||||
FakeTorch.events.append(("empty_cache",))
|
||||
|
||||
@classmethod
|
||||
def get_device_name(cls, _i):
|
||||
return "Fake RTX"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_fake_torch():
|
||||
FakeTorch.events = []
|
||||
c = FakeTorch.cuda
|
||||
c.available, c.capability, c.arch_list, c.reserved = True, (12, 0), ["sm_90", "sm_120"], 0
|
||||
c.watched, c.empties = [], []
|
||||
yield
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# A fake checkpoint: snapshots/<REVISION>/inference.py defining a DecisionEngine shaped like the real one.
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
FAKE_INFERENCE = textwrap.dedent('''
|
||||
import json
|
||||
MODEL_NAME = "Intern-Decision-4B"
|
||||
EVENTS = []
|
||||
WARMUP_SHIFT = {"after_swap": 0.0}
|
||||
TOKEN_SKEW = {"n": 0}
|
||||
|
||||
class Compiled:
|
||||
def __init__(self, messages):
|
||||
self.messages = messages
|
||||
|
||||
def validate_request(request):
|
||||
return {"state": request["state"], "questions": request["questions"]}
|
||||
|
||||
def compile_row(row):
|
||||
return Compiled([{"role": "user", "content": json.dumps(row, sort_keys=True)}])
|
||||
|
||||
class Tokenizer:
|
||||
def apply_chat_template(self, messages, tokenize, add_generation_prompt, enable_thinking, add_vision_id):
|
||||
assert (tokenize, add_generation_prompt, enable_thinking, add_vision_id) == (False, False, False, True)
|
||||
return "<|im_start|>" + messages[0]["content"]
|
||||
|
||||
def __call__(self, text, add_special_tokens):
|
||||
return {"input_ids": list(range(len(text.split()) + TOKEN_SKEW["n"]))}
|
||||
|
||||
class Param:
|
||||
class device:
|
||||
type = "cuda"
|
||||
|
||||
class Inner:
|
||||
def __init__(self):
|
||||
self.visual = "the vision tower"
|
||||
|
||||
class Model:
|
||||
def __init__(self):
|
||||
self.model = Inner()
|
||||
|
||||
def parameters(self):
|
||||
yield Param()
|
||||
|
||||
class Backend:
|
||||
def __init__(self):
|
||||
self.tokenizer = Tokenizer()
|
||||
self.model = Model()
|
||||
|
||||
class DecisionEngine:
|
||||
def __init__(self, checkpoint=None, *, max_length=8192, device="cuda", **kw):
|
||||
EVENTS.append(("construct", checkpoint, max_length, device))
|
||||
self.backend = Backend()
|
||||
self.tokenizer = self.backend.tokenizer
|
||||
self.temperature = 1.99241824
|
||||
|
||||
def predict(self, request):
|
||||
row = validate_request(request)
|
||||
swapped = self.backend.model.model.visual != "the vision tower"
|
||||
p = 0.75 + (WARMUP_SHIFT["after_swap"] if swapped else 0.0)
|
||||
text = self.tokenizer.apply_chat_template(compile_row(row).messages, tokenize=False,
|
||||
add_generation_prompt=False, enable_thinking=False, add_vision_id=True)
|
||||
answers = {f: {"type": "choice", "probabilities": dict(zip(q["criteria"], [p, 1 - p])),
|
||||
"confidence": p, "choice": list(q["criteria"])[0], "source": "local",
|
||||
"decision": list(q["criteria"])[0]} for f, q in row["questions"].items()}
|
||||
return {"answers": answers, "usage": {"input_tokens": len(text.split())},
|
||||
"timing": {"inference_ms": 3.0}, "calibration": {"method": "temperature-scaling",
|
||||
"temperature": self.temperature}, "model": MODEL_NAME, "backend": "hf"}
|
||||
''')
|
||||
|
||||
|
||||
def fake_checkpoint(tmp_path, source=FAKE_INFERENCE, revision=REVISION):
|
||||
snap = tmp_path / "hub" / "models--internlm--Intern-Decision-4B" / "snapshots" / revision
|
||||
snap.mkdir(parents=True)
|
||||
(snap / "inference.py").write_text(source)
|
||||
return snap, hashlib.sha256(source.encode()).hexdigest()
|
||||
|
||||
|
||||
def load(tmp_path, source=FAKE_INFERENCE, revision=REVISION, sha=None, **settings):
|
||||
snap, real_sha = fake_checkpoint(tmp_path, source, revision)
|
||||
s = Settings(api_token=TOKEN, checkpoint=str(snap), **settings)
|
||||
return TorchEngine.load(s, torch=FakeTorch, inference_sha256=sha or real_sha)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Load (INV-3, INV-7, INV-8)
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
def test_load_serves_the_checkpoint_and_hashes_the_rendered_prompt(tmp_path):
|
||||
engine = load(tmp_path, vram_cap_gib=12.0, max_tokens=6000)
|
||||
response, sha = engine.predict(REQUEST)
|
||||
assert response["answers"]["q"]["probabilities"] == {"a": 0.75, "b": 0.25}
|
||||
text = "<|im_start|>" + __import__("json").dumps(REQUEST, sort_keys=True)
|
||||
assert sha == hashlib.sha256(text.encode()).hexdigest()
|
||||
assert engine.metadata["revision"] == REVISION and engine.metadata["vision_tower"] == "removed"
|
||||
assert engine.health()["device_name"] == "Fake RTX"
|
||||
|
||||
|
||||
def test_the_cap_is_applied_before_the_engine_is_constructed(tmp_path):
|
||||
engine = load(tmp_path, vram_cap_gib=12.0)
|
||||
inference = engine._inference
|
||||
assert FakeTorch.events[0] == ("cap", 0.125, 0)
|
||||
assert inference.EVENTS[0][0] == "construct" and inference.EVENTS[0][2:] == (8192, "cuda")
|
||||
|
||||
|
||||
def test_a_cap_larger_than_the_card_is_refused(tmp_path):
|
||||
with pytest.raises(ValueError, match="VRAM_CAP_GIB"):
|
||||
load(tmp_path, vram_cap_gib=200.0)
|
||||
|
||||
|
||||
def test_a_changed_inference_py_is_refused_before_anything_loads(tmp_path):
|
||||
with pytest.raises(RuntimeError, match="inference.py"):
|
||||
load(tmp_path, sha="0" * 64)
|
||||
assert FakeTorch.events == []
|
||||
|
||||
|
||||
def test_a_checkpoint_that_is_not_the_pinned_snapshot_is_refused(tmp_path):
|
||||
with pytest.raises(RuntimeError, match=REVISION):
|
||||
load(tmp_path, revision="f" * 40)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("problem", ["no_cuda", "no_arch"])
|
||||
def test_a_card_torch_cannot_drive_is_refused(tmp_path, problem):
|
||||
if problem == "no_cuda":
|
||||
FakeTorch.cuda.available = False
|
||||
else:
|
||||
FakeTorch.cuda.arch_list = ["sm_80", "sm_90"]
|
||||
with pytest.raises(RuntimeError):
|
||||
load(tmp_path)
|
||||
|
||||
|
||||
def test_the_vision_tower_is_replaced_by_a_stub_that_refuses_to_run(tmp_path):
|
||||
engine = load(tmp_path)
|
||||
stub = engine._engine.backend.model.model.visual
|
||||
assert stub != "the vision tower"
|
||||
with pytest.raises(RuntimeError, match="vision tower"):
|
||||
stub(None)
|
||||
|
||||
|
||||
def test_keep_vision_leaves_the_tower_alone(tmp_path):
|
||||
engine = load(tmp_path, keep_vision=True)
|
||||
assert engine._engine.backend.model.model.visual == "the vision tower"
|
||||
assert engine.metadata["vision_tower"] == "loaded"
|
||||
|
||||
|
||||
def test_startup_refuses_when_removing_the_vision_tower_changes_the_warm_up(tmp_path):
|
||||
source = FAKE_INFERENCE.replace('WARMUP_SHIFT = {"after_swap": 0.0}', 'WARMUP_SHIFT = {"after_swap": 0.001}')
|
||||
with pytest.raises(RuntimeError, match="INV-7"):
|
||||
load(tmp_path, source)
|
||||
|
||||
|
||||
def test_startup_refuses_a_prompt_hash_the_model_never_saw(tmp_path):
|
||||
source = FAKE_INFERENCE.replace('TOKEN_SKEW = {"n": 0}', 'TOKEN_SKEW = {"n": 1}')
|
||||
with pytest.raises(RuntimeError, match="INV-8"):
|
||||
load(tmp_path, source)
|
||||
|
||||
|
||||
def test_the_baseline_is_taken_after_warm_up_and_a_release(tmp_path):
|
||||
FakeTorch.cuda.reserved = 9 * 2**30
|
||||
engine = load(tmp_path, release_slack_mib=256)
|
||||
assert engine._release_above == 9 * 2**30 + 256 * 2**20
|
||||
assert ("empty_cache",) in FakeTorch.events
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# The guard around every call (INV-4), on an engine built directly
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
class Tensor:
|
||||
pass
|
||||
|
||||
|
||||
class StubEngine:
|
||||
def __init__(self, fn):
|
||||
self.predict = fn
|
||||
|
||||
|
||||
def direct(fn, release_above=None):
|
||||
import types
|
||||
inference = types.SimpleNamespace(validate_request=lambda r: r,
|
||||
compile_row=lambda row: types.SimpleNamespace(messages=[]))
|
||||
tokenizer = types.SimpleNamespace(apply_chat_template=lambda *a, **k: "prompt")
|
||||
return TorchEngine(FakeTorch, StubEngine(fn), inference, tokenizer, metadata={},
|
||||
settings=Settings(api_token=TOKEN), release_above_bytes=release_above)
|
||||
|
||||
|
||||
def failing(exc_factory):
|
||||
def predict(_request):
|
||||
kv = Tensor() # stands in for the failed forward's activations
|
||||
FakeTorch.cuda.watched.append(weakref.ref(kv))
|
||||
raise exc_factory()
|
||||
return predict
|
||||
|
||||
|
||||
def test_an_oom_frees_the_failed_call_before_emptying_the_cache_and_is_unchained():
|
||||
engine = direct(failing(lambda: FakeTorch.cuda.OutOfMemoryError("CUDA out of memory. Tried to allocate 2 GiB.\nmore")))
|
||||
with pytest.raises(OutOfMemory) as info:
|
||||
engine.predict(REQUEST)
|
||||
assert str(info.value) == "CUDA out of memory. Tried to allocate 2 GiB."
|
||||
assert info.value.__cause__ is None and info.value.__context__ is None
|
||||
assert FakeTorch.cuda.watched[0]() is None and FakeTorch.cuda.empties == [True]
|
||||
|
||||
|
||||
def test_an_empty_oom_message_still_reads_as_an_oom():
|
||||
with pytest.raises(OutOfMemory, match="CUDA out of memory"):
|
||||
direct(failing(lambda: FakeTorch.cuda.OutOfMemoryError(""))).predict(REQUEST)
|
||||
|
||||
|
||||
def test_a_runtime_error_saying_out_of_memory_is_an_oom():
|
||||
with pytest.raises(OutOfMemory):
|
||||
direct(failing(lambda: RuntimeError("CUBLAS_STATUS_ALLOC_FAILED: out of memory"))).predict(REQUEST)
|
||||
|
||||
|
||||
def test_any_other_failure_is_logged_released_and_raised_unchained_as_scoring_failed(caplog):
|
||||
engine = direct(failing(lambda: KeyError("q")))
|
||||
with caplog.at_level(logging.ERROR), pytest.raises(ScoringFailed) as info:
|
||||
engine.predict(REQUEST)
|
||||
assert "KeyError" in str(info.value) and info.value.__context__ is None
|
||||
assert "Traceback" in caplog.text
|
||||
assert FakeTorch.cuda.empties == [True]
|
||||
|
||||
|
||||
def test_value_errors_pass_through_untouched():
|
||||
def bad(_request):
|
||||
raise ValueError("Example has 9000 tokens, above 8192; truncation is forbidden")
|
||||
with pytest.raises(ValueError, match="truncation is forbidden"):
|
||||
direct(bad).predict(REQUEST)
|
||||
assert FakeTorch.cuda.empties == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reserved, released", [(2 * 2**30, True), (2**30, False), (2**30 - 1, False)])
|
||||
def test_a_burst_over_the_baseline_plus_slack_is_released(reserved, released):
|
||||
def ok(_request):
|
||||
FakeTorch.cuda.reserved = reserved
|
||||
return {"answers": {}}
|
||||
direct(ok, release_above=2**30).predict(REQUEST)
|
||||
assert (FakeTorch.cuda.empties != []) is released
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Entry point: INV-5, offline before torch/transformers load. Contract: intern-decision-serve.contract.md."""
|
||||
import os
|
||||
|
||||
from fake_engine import FakeEngine
|
||||
|
||||
|
||||
def test_app_from_env_goes_offline_before_the_engine_loads(monkeypatch):
|
||||
from intern_decision_serve import engine as engine_module
|
||||
from intern_decision_serve import main
|
||||
seen = {}
|
||||
|
||||
def fake_load(settings, **_kw):
|
||||
seen["offline"] = (os.environ.get("HF_HUB_OFFLINE"), os.environ.get("TRANSFORMERS_OFFLINE"))
|
||||
seen["token"] = settings.api_token
|
||||
return FakeEngine()
|
||||
|
||||
monkeypatch.delenv("HF_HUB_OFFLINE", raising=False)
|
||||
monkeypatch.delenv("TRANSFORMERS_OFFLINE", raising=False)
|
||||
monkeypatch.setenv("INTERN_DECISION_API_TOKEN", "k" * 40)
|
||||
monkeypatch.setattr(engine_module.TorchEngine, "load", staticmethod(fake_load))
|
||||
app = main.app_from_env()
|
||||
assert seen == {"offline": ("1", "1"), "token": "k" * 40}
|
||||
assert app.title == "intern-decision-serve"
|
||||
Reference in New Issue
Block a user