fix(intern-decision-serve): load and score on one dedicated inference thread

torch keeps CUDA state per host thread (cuBLAS handles and workspaces), partly outside the
per-process VRAM cap. Scoring on anyio's threadpool let 40 threads each create it: measured on
fv-ml1 GPU 3, +252 MiB outside the cap and +326 MiB inside, which pushed the process past the
10,300 MiB GPU 1 budget. Load, warm-up and every call now run on the same single thread.
This commit is contained in:
vh
2026-09-30 09:18:46 -07:00
parent f7415db5c9
commit f21369e4ac
5 changed files with 87 additions and 11 deletions
@@ -21,3 +21,28 @@ def test_app_from_env_goes_offline_before_the_engine_loads(monkeypatch):
app = main.app_from_env()
assert seen == {"offline": ("1", "1"), "token": "k" * 40}
assert app.title == "intern-decision-serve"
def test_the_engine_loads_on_the_same_thread_every_call_later_runs_on(monkeypatch):
import threading
from intern_decision_serve import engine as engine_module
from intern_decision_serve import main
from fastapi.testclient import TestClient
seen = {}
class Recording(FakeEngine):
def predict(self, request):
seen.setdefault("calls", set()).add(threading.get_ident())
return super().predict(request)
def fake_load(settings, **_kw):
seen["load"] = threading.get_ident()
return Recording()
monkeypatch.setenv("INTERN_DECISION_API_TOKEN", "k" * 40)
monkeypatch.setattr(engine_module.TorchEngine, "load", staticmethod(fake_load))
client = TestClient(main.app_from_env())
body = {"id": "r", "state": "s", "question": "q?", "options": [{"id": "a", "description": "A"},
{"id": "b", "description": "B"}]}
assert client.post("/decide", json=body, headers={"Authorization": "Bearer " + "k" * 40}).status_code == 200
assert seen["calls"] == {seen["load"]} and seen["load"] != threading.get_ident()