fix(intern-decision-serve): code-review fixes

- a failure while building the response (a non-finite number included) is a 500 inside the
  envelope, never a 422 or a render crash outside it
- an engine ValueError keeps its message but is released and raised unchained, like an OOM
- the prompt is built (and the model's own validation run) before the forward
- the row cap is counted before any ordering is built
- /health reads a device name cached at load, so it makes no driver call off the inference thread
This commit is contained in:
vh
2026-09-30 09:30:40 -07:00
parent 618390c5fa
commit 21d16d7ad8
5 changed files with 110 additions and 21 deletions
@@ -464,3 +464,28 @@ def test_the_app_uses_the_executor_it_is_given():
assert client.post("/decide", json=ROW, headers=AUTH).status_code == 200
assert engine.threads == {home}
executor.shutdown()
class InfiniteEngine(FakeEngine):
"""A server-side defect: a probability of +inf, which breaks the averaging arithmetic."""
def predict(self, request):
response, sha = super().predict(request)
for answer in response["answers"].values():
first = next(iter(answer["probabilities"]))
answer["probabilities"][first] = float("inf")
return response, sha
def test_a_failure_while_building_the_response_is_500_never_a_422():
response = make_client(InfiniteEngine()).post("/decide", json={**ROW, "orderings": "rotations"}, headers=AUTH)
assert response.status_code == 500
assert response.json()["error"]["code"] == "scoring_failed"
def test_a_huge_orderings_request_is_refused_with_its_true_row_count():
engine = FakeEngine()
body = {"state": "s", "decisions": [{**d, "orderings": "rotations"} for d in decisions(3000, opts(16))]}
response = make_client(engine, max_body_bytes=16 * 2**20).post("/decide/shared", json=body, headers=AUTH)
assert response.status_code == 422
assert "48000 rows" in response.json()["error"]["message"]
assert engine.calls == []
@@ -306,12 +306,35 @@ def test_any_other_failure_is_logged_released_and_raised_unchained_as_scoring_fa
assert FakeTorch.cuda.empties == [True]
def test_value_errors_pass_through_untouched():
def bad(_request):
raise ValueError("Example has 9000 tokens, above 8192; truncation is forbidden")
with pytest.raises(ValueError, match="truncation is forbidden"):
direct(bad).predict(REQUEST)
assert FakeTorch.cuda.empties == []
def test_a_value_error_keeps_its_message_but_is_released_and_raised_unchained():
"""A ValueError can come from inside the forward too (a transformers shape check): it must not pin
the failed forward's tensors through a chained traceback (review 2026-09-30)."""
engine = direct(failing(lambda: ValueError("Example has 9000 tokens, above 8192; truncation is forbidden")))
with pytest.raises(ValueError, match="truncation is forbidden") as info:
engine.predict(REQUEST)
assert info.value.__cause__ is None and info.value.__context__ is None
assert FakeTorch.cuda.watched[0]() is None and FakeTorch.cuda.empties == [True]
def test_a_request_the_prompt_builder_rejects_never_reaches_the_model():
import types
called = []
inference = types.SimpleNamespace(validate_request=lambda r: (_ for _ in ()).throw(ValueError("Supply 1-16 questions.")),
compile_row=lambda row: None)
engine = TorchEngine(FakeTorch, StubEngine(lambda r: called.append(r)), inference, tokenizer=None, metadata={},
settings=Settings(api_token=TOKEN))
with pytest.raises(ValueError, match="1-16 questions"):
engine.predict(REQUEST)
assert called == []
def test_health_does_not_ask_the_driver_for_the_device_name_again(tmp_path):
engine = load(tmp_path)
FakeTorch.cuda.get_device_name = classmethod(lambda cls, _i: (_ for _ in ()).throw(AssertionError("driver call")))
try:
assert engine.health()["device_name"] == "Fake RTX"
finally:
FakeTorch.cuda.get_device_name = classmethod(lambda cls, _i: "Fake RTX")
@pytest.mark.parametrize("reserved, released", [(2 * 2**30, True), (2**30, False), (2**30 - 1, False)])