feat(semif): 0.1.3 — order averaging, fast kernels, bug-hunt hardening (Prime)

Order averaging (Prime, after the 739aa03 spike): - A decision may set orderings: rotations|all (all only for <= 4 options). Every ordering goes to the engine in one shared batch. - The reply keeps each native result and adds combined {probabilities (log-mean), top, agreement, spread}. - Through the service on SemIf's labelled sets (252 rows): 78.6% -> 88.1% (group-bootstrap 95% CI +5.1..+14.3). Unanimous agreement is 94.5% accurate. Fast kernels: flash-linear-attention 0.5.2 and causal-conv1d 1.7.0 are now the default build. A/B on the empty GPU 3: - parity with upstream went from 142/144 to 144/144; - a ~2k-token /decide went from 169 to 92 ms server-side; - short 3-rotation batches cost ~3-6 ms more. triton builds a C shim at runtime, so the image carries gcc. Without it the warm-up failed and startup failed closed. Heid bug-hunt panel (4/4 arms, thread 01M3H3F4RR7XBP90KQ3A39H4SX), folded: - Startup validation: VRAM cap 0 no longer means uncapped (C1); limits must be >= 1 (S1); the token must be visible ASCII (S2); the calibration file must exist and parse, with T in [0.05, 20] (S8, and C3's NaN leg). - The body limit is checked before a chunk is kept, and a Unicode-digit Content-Length no longer crashes (C2, S3). - Failures while building the response now get the 500 envelope (C3). - 429 busy past SEMIF_MAX_QUEUE requests in progress (C6). - The engine releases memory on every non-validation failure, unchained after gc; an empty OOM message is handled; 'out of memory' RuntimeErrors map to 503 (C4, C5, S9). - The entry point forces HF_HUB_OFFLINE (S10). README wording fixed (S5, S6). - New guard tests close the gaps the arms' mutation grids exposed: early stop of the body read, a shared-route lock, calibration pass-through, the gc cycle, the exact caps, TorchEngine.load's arch and device checks, and the offline entry point. 86 tests. Deployed on fv-ml1 GPU 1: parity 144/144, OOM and burst release verified, shared capacity 63/51/26/16 rows at ~140/520/1960/3900 prefix tokens.
2026-09-27 03:27:15 -07:00
parent d7ad235365
commit 77b8cb449c
22 changed files with 1018 additions and 92 deletions
@@ -2,9 +2,10 @@
 from __future__ import annotations

 import hmac
+import itertools
 import math
 import threading
-from typing import Any
+from typing import Any, Literal

 from fastapi import FastAPI, Request
 from fastapi.concurrency import run_in_threadpool
@@ -29,6 +30,7 @@ class Decision(BaseModel):
    id: str
    question: str
    options: list[Option]
+    orderings: Literal["none", "rotations", "all"] = "none"

    def row(self, state: State) -> dict:
        """The SemIf row shape: exactly id, state, question, options."""
@@ -64,6 +66,75 @@ def _first_error(exc: ValidationError) -> str:
    return f"{where}: {first.get('msg', 'invalid')}"


+MAX_OPTIONS_FOR_ALL = 4
+
+
+def ordering_perms(decision: Decision) -> list[tuple[int, ...]]:
+    """Index permutations of the caller's options, the caller's own order first."""
+    n = len(decision.options)
+    if decision.orderings == "rotations":
+        return [tuple((start + k) % n for k in range(n)) for start in range(n)]
+    if n > MAX_OPTIONS_FOR_ALL:
+        raise ApiError(422, "invalid_request",
+                       f"orderings 'all' allows at most {MAX_OPTIONS_FOR_ALL} options ({n} given); use 'rotations'")
+    return list(itertools.permutations(range(n)))
+
+
+def expanded_rows(decision: Decision, state: State, perms: list[tuple[int, ...]]) -> list[dict]:
+    base = decision.row(state)
+    return [{**base, "id": f"{decision.id}#o{k}", "options": [base["options"][i] for i in perm]}
+            for k, perm in enumerate(perms)]
+
+
+def combine(decision: Decision, results: list[dict]) -> dict:
+    """Average per-ordering log-softmax by option id; the native results ride along unchanged."""
+    option_ids = [o.id for o in decision.options]
+    logp: dict[str, list[float]] = {i: [] for i in option_ids}
+    probs: dict[str, list[float]] = {i: [] for i in option_ids}
+    tops = []
+    for result in results:
+        logits = result["option_logits"]
+        top = max(logits)
+        lse = top + math.log(sum(math.exp(x - top) for x in logits))
+        for oid, x, p in zip(result["option_ids"], logits, result["probabilities"]):
+            logp[oid].append(x - lse)
+            probs[oid].append(p)
+        tops.append(result["option_ids"][logits.index(top)])
+    means = [sum(logp[i]) / len(logp[i]) for i in option_ids]
+    peak = max(means)
+    weights = [math.exp(m - peak) for m in means]
+    combined_p = [w / sum(weights) for w in weights]
+    winner = option_ids[combined_p.index(max(combined_p))]
+    return {
+        "id": decision.id,
+        "option_ids": option_ids,
+        "combined": {
+            "method": decision.orderings,
+            "orderings": len(results),
+            "probabilities": combined_p,
+            "top": winner,
+            "agreement": tops.count(winner) / len(tops),
+            "spread": {i: [min(probs[i]), max(probs[i])] for i in option_ids},
+        },
+        "orderings": results,
+    }
+
+
+async def read_limited(stream, declared: str | None, limit: int) -> bytes:
+    """Read a request body, refusing it once it would exceed `limit` bytes. The check runs BEFORE a
+    chunk is kept, and nothing after the crossing chunk is read (bug hunt C2). A declared length is
+    trusted only as ASCII digits: `"²".isdigit()` is True but `int("²")` raises (S3)."""
+    too_large = ApiError(413, "request_too_large", f"request body exceeds {limit} bytes")
+    if declared is not None and declared.isascii() and declared.isdigit() and int(declared) > limit:
+        raise too_large
+    body = bytearray()
+    async for chunk in stream:
+        if len(body) + len(chunk) > limit:
+            raise too_large
+        body.extend(chunk)
+    return bytes(body)
+
+
 def calibrated_view(result: dict, workload: str, temperature: float) -> dict:
    """softmax(option_logits / T): the native fields are left exactly as SemIf returned them (INV-1)."""
    scaled = [x / temperature for x in result["option_logits"]]
@@ -77,6 +148,27 @@ def create_app(settings: Settings, engine: Any) -> FastAPI:
    app = FastAPI(title="semif-serve")
    expected = f"Bearer {settings.api_token}".encode()
    inference = threading.Lock()   # INV-2: one scorer call at a time, off the event loop
+    in_progress = 0                 # POSTs admitted and not yet answered (bug hunt C6)
+
+    def admit():
+        nonlocal in_progress
+        if in_progress >= settings.max_queue:
+            raise ApiError(429, "busy", f"{in_progress} requests already in progress (limit {settings.max_queue})")
+        in_progress += 1
+
+    def leave():
+        nonlocal in_progress
+        in_progress -= 1
+
+    def build(fn, *args):
+        """Response construction from a scorer result (calibration, averaging) maps its failures to
+        the 500 envelope too, instead of escaping as a bare 500 (bug hunt C3)."""
+        try:
+            return fn(*args)
+        except ApiError:
+            raise
+        except Exception as exc:  # noqa: BLE001
+            raise ApiError(500, "scoring_failed", f"building the response failed: {type(exc).__name__}: {exc}") from exc

    def locked(fn, *args):
        with inference:
@@ -94,22 +186,10 @@ def create_app(settings: Settings, engine: Any) -> FastAPI:
    async def api_error(_request: Request, exc: ApiError):
        return error(exc.status, exc.code, exc.message)

-    async def read_limited(request: Request) -> bytes:
-        limit = settings.max_body_bytes
-        too_large = ApiError(413, "request_too_large", f"request body exceeds {limit} bytes")
-        declared = request.headers.get("content-length")
-        if declared is not None and declared.isdigit() and int(declared) > limit:
-            raise too_large
-        body = bytearray()
-        async for chunk in request.stream():   # also caps bodies that declare no length
-            body.extend(chunk)
-            if len(body) > limit:
-                raise too_large
-        return bytes(body)
-
    async def parse(request: Request, model: type[BaseModel]):
        try:
-            return model.model_validate_json(await read_limited(request))
+            return model.model_validate_json(
+                await read_limited(request.stream(), request.headers.get("content-length"), settings.max_body_bytes))
        except ValidationError as exc:
            raise ApiError(422, "invalid_request", _first_error(exc)) from exc

@@ -142,20 +222,59 @@ def create_app(settings: Settings, engine: Any) -> FastAPI:
                "vram_cap_gib": settings.vram_cap_gib, "max_tokens": settings.max_tokens,
                "max_decisions": settings.max_decisions, "workloads": sorted(settings.calibration)}

+    async def score_batch(decisions: list[Decision], state: State, workload: str | None) -> tuple[list[dict], dict]:
+        """One engine.shared call for every row of every decision; results in request order."""
+        plan = []            # (decision, perms or None, row count)
+        rows: list[dict] = []
+        for d in decisions:
+            if d.orderings == "none":
+                plan.append((d, None, 1))
+                rows.append(d.row(state))
+            else:
+                if workload is not None:
+                    raise ApiError(422, "invalid_request",
+                                   "workload calibration is not available together with orderings")
+                perms = ordering_perms(d)
+                plan.append((d, perms, len(perms)))
+                rows.extend(expanded_rows(d, state, perms))
+        if not 1 <= len(rows) <= settings.max_decisions:
+            raise ApiError(422, "invalid_request",
+                           f"this request expands to {len(rows)} scored rows; the limit is 1..{settings.max_decisions}")
+        temperature = temperature_for(workload)
+        results, timing = await score(engine.shared, rows)
+        out, cursor = [], 0
+        for d, perms, count in plan:
+            chunk = results[cursor:cursor + count]
+            cursor += count
+            out.append(build(with_calibration, chunk[0], workload, temperature) if perms is None
+                       else build(combine, d, chunk))
+        return out, timing
+
    @app.post("/decide")
    async def decide(request: Request):
-        body = await parse(request, DecideBody)
-        temperature = temperature_for(body.workload)
-        return with_calibration(await score(engine.direct, body.row(body.state)), body.workload, temperature)
+        admit()
+        try:
+            body = await parse(request, DecideBody)
+            if body.orderings != "none":
+                results, _timing = await score_batch([body], body.state, body.workload)
+                return results[0]
+            temperature = temperature_for(body.workload)
+            result = await score(engine.direct, body.row(body.state))
+            return build(with_calibration, result, body.workload, temperature)
+        finally:
+            leave()

    @app.post("/decide/shared")
    async def decide_shared(request: Request):
-        body = await parse(request, SharedBody)
-        if not 1 <= len(body.decisions) <= settings.max_decisions:
-            raise ApiError(422, "invalid_request",
-                           f"decisions must hold 1..{settings.max_decisions} entries, got {len(body.decisions)}")
-        temperature = temperature_for(body.workload)
-        results, timing = await score(engine.shared, [d.row(body.state) for d in body.decisions])
-        return {"results": [with_calibration(r, body.workload, temperature) for r in results], "timing": timing}
+        admit()
+        try:
+            body = await parse(request, SharedBody)
+            if not 1 <= len(body.decisions) <= settings.max_decisions:
+                raise ApiError(422, "invalid_request",
+                               f"decisions must hold 1..{settings.max_decisions} entries, got {len(body.decisions)}")
+            results, timing = await score_batch(body.decisions, body.state, body.workload)
+            return {"results": results, "timing": timing}
+        finally:
+            leave()

    return app