feat(intern-decision): persist the triton autotune cache across recreates
The ~6.5-9 s first-call-per-bucket autotune lived in the container's writable layer and died on every recreate. 0.1.3 creates /tmp/triton-cache in the image owned by 10001 so the named volume intern-decision_triton-cache inherits a writable mount point, and compose mounts it. Bucket model PROVEN, not inferred: 2,048-token buckets, 16 up to 32,768. After one warmed call per bucket, 12 random sizes across 8k-32k were all warm (worst 2.09 s); cold entries cost 6.5-9 s. Full cold warm-up 109 s; warm re-run 17 s. scripts/intern-decision-warmup: one noul call per bucket, MAX_TOKENS from /health, two-point live calibration of the tokenizer's linear token model (a single probe overcorrects and the aim oscillates around the bucket edge), per-bucket wall times, non-zero exit on a missed bucket. Run it after an IMAGE CHANGE only; the volume carries ordinary recreates (measured: force-recreate, then a warmed 32k call answered in 2.11 s). Acceptance on 0.1.3: JevBench 202/231, hard 83/111, 0 diffs / 924; warm 32k GPU 1 peak 15,218 MiB (budget 15,220; a COLD autotune touched 15,224 once, README caveat); /decide answers. Artifacts in the acceptance dir.
This commit is contained in:
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Warm every length bucket of intern-decision's kernel autotune cache.
|
||||
|
||||
The fast kernels (triton + fla) autotune per shape BUCKET: the first call that lands in a new
|
||||
bucket costs seconds of autotune. The cache lives in the container's `triton-cache` volume, so
|
||||
it survives ordinary recreates — this script is for AFTER AN IMAGE CHANGE (a new triton/fla
|
||||
version keys new entries). Idempotent: a second run is all fast.
|
||||
|
||||
One noul /v1/systemone call per bucket, smallest up to the service's own MAX_TOKENS (read from
|
||||
/health; never hard-coded). Aims just under each bucket's top edge; the aim is calibrated from
|
||||
the service's own reported input_tokens after every call, so drift cannot leave a bucket cold.
|
||||
Prints each call's bucket, the model's own token count, and the wall time. Exits non-zero if
|
||||
any call fails or lands outside its bucket.
|
||||
|
||||
scripts/intern-decision-warmup [--endpoint URL] [--bucket-width N]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
UNIT = "the quick brown fox. " # ~5.13 tokens against this service (measured 2026-09-30)
|
||||
|
||||
|
||||
def post(endpoint: str, token: str, body: dict, timeout: float = 180.0):
|
||||
req = urllib.request.Request(
|
||||
endpoint.rstrip("/") + "/v1/systemone",
|
||||
data=json.dumps(body).encode(), method="POST",
|
||||
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"})
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.status, json.loads(r.read()), time.perf_counter() - t0
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, json.loads(e.read() or b"{}"), time.perf_counter() - t0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--endpoint", default="http://intern-decision.fv.internal:8033")
|
||||
ap.add_argument("--bucket-width", type=int, default=2048,
|
||||
help="asserted bucket width; the acceptance run proves or corrects it")
|
||||
args = ap.parse_args()
|
||||
|
||||
token = subprocess.run(["secret", "get", "intern-decision/api-token"],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
if not token:
|
||||
print("no token: secret get intern-decision/api-token", file=sys.stderr)
|
||||
return 2
|
||||
health = json.load(urllib.request.urlopen(args.endpoint.rstrip("/") + "/health", timeout=15))
|
||||
max_tokens = int(health["max_tokens"])
|
||||
width = args.bucket_width
|
||||
buckets = list(range(width, max_tokens + 1, width))
|
||||
print(f"endpoint={args.endpoint} max_tokens={max_tokens} bucket_width={width} "
|
||||
f"buckets={len(buckets)}")
|
||||
|
||||
def ask(reps: int):
|
||||
return post(args.endpoint, token, {"state": UNIT * reps,
|
||||
"questions": {"warm": {"type": "noul",
|
||||
"instructions": "Is the system healthy?"}}})
|
||||
|
||||
# Two-point calibration of the tokenizer's linear model, input_tokens = a*reps + b. A single
|
||||
# probe cannot separate a from b and its "calibration" overcorrects on the next aim (measured:
|
||||
# the aim oscillates ±500 tokens around the bucket edge). Two probes 900 reps apart fit exactly.
|
||||
s1, o1, _ = ask(100)
|
||||
s2, o2, _ = ask(1000)
|
||||
if s1 != 200 or s2 != 200:
|
||||
print("calibration failed", s1, s2, file=sys.stderr)
|
||||
return 2
|
||||
t1 = o1["usage"]["input_tokens"]; t2 = o2["usage"]["input_tokens"]
|
||||
a = (t2 - t1) / 900.0
|
||||
b = t1 - a * 100
|
||||
print(f"calibrated: input_tokens = {a:.4f}*reps + {b:.1f}")
|
||||
|
||||
failed, total = False, 0.0
|
||||
print(f"{'bucket<=':>9} {'tokens':>7} {'wall_s':>7} note")
|
||||
for hi in buckets:
|
||||
note = ""
|
||||
got = status = 0
|
||||
wall = " n/a"
|
||||
for attempt in (1, 2): # a second aim fixes a rounding miss; never miss a bucket
|
||||
reps = max(1, round((hi - 60 - b) / a))
|
||||
status, out, w = ask(reps)
|
||||
total += w
|
||||
wall = f"{w:.2f}"
|
||||
got = (out.get("usage") or {}).get("input_tokens", 0)
|
||||
if status != 200 or hi - width < got <= hi:
|
||||
break
|
||||
if status != 200:
|
||||
note = f"FAILED {status}: {(out.get('error') or {}).get('message', '')[:60]}"
|
||||
failed = True
|
||||
elif not (hi - width < got <= hi):
|
||||
note = f"OUT OF BUCKET ({hi-width} < {got} <= {hi})"
|
||||
failed = True
|
||||
print(f"{hi:>9} {got:>7} {wall:>7} {note}")
|
||||
print(f"total {total:.1f} s over {len(buckets)} buckets")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user