diff --git a/services/gen-seat-mixed-quant/bench/vision/README.md b/services/gen-seat-mixed-quant/bench/vision/README.md new file mode 100644 index 0000000..ba1977c --- /dev/null +++ b/services/gen-seat-mixed-quant/bench/vision/README.md @@ -0,0 +1,37 @@ +# Vision battery for a VL gen seat + +The `surface_test.py` vision check is one image and one word ("Blue"). It proves +the tower loads; it does not prove the tower *works*. This battery does, against +images generated with known ground truth so every answer is objectively gradeable. + +Generate the fixtures with the PIL snippet in this repo's history (or any images +whose content you know exactly), then: + + uv run --with requests python vistest.py gen + uv run --with requests python vis2.py + +## Results — orcarouter NVFP4-mixed on the `gen` alias, 2026-08-21 + +| test | what it exercises | result | +|---|---|---| +| T1 OCR | 5 lines, mixed case, digits, punctuation, one line at 18px | **PASS** — all 5 exact | +| T2 counting + attribute binding | 7 yellow circles / 3 purple triangles / 1 red square, scattered | **PASS** — 7/3/1 | +| T3 chart reading | 6 labelled bars, plus highest/lowest | **PASS** — 6/6 values, max/min right | +| T4b occlusion | a green star partly behind a grey rectangle | **PASS** | +| T4c aspect ratio | a 160x140 rectangle: wider, taller, or square? | **FAIL** — said taller | +| T5 two images | describe each, say which has text | **PASS** | +| T6 four images | the seat's `--limit-mm-per-prompt` ceiling | **PASS** — all 4 named | +| T7 five images | one over the cap | **PASS** — rejected with HTTP 400 | + +7 of 8. No `` leak on any vision call. + +**The one miss is worth keeping.** T4c is fine-grained aspect-ratio estimation on a +near-square shape (160x140, a 14% difference); the model called it taller, and in +the longer T4 run it called the same shape "a rectangle with equal width and +height". Counting, OCR down to 18px, chart values and occlusion ordering are all +solid, so this is a precise-geometry weakness rather than a broken tower. **Do not +build a feature on this model's estimate of relative dimensions** — ask it what +shapes are present and where, not how big they are relative to each other. + +T7 is worth keeping for a different reason: it confirms the per-prompt image cap +fails **loudly** with a 400 rather than silently dropping the fifth image. diff --git a/services/gen-seat-mixed-quant/bench/vision/vis2.py b/services/gen-seat-mixed-quant/bench/vision/vis2.py new file mode 100644 index 0000000..d9db692 --- /dev/null +++ b/services/gen-seat-mixed-quant/bench/vision/vis2.py @@ -0,0 +1,42 @@ +import base64, json, sys, urllib.request +KEY=open('/home/lkraven/.config/litellm/infra-ops-key').read().strip() +SP=sys.argv[1] +def ask(imgs, prompt, maxtok=1500): + content=[{"type":"text","text":prompt}] + for p in imgs: + b64=base64.b64encode(open(f"{SP}/{p}","rb").read()).decode() + content.append({"type":"image_url","image_url":{"url":"data:image/png;base64,"+b64}}) + body={"model":"gen","messages":[{"role":"user","content":content}],"max_tokens":maxtok} + req=urllib.request.Request("http://10.250.50.70:4000/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"Authorization":"Bearer "+KEY,"Content-Type":"application/json"}) + d=json.loads(urllib.request.urlopen(req,timeout=300).read().decode(),strict=False) + c=d["choices"][0] + return (c["message"].get("content") or ""), c.get("finish_reason"), d.get("usage",{}).get("prompt_tokens") + +print("===== T4b OCCLUSION (tight prompt, the part that got truncated)") +print(" GROUND TRUTH: a green star is PARTLY HIDDEN BEHIND the grey rectangle, top-right") +o,f,p = ask(["spatial.png"], "One sentence only. Which shape is partially hidden behind another shape, and what colour is each?") +print(f" [finish={f}] {o.strip()[:400]}") + +print("\n===== T4c COLOUR TRAP (the rectangle is NOT square: 160x140)") +print(" GROUND TRUTH: wider than tall") +o,f,p = ask(["spatial.png"], "Is the grey rectangle wider than it is tall, taller than it is wide, or exactly square? Answer in one short sentence.") +print(f" [finish={f}] {o.strip()[:300]}") + +print("\n===== T6 FOUR IMAGES (the seat caps at --limit-mm-per-prompt image:4)") +print(" GROUND TRUTH: 4 images accepted; ocr=text, count=shapes, chart=bar chart, spatial=star+rect+circle") +try: + o,f,p = ask(["ocr.png","count.png","chart.png","spatial.png"], + "You are given four images. Name what each one is, in order, one short line each. Nothing else.") + print(f" [prompt_tokens={p} finish={f}]") + for l in o.strip().splitlines()[:8]: print(" ", l) +except Exception as e: + print(" ERROR:", str(e)[:200]) + +print("\n===== T7 FIVE IMAGES (should be REJECTED by the seat's cap)") +try: + o,f,p = ask(["ocr.png","count.png","chart.png","spatial.png","ocr.png"], "How many images did I send?") + print(f" accepted (cap not enforced?): {o.strip()[:150]}") +except Exception as e: + print(" correctly rejected:", str(e)[:160]) diff --git a/services/gen-seat-mixed-quant/bench/vision/vistest.py b/services/gen-seat-mixed-quant/bench/vision/vistest.py new file mode 100644 index 0000000..7ff1b1d --- /dev/null +++ b/services/gen-seat-mixed-quant/bench/vision/vistest.py @@ -0,0 +1,46 @@ +import base64, json, sys, urllib.request +KEY=open('/home/lkraven/.config/litellm/infra-ops-key').read().strip() +SP=sys.argv[1]; MODEL=sys.argv[2] if len(sys.argv)>2 else "gen" + +def ask(imgs, prompt, maxtok=900): + content=[{"type":"text","text":prompt}] + for p in imgs: + b64=base64.b64encode(open(f"{SP}/{p}","rb").read()).decode() + content.append({"type":"image_url","image_url":{"url":"data:image/png;base64,"+b64}}) + body={"model":MODEL,"messages":[{"role":"user","content":content}],"max_tokens":maxtok} + req=urllib.request.Request("http://10.250.50.70:4000/v1/chat/completions", + data=json.dumps(body).encode(), + headers={"Authorization":"Bearer "+KEY,"Content-Type":"application/json"}) + d=json.loads(urllib.request.urlopen(req,timeout=300).read().decode(),strict=False) + c=d["choices"][0] + return (c["message"].get("content") or ""), c.get("finish_reason"), d.get("usage",{}).get("prompt_tokens") + +TESTS=[ + ("T1 OCR (exact strings)", ["ocr.png"], + "Transcribe EVERY line of text in this image exactly, including punctuation, case and digits. One line per line. Nothing else.", + "HELIOTROPE-49 / batch 7734 / rev 2b / Expires: 2027-03-14 / lot# aQ8-zX2-004 / tiny print: verify seal"), + ("T2 counting + attribute binding", ["count.png"], + "Count each kind of shape. Answer in exactly three lines: 'yellow circles: N', 'purple triangles: N', 'red squares: N'.", + "yellow circles: 7, purple triangles: 3, red squares: 1"), + ("T3 chart reading", ["chart.png"], + "Read this bar chart. Give the value for every day, then state which day is highest and which is lowest.", + "Mon 34, Tue 71, Wed 22, Thu 58, Fri 93, Sat 47; highest Fri, lowest Wed"), + ("T4 spatial + occlusion", ["spatial.png"], + "Describe the image: every shape, its colour, its position, and state which shape is partially hidden behind another.", + "green star top-right PARTLY BEHIND a grey rectangle; orange circle lower-left; text 'left side text' upper-left"), + ("T5 multi-image comparison (2 images)", ["count.png","chart.png"], + "You are given two images. In one sentence each, say what image 1 shows and what image 2 shows, then state which one contains text.", + "img1 = scattered shapes (no text); img2 = bar chart (has text)"), +] + +for name, imgs, prompt, truth in TESTS: + try: + out, fin, ptok = ask(imgs, prompt) + except Exception as e: + print(f"\n===== {name}\n ERROR: {e}"); continue + leak = "" in out + print(f"\n===== {name} [prompt_tokens={ptok} finish={fin} think_leak={leak}]") + print(f" GROUND TRUTH: {truth}") + print(" MODEL:") + for line in out.strip().splitlines()[:16]: + print(" ", line)