From ae6731017f52b01c7ab9fc515091c3b408ba0b8c Mon Sep 17 00:00:00 2001 From: Chris Tsang Date: Tue, 28 Jul 2026 16:08:45 +0100 Subject: [PATCH] add vtracer-bench --- Cargo.toml | 1 + crates/vtracer-bench/Cargo.toml | 18 +++ crates/vtracer-bench/README.md | 104 +++++++++++++ crates/vtracer-bench/src/lib.rs | 247 +++++++++++++++++++++++++++++++ crates/vtracer-bench/src/main.rs | 66 +++++++++ 5 files changed, 436 insertions(+) create mode 100644 crates/vtracer-bench/Cargo.toml create mode 100644 crates/vtracer-bench/README.md create mode 100644 crates/vtracer-bench/src/lib.rs create mode 100644 crates/vtracer-bench/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 4d23ca9..7a2f605 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "crates/vtracer", "crates/vtracer-cli", + "crates/vtracer-bench", ] # The pre-1.0 webapp is kept in the tree for now but is no longer part of the diff --git a/crates/vtracer-bench/Cargo.toml b/crates/vtracer-bench/Cargo.toml new file mode 100644 index 0000000..9bd4a2d --- /dev/null +++ b/crates/vtracer-bench/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "vtracer-bench" +description = "Blind fidelity benchmark for raster-to-vector tracers: compare the original raster with a rendered reconstruction and get one 0..1 fidelity score built from PSNR, SSIM and a clustered-diff patch metric." +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +categories = ["graphics", "development-tools::testing"] +keywords = ["vectorization", "benchmark", "fidelity", "ssim", "psnr"] + +[dependencies] +visioncortex.workspace = true +dssim-core = "3" +rgb = "0.8" +# Decode-only: trimmed to real input formats (drops the AV1 encoder + OpenEXR). +image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"] } diff --git a/crates/vtracer-bench/README.md b/crates/vtracer-bench/README.md new file mode 100644 index 0000000..3661245 --- /dev/null +++ b/crates/vtracer-bench/README.md @@ -0,0 +1,104 @@ +# vtracer-bench + +Blind fidelity benchmark for raster-to-vector tracers. + +It compares an **original raster** with a **rendered reconstruction** and reports one number — a fidelity score in **[0, 1]** — built from three complementary axes. It is *blind* in the sense that it knows nothing about how the reconstruction was produced: any tracer, any format, any renderer. Render your vector output to pixels (same dimensions as the original), then let the benchmark judge. + +```console +$ vtracer-bench original.png reconstruction.png +psnr 34.77 dB (rmse 4.66) -> 0.6875 +ssim 0.99541 (dssim 0.00461) -> 0.9954 +patch 157.8 px rms (14503 bad px, 74 clusters, largest 73) -> 0.9726 +fidelity 0.9022 +[csv] 0.9022,34.77,0.00461,4.66,157.8,0.6875,0.9954,0.9726 +``` + +## Why another metric? + +Every classic metric has a blind spot, and tracers exploit all of them: + +- **PSNR** over-values invisible dust and undersells small salient regions — a tracer that drops an eye but nails the background can post a great PSNR. +- **SSIM** tracks perceived quality well, but averages globally: a small, fully-lost region barely moves it. +- Neither can tell **a thousand scattered ±1 pixels** apart from **one coherent missing patch** of the same total mass — and the missing patch is the failure that actually matters. + +`vtracer-bench` scores all three axes and combines them so that no single blind spot survives: + +| axis | raw metric | subscore in [0, 1] | +| --- | --- | --- | +| `psnr` | sRGB PSNR over RGB | `1 − log(1+rmse) / log(256)` | +| `ssim` | multiscale DSSIM (`dssim-core`) | `SSIM = 1 / (1 + DSSIM)` | +| `patch` | clustered-diff "missing patch" detector | `2^(−P / 0.005)` | + +**fidelity = ( psnr¹ · ssim² · patch¹ ) ^ (1/4)** — a *weighted geometric mean*. Geometric, not arithmetic, so a single collapsed axis drags the composite down: a missing face region cannot hide behind good global PSNR. SSIM carries double weight because it tracks visual accuracy best and is the axis most robust to an imperfect source. + +## The three axes + +### psnr — parameter-free squash + +The squash `1 − log(1+rmse)/log(256)` is anchored at the only two natural error scales an 8-bit image has: + +- `rmse = 255` (the full range — noise indistinguishable from a random image) → **0** +- `rmse ≤ 1` (the quantization step — errors 8-bit can barely represent) → saturates to **1** + +For `rmse ≫ 1` it equals `psnr / 48.13 dB`, i.e. it stays linear in decibels, but with no hand-picked anchor constants. + +### ssim — perceptual structure + +`dssim-core` computes multiscale structural dissimilarity `d = 1/SSIM − 1`; the subscore is simply `SSIM = 1/(1+d)`, already a natural 0..1. Differences the eye can't see score ~1 regardless of how many pixels they touch. + +### patch — the missing-patch detector + +This is the axis PSNR and SSIM both lack: + +1. A pixel is **bad** iff its RGB Euclidean distance to the original exceeds `--thresh` (default 24 — roughly 14 per channel). +2. The bad mask is **opened** (one round of 4-connected erode + dilate). A slightly blurred or recompressed *source* shifts every edge and paints ≤2 px filaments along all boundaries; those vanish under the opening, while genuine missing patches survive. This is what makes the benchmark tolerant of mildly compressed or blurred originals. +3. The surviving mask is clustered (4-connected). With cluster areas `aᵢ`, the **patch mass** is `√(Σ aᵢ²)` — a sum of *squares*, so one coherent blob dominates any amount of scattered dust of equal total area. +4. With `P = patch mass / (w·h)`, the subscore is `2^(−P/0.005)`: a single coherent blob at 0.5 % of image mass halves the score; scattered dust barely dents it. + +## Calibration + +Scored on a 768×1024 flat-shaded illustration, comparing the original against distorted versions of **itself** — this is how much slack the benchmark gives an imperfect source, and what the top of the scale means: + +| candidate | psnr | ssim | patch | **fidelity** | +| --- | --- | --- | --- | --- | +| the original itself | 1.000 | 1.000 | 1.000 | **1.0000** | +| JPEG quality 95 | 0.816 | 1.000 | 1.000 | **0.9502** | +| JPEG quality 75 | 0.718 | 0.999 | 0.994 | **0.9186** | +| 0.8 px Gaussian blur | 0.596 | 0.995 | 0.861 | **0.8443** | + +Rule of thumb: **≥ 0.95** is visually indistinguishable, **≥ 0.90** is a faithful trace, **≤ 0.80** has visible geometry or color errors, and a score that *collapses* while PSNR/SSIM stay high means the patch axis found a coherent missing region — look at the `--mask` output. + +## Usage + +### CLI + +```console +vtracer-bench [--thresh N] [--mask out.png] +``` + +- `original`, `candidate` — rasters of identical dimensions (any format `image` decodes). Rendering an SVG to pixels is deliberately out of scope: use the renderer whose output you actually ship (resvg, Chromium, librsvg, …) so the benchmark judges what users see. +- `--thresh N` — RGB Euclidean bad-pixel gate for the patch axis (default 24). +- `--mask out.png` — write the raw bad-pixel mask (before the opening) for visual inspection. + +The last stdout line is machine-readable: + +``` +[csv] fidelity,psnr,dssim,rmse,patch_mass,s_psnr,s_ssim,s_patch +``` + +(RMSE is reported for reference but carries no weight — it is the same MSE that PSNR measures, only on a linear curve; scoring both would double-weight one error.) + +### Library + +```rust +use vtracer_bench::{fidelity, DEFAULT_THRESH}; + +// orig and cand are interleaved RGB8, both w×h +let (report, bad_mask) = fidelity(&orig, &cand, w, h, DEFAULT_THRESH); +println!("fidelity {:.4} (psnr {:.2} dB, dssim {:.5})", + report.fidelity, report.psnr, report.dssim); +``` + +`FidelityReport` exposes every raw metric and subscore; the tuning constants (`PATCH_HALF`, `DEFAULT_THRESH`, and the `W_PSNR`/`W_SSIM`/`W_PATCH` weights) are public and documented in `lib.rs`. + +The benchmark is fully deterministic: identical inputs produce byte-identical output. diff --git a/crates/vtracer-bench/src/lib.rs b/crates/vtracer-bench/src/lib.rs new file mode 100644 index 0000000..767e494 --- /dev/null +++ b/crates/vtracer-bench/src/lib.rs @@ -0,0 +1,247 @@ +//! Universal tracer fidelity benchmark — original vs reconstruction, blind to +//! how the reconstruction was made. Three raw metrics, each squashed to [0,1], +//! composed by geometric mean into ONE fidelity score (0 = garbage, 1 = exact): +//! +//! psnr sRGB PSNR over RGB. Squash: 1 − log(1+rmse)/log(256) — anchored +//! at the two natural scales of 8-bit imagery and nothing else: +//! rmse = 255 (full range) → 0, rmse ≤ 1 (the quantization step) +//! saturates to 1. Equals psnr/48.13dB for rmse ≫ 1, i.e. still +//! linear in dB, without arbitrary anchor constants. +//! ssim dssim-core multiscale DSSIM d (= 1/SSIM − 1) → SSIM = 1/(1+d), +//! already a natural 0..1. +//! patch the "missing patch" / systematic-bias detector: bad ⟺ RGB +//! Euclidean diff > thresh, OPEN the bad mask (1-round 4-conn +//! erode+dilate — a slightly blurred or compressed source shifts +//! every edge and paints ≤2px filaments along all boundaries; those +//! vanish, real patches survive), then cluster it (visioncortex, +//! 4-conn), S = Σ area². Patch mass fraction P = √S / (w·h) — the RMS +//! coherent-blob size as a fraction of the image. Squash: 2^(−P/0.005), +//! so ONE coherent blob at 0.5% image mass halves the score while the +//! same pixel count scattered as dust barely dents it. Exactly the +//! failure mode PSNR/SSIM average away. +//! +//! Composite: weighted geometric mean, fidelity = (psnr¹ · ssim² · patch¹)^(1/4). +//! Geometric (not arithmetic) so a single collapsed axis drags the composite +//! down — a missing eye can't hide behind good global PSNR. SSIM carries double +//! weight: it tracks visual accuracy best and is the axis most robust to a +//! mildly compressed or blurred source. + +use visioncortex::BinaryImage; + +/// Patch mass fraction that halves the patch subscore. +pub const PATCH_HALF: f64 = 0.005; +/// Default RGB Euclidean distance for a pixel to count as "bad". +pub const DEFAULT_THRESH: f64 = 24.0; +/// Composite weights (geometric): fidelity = (psnr^1 · ssim^2 · patch^1)^(1/4). +pub const W_PSNR: f64 = 1.0; +pub const W_SSIM: f64 = 2.0; +pub const W_PATCH: f64 = 1.0; + +#[derive(Debug, Clone, Copy)] +pub struct FidelityReport { + // raw + pub psnr: f64, + pub dssim: f64, + /// sRGB RMSE — reported for reference, carries no weight (PSNR is the + /// same MSE on a log curve; scoring both would double-weight it) + pub rmse: f64, + /// bad pixels (‖Δrgb‖ > thresh), before the opening + pub bad_px: usize, + /// 4-conn clusters of bad pixels after the opening + pub clusters: usize, + /// largest cluster area (px) + pub largest: usize, + /// √(Σ area²) — RMS coherent-blob mass, in px + pub patch_mass: f64, + // subscores in [0,1] + pub s_psnr: f64, + pub s_ssim: f64, + pub s_patch: f64, + /// geometric mean of the three subscores + pub fidelity: f64, +} + +fn dssim_score(a_rgb: &[u8], b_rgb: &[u8], w: usize, h: usize) -> f64 { + let d = dssim_core::Dssim::new(); + let to = |buf: &[u8]| { + let px: Vec> = + (0..w * h).map(|i| rgb::RGB { r: buf[i * 3], g: buf[i * 3 + 1], b: buf[i * 3 + 2] }).collect(); + d.create_image_rgb(&px, w, h).expect("dssim image") + }; + let (val, _) = d.compare(&to(a_rgb), &to(b_rgb)); + val.into() +} + +/// Compare an original against a candidate reconstruction, both RGB8, w×h. +/// `thresh` is the RGB Euclidean bad-pixel gate (use [`DEFAULT_THRESH`]). +/// Returns the report plus the bad-pixel mask (255/0, one byte per pixel). +pub fn fidelity(orig_rgb: &[u8], cand_rgb: &[u8], w: usize, h: usize, thresh: f64) -> (FidelityReport, Vec) { + assert_eq!(orig_rgb.len(), w * h * 3); + assert_eq!(cand_rgb.len(), w * h * 3); + + // PSNR + RMSE + bad-pixel binarization in one pass + let mut sse = 0f64; + let mut mask = vec![0u8; w * h]; + let mut bad_px = 0usize; + let t2 = thresh * thresh; + for y in 0..h { + for x in 0..w { + let i = y * w + x; + let mut d2 = 0f64; + for c in 0..3 { + let e = orig_rgb[i * 3 + c] as f64 - cand_rgb[i * 3 + c] as f64; + d2 += e * e; + } + sse += d2; + if d2 > t2 { + mask[i] = 255; + bad_px += 1; + } + } + } + let rmse = (sse / (w * h * 3) as f64).sqrt(); + let psnr = 20.0 * (255.0 / rmse.max(1e-6)).log10(); + + let dssim = dssim_score(orig_rgb, cand_rgb, w, h); + + // opening: 1-round 4-conn erode + dilate. Edge-shift filaments (≤2px wide, + // the signature of a slightly blurred/compressed source) vanish; genuine + // missing patches survive. The reported mask keeps the raw bad pixels. + let at = |m: &[u8], x: i64, y: i64| { + x >= 0 && y >= 0 && (x as usize) < w && (y as usize) < h && m[y as usize * w + x as usize] != 0 + }; + let mut eroded = vec![0u8; w * h]; + for y in 0..h as i64 { + for x in 0..w as i64 { + if at(&mask, x, y) + && at(&mask, x - 1, y) + && at(&mask, x + 1, y) + && at(&mask, x, y - 1) + && at(&mask, x, y + 1) + { + eroded[y as usize * w + x as usize] = 255; + } + } + } + let mut bin = BinaryImage::new_w_h(w, h); + for y in 0..h as i64 { + for x in 0..w as i64 { + if at(&eroded, x, y) + || at(&eroded, x - 1, y) + || at(&eroded, x + 1, y) + || at(&eroded, x, y - 1) + || at(&eroded, x, y + 1) + { + bin.set_pixel(x as usize, y as usize, true); + } + } + } + + let sizes: Vec = bin.to_clusters(false).iter().map(|c| c.size()).collect(); + let largest = sizes.iter().copied().max().unwrap_or(0); + let patch_mass = if sizes.is_empty() { + 0.0 + } else { + sizes.iter().map(|&a| (a as f64) * (a as f64)).sum::().sqrt() + }; + let p_frac = patch_mass / (w * h) as f64; + + let s_psnr = 1.0 - (1.0 + rmse).ln() / 256f64.ln(); + let s_ssim = 1.0 / (1.0 + dssim); + let s_patch = (-p_frac / PATCH_HALF * std::f64::consts::LN_2).exp(); + let fidelity = (s_psnr.powf(W_PSNR) * s_ssim.powf(W_SSIM) * s_patch.powf(W_PATCH)) + .powf(1.0 / (W_PSNR + W_SSIM + W_PATCH)); + + ( + FidelityReport { + psnr, + dssim, + rmse, + bad_px, + clusters: sizes.len(), + largest, + patch_mass, + s_psnr, + s_ssim, + s_patch, + fidelity, + }, + mask, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn flat(w: usize, h: usize, c: [u8; 3]) -> Vec { + (0..w * h).flat_map(|_| c).collect() + } + + #[test] + fn identical_is_one() { + let a = flat(64, 64, [120, 90, 200]); + let (r, mask) = fidelity(&a, &a, 64, 64, DEFAULT_THRESH); + assert_eq!(r.bad_px, 0); + assert!(mask.iter().all(|&m| m == 0)); + assert!((r.fidelity - 1.0).abs() < 1e-9, "fidelity {}", r.fidelity); + } + + #[test] + fn coherent_patch_scores_below_scattered_dust() { + // same 256 bad pixels: one 16×16 blob vs isolated singles on a 64×64 grid + let clean = flat(64, 64, [200, 200, 200]); + let mut blob = clean.clone(); + for y in 24..40 { + for x in 24..40 { + blob[(y * 64 + x) * 3..(y * 64 + x) * 3 + 3].fill(0); + } + } + let mut dust = clean.clone(); + for k in 0..256 { + let (x, y) = ((k % 16) * 4, (k / 16) * 4); // 4px spacing: 256 singleton clusters + dust[(y * 64 + x) * 3..(y * 64 + x) * 3 + 3].fill(0); + } + let (rb, _) = fidelity(&clean, &blob, 64, 64, DEFAULT_THRESH); + let (rd, _) = fidelity(&clean, &dust, 64, 64, DEFAULT_THRESH); + assert_eq!(rb.bad_px, 256); + assert_eq!(rd.bad_px, 256); + // dust vanishes under the opening entirely; the blob survives + assert_eq!(rb.clusters, 1); + assert_eq!(rd.clusters, 0); + assert!((rd.s_patch - 1.0).abs() < 1e-9); + // identical PSNR/RMSE by construction; the patch axis must separate them + assert!((rb.rmse - rd.rmse).abs() < 1e-9); + assert!(rb.s_patch < rd.s_patch * 0.25, "blob {} dust {}", rb.s_patch, rd.s_patch); + assert!(rb.fidelity < rd.fidelity); + } + + #[test] + fn edge_shift_filaments_are_tolerated() { + // a slightly blurred/compressed source shifts edges: thin bad-px lines + // along boundaries. A 2px-wide full-width filament (256 px) must open + // away; the same mass as a compact blob must not. + let clean = flat(64, 64, [200, 200, 200]); + let mut fil = clean.clone(); + for y in 30..32 { + for x in 0..64 { + fil[(y * 64 + x) * 3..(y * 64 + x) * 3 + 3].fill(0); + } + } + let (rf, _) = fidelity(&clean, &fil, 64, 64, DEFAULT_THRESH); + assert_eq!(rf.bad_px, 128); + assert_eq!(rf.clusters, 0); + assert!((rf.s_patch - 1.0).abs() < 1e-9, "filament must not count as a patch"); + } + + #[test] + fn worse_is_lower() { + let a = flat(32, 32, [100, 100, 100]); + let mild: Vec = a.iter().map(|&v| v + 4).collect(); + let harsh: Vec = a.iter().map(|&v| v + 60).collect(); + let (rm, _) = fidelity(&a, &mild, 32, 32, DEFAULT_THRESH); + let (rh, _) = fidelity(&a, &harsh, 32, 32, DEFAULT_THRESH); + assert!(rm.fidelity > rh.fidelity); + assert!(rh.fidelity < 0.4, "harsh {}", rh.fidelity); + } +} diff --git a/crates/vtracer-bench/src/main.rs b/crates/vtracer-bench/src/main.rs new file mode 100644 index 0000000..cf947c1 --- /dev/null +++ b/crates/vtracer-bench/src/main.rs @@ -0,0 +1,66 @@ +//! Blind fidelity benchmark for raster-to-vector tracers. +//! +//! vtracer-bench [--thresh N] [--mask out.png] +//! +//! Both arguments are rasters of identical dimensions — rendering a vector +//! reconstruction to pixels is the caller's responsibility. Prints the raw +//! metrics, their [0,1] subscores, the composite fidelity, and a +//! machine-readable csv line. + +use vtracer_bench::{fidelity, DEFAULT_THRESH}; + +fn main() { + let args: Vec = std::env::args().collect(); + if args.len() < 3 { + eprintln!("usage: vtracer-bench [--thresh N] [--mask out.png]"); + std::process::exit(2); + } + let mut thresh = DEFAULT_THRESH; + let mut mask_out: Option = None; + let mut i = 3; + while i < args.len() { + match args[i].as_str() { + "--thresh" => { + i += 1; + thresh = args[i].parse().expect("--thresh N"); + } + "--mask" => { + i += 1; + mask_out = Some(args[i].clone()); + } + a => { + eprintln!("unknown flag {a}"); + std::process::exit(2); + } + } + i += 1; + } + + let orig = image::open(&args[1]).expect("open original").to_rgb8(); + let (w, h) = (orig.width() as usize, orig.height() as usize); + let img = image::open(&args[2]).expect("open candidate").to_rgb8(); + assert_eq!( + (img.width() as usize, img.height() as usize), + (w, h), + "candidate raster must match original dimensions" + ); + let cand: Vec = img.into_raw(); + + let (r, mask) = fidelity(orig.as_raw(), &cand, w, h, thresh); + + if let Some(out) = mask_out { + image::GrayImage::from_raw(w as u32, h as u32, mask).unwrap().save(&out).expect("save mask"); + } + + println!("psnr {:>8.2} dB (rmse {:.2}) -> {:.4}", r.psnr, r.rmse, r.s_psnr); + println!("ssim {:>8.5} (dssim {:.5}) -> {:.4}", r.s_ssim, r.dssim, r.s_ssim); + println!( + "patch {:>8.1} px rms ({} bad px, {} clusters, largest {}) -> {:.4}", + r.patch_mass, r.bad_px, r.clusters, r.largest, r.s_patch + ); + println!("fidelity {:.4}", r.fidelity); + println!( + "[csv] {:.4},{:.2},{:.5},{:.2},{:.1},{:.4},{:.4},{:.4}", + r.fidelity, r.psnr, r.dssim, r.rmse, r.patch_mass, r.s_psnr, r.s_ssim, r.s_patch + ); +}