From 571c419e512c7d0c8f85cdb631c8f28c6bab2d7b Mon Sep 17 00:00:00 2001 From: SGavrl <277843151+SGavrl@users.noreply.github.com> Date: Thu, 23 Jul 2026 10:07:04 +0000 Subject: [PATCH] test: add deterministic render parity checks --- render-repros/check.py | 139 +++++++++++++++++++ render-repros/checks.json | 35 +++++ render-repros/paired-corpus.py | 238 +++++++++++++++++++++++++++++++++ render-repros/run.sh | 6 + render-repros/spot-sites.txt | 9 ++ 5 files changed, 427 insertions(+) create mode 100755 render-repros/check.py create mode 100644 render-repros/checks.json create mode 100755 render-repros/paired-corpus.py create mode 100644 render-repros/spot-sites.txt diff --git a/render-repros/check.py b/render-repros/check.py new file mode 100755 index 0000000..ad4bd1a --- /dev/null +++ b/render-repros/check.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Check deterministic repro behavior and report native-coordinate image deltas. + +Behavior assertions are based on solid-color component geometry, so font +anti-aliasing does not decide pass/fail. Pair metrics are diagnostics only: +they use the full equal-sized canvas with no registration, cropping, blank-page +exclusion, or opaque aggregate "parity" verdict. +""" + +import json +import sys +from pathlib import Path + +import numpy as np +from PIL import Image +from scipy.ndimage import find_objects, label + + +def rgb(value): + value = value.removeprefix("#") + return tuple(int(value[i : i + 2], 16) for i in (0, 2, 4)) + + +def components(array, color): + mask = np.all(array == color, axis=2) + labels, _ = label(mask) + found = [] + for slices in find_objects(labels): + if slices is None: + continue + height = slices[0].stop - slices[0].start + width = slices[1].stop - slices[1].start + area = int(mask[slices].sum()) + if area >= 20: + found.append({"width": width, "height": height, "area": area}) + return found + + +def bbox(mask): + ys, xs = np.nonzero(mask) + if not len(xs): + return None + return [int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1] + + +def pair_metrics(ours, chromium): + if ours.shape != chromium.shape: + return {"size_mismatch": [list(ours.shape), list(chromium.shape)]} + delta = np.abs(ours.astype(np.int16) - chromium.astype(np.int16)) + max_channel = delta.max(axis=2) + ours_ink = np.any(ours < 245, axis=2) + chromium_ink = np.any(chromium < 245, axis=2) + ours_bbox = bbox(ours_ink) + chromium_bbox = bbox(chromium_ink) + bbox_delta = None + if ours_bbox and chromium_bbox: + bbox_delta = max(abs(a - b) for a, b in zip(ours_bbox, chromium_bbox)) + height, width = ours_ink.shape + row_projection = float( + np.abs(ours_ink.sum(axis=1) - chromium_ink.sum(axis=1)).mean() / width + ) + col_projection = float( + np.abs(ours_ink.sum(axis=0) - chromium_ink.sum(axis=0)).mean() / height + ) + return { + "rgb_mae": round(float(delta.mean() / 255.0), 6), + "pixels_gt_10": round(float((max_channel > 10).mean()), 6), + "pixels_gt_50": round(float((max_channel > 50).mean()), 6), + "ours_content_bbox": ours_bbox, + "chromium_content_bbox": chromium_bbox, + "content_bbox_max_delta": bbox_delta, + "row_projection_delta": round(row_projection, 6), + "column_projection_delta": round(col_projection, 6), + } + + +def main(): + if len(sys.argv) != 2: + raise SystemExit("usage: check.py OUTDIR") + out = Path(sys.argv[1]) + checks = json.loads((Path(__file__).with_name("checks.json")).read_text()) + failures = [] + report = {"fixtures": {}} + + for ours_path in sorted(out.glob("*.obscura.png")): + name = ours_path.name.removesuffix(".obscura.png") + chromium_path = out / f"{name}.chrome.png" + if not chromium_path.is_file(): + failures.append(f"{name}: missing Chromium screenshot") + continue + ours = np.asarray(Image.open(ours_path).convert("RGB")) + chromium = np.asarray(Image.open(chromium_path).convert("RGB")) + fixture = { + "metrics": pair_metrics(ours, chromium), + "behavior": {"obscura": [], "chromium": []}, + } + for engine, array in (("obscura", ours), ("chromium", chromium)): + for check in checks.get(name, []): + expected_count = check.get("count", 1) + matches = [] + for component in components(array, rgb(check["color"])): + width_ok = "width" not in check or abs(component["width"] - check["width"]) <= 1 + height_ok = "height" not in check or abs(component["height"] - check["height"]) <= 1 + if width_ok and height_ok: + matches.append(component) + passed = len(matches) >= expected_count + fixture["behavior"][engine].append( + { + "name": check["name"], + "passed": passed, + "expected_count": expected_count, + "matches": matches, + } + ) + if not passed: + failures.append( + f"{name} {engine}: {check['name']} expected {expected_count}, found {len(matches)}" + ) + report["fixtures"][name] = fixture + metrics = fixture["metrics"] + print( + f"{name:24} rgb_mae={metrics.get('rgb_mae', 'size-mismatch'):>8} " + f"p>50={metrics.get('pixels_gt_50', '-'):>8} " + f"bbox={metrics.get('content_bbox_max_delta', '-'):>3} " + f"row={metrics.get('row_projection_delta', '-'):>8} " + f"col={metrics.get('column_projection_delta', '-'):>8}" + ) + + report["failures"] = failures + (out / "analysis.json").write_text(json.dumps(report, indent=2) + "\n") + if failures: + for failure in failures: + print(f"FAILED behavior: {failure}", file=sys.stderr) + raise SystemExit(1) + print(f"behavior checks passed; diagnostics in {out / 'analysis.json'}") + + +if __name__ == "__main__": + main() diff --git a/render-repros/checks.json b/render-repros/checks.json new file mode 100644 index 0000000..da917fd --- /dev/null +++ b/render-repros/checks.json @@ -0,0 +1,35 @@ +{ + "abs-fill": [ + {"name": "blue fill", "color": "#0000ff", "width": 400, "height": 100}, + {"name": "teal fill", "color": "#008080", "width": 400, "height": 100}, + {"name": "purple fill", "color": "#800080", "width": 400, "height": 100} + ], + "box-sizing": [ + {"name": "content-box fixed", "color": "#22aa55", "width": 220, "height": 90}, + {"name": "border-box fixed", "color": "#228866", "width": 190, "height": 60}, + {"name": "content-box percent", "color": "#668822", "width": 220, "height": 70}, + {"name": "border-box percent", "color": "#668822", "width": 190, "height": 40}, + {"name": "inherited reset border-box", "color": "#aa3377", "width": 190, "height": 60} + ], + "carousel-clip": [ + {"name": "visible translated slide", "color": "#33aa33", "width": 300, "height": 100}, + {"name": "visible control slide", "color": "#ee3333", "width": 300, "height": 100} + ], + "cascade-priority": [ + {"name": "important winners", "color": "#008000", "width": 320, "height": 36, "count": 6} + ], + "clear-float": [ + {"name": "float", "color": "#88bbdd", "width": 120, "height": 80}, + {"name": "cleared block", "color": "#ffcc66", "width": 500, "height": 40} + ], + "mixed-content-block": [ + {"name": "full-width block child", "color": "#ccddff", "width": 400} + ], + "tables": [ + {"name": "content-box fixed cell column", "color": "#ffcc99", "width": 208} + ], + "z-index": [ + {"name": "positive layer", "color": "#008000", "width": 200, "height": 100}, + {"name": "negative layer", "color": "#ffa500", "width": 260, "height": 110} + ] +} diff --git a/render-repros/paired-corpus.py b/render-repros/paired-corpus.py new file mode 100755 index 0000000..163062a --- /dev/null +++ b/render-repros/paired-corpus.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Capture Obscura and Chromium concurrently with the same real settle delay. + +Every run uses a new output directory, checks process status and non-empty +screenshots, records browser versions and timings, and reports the raw +full-canvas metrics from check.py. It deliberately emits no aggregate parity +verdict. An optional pre-change Obscura binary can be captured concurrently so +regressions are compared against the same live-page moment. +""" + +import argparse +import concurrent.futures +import json +import os +import re +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np +from PIL import Image +from playwright.sync_api import sync_playwright + +from check import pair_metrics + + +def slug(url): + value = re.sub(r"[^a-z0-9]+", "-", url.lower()).strip("-") + return value[:80] or "page" + + +def binary_version(binary): + result = subprocess.run( + [binary, "--version"], capture_output=True, text=True, timeout=10 + ) + text = (result.stdout or result.stderr).strip() + return {"status": result.returncode, "text": text} + + +def capture_obscura(binary, url, screenshot, log, width, height, settle_ms): + env = dict( + os.environ, + OBSCURA_SHOT_W=str(width), + OBSCURA_SHOT_H=str(height), + OBSCURA_ALLOW_PRIVATE_NETWORK="1", + ) + command = [ + binary, + "fetch", + url, + "--screenshot", + str(screenshot), + "--timeout", + "50000", + "--wait", + f"{settle_ms / 1000:g}", + ] + started = time.time() + try: + result = subprocess.run( + command, capture_output=True, text=True, timeout=75, env=env + ) + log.write_text(result.stdout + result.stderr) + ok = result.returncode == 0 and screenshot.is_file() and screenshot.stat().st_size > 0 + return { + "ok": ok, + "status": result.returncode, + "elapsed_s": round(time.time() - started, 3), + } + except subprocess.TimeoutExpired as error: + text = (error.stdout or "") + (error.stderr or "") + log.write_text(text if isinstance(text, str) else text.decode(errors="replace")) + return {"ok": False, "status": "timeout", "elapsed_s": round(time.time() - started, 3)} + + +def load_rgb(path): + return np.asarray(Image.open(path).convert("RGB")) + + +def write_results(path, manifest): + path.write_text(json.dumps(manifest, indent=2) + "\n") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("urls", help="one URL per line; # comments allowed") + parser.add_argument("--obscura-bin", required=True) + parser.add_argument("--baseline-bin") + parser.add_argument( + "--chromium-bin", + help="Chromium executable (default: Playwright's pinned Chromium)", + ) + parser.add_argument("--out", required=True, help="must not already exist") + parser.add_argument("--width", type=int, default=1280) + parser.add_argument("--height", type=int, default=1400) + parser.add_argument("--settle-ms", type=int, default=3000) + args = parser.parse_args() + + out = Path(args.out) + out.mkdir(parents=True, exist_ok=False) + urls = [ + line.strip() + for line in Path(args.urls).read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + manifest = { + "started_utc": datetime.now(timezone.utc).isoformat(), + "viewport": {"width": args.width, "height": args.height, "dpr": 1}, + "settle_ms_after_load": args.settle_ms, + "obscura": binary_version(args.obscura_bin), + "baseline": binary_version(args.baseline_bin) if args.baseline_bin else None, + "pages": [], + } + results_path = out / "results.json" + write_results(results_path, manifest) + + with sync_playwright() as playwright: + chromium_executable = args.chromium_bin or playwright.chromium.executable_path + manifest["chromium_executable"] = chromium_executable + browser = playwright.chromium.launch( + executable_path=chromium_executable, + headless=True, + args=[ + "--no-sandbox", + "--hide-scrollbars", + "--disable-background-networking", + "--force-device-scale-factor=1", + ], + ) + manifest["chromium_version"] = browser.version + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + for index, url in enumerate(urls): + name = f"{index:03d}-{slug(url)}" + ours_path = out / f"{name}.obscura.png" + chrome_path = out / f"{name}.chrome.png" + baseline_path = out / f"{name}.baseline.png" + page_result = {"url": url, "name": name} + context = browser.new_context( + viewport={"width": args.width, "height": args.height}, + device_scale_factor=1, + color_scheme="light", + reduced_motion="no-preference", + locale="en-US", + timezone_id="UTC", + ) + page = context.new_page() + chrome_messages = [] + page.on("console", lambda message: chrome_messages.append(f"console {message.type}: {message.text}")) + page.on("pageerror", lambda error: chrome_messages.append(f"pageerror: {error}")) + + ours_future = executor.submit( + capture_obscura, + args.obscura_bin, + url, + ours_path, + out / f"{name}.obscura.log", + args.width, + args.height, + args.settle_ms, + ) + baseline_future = None + if args.baseline_bin: + baseline_future = executor.submit( + capture_obscura, + args.baseline_bin, + url, + baseline_path, + out / f"{name}.baseline.log", + args.width, + args.height, + args.settle_ms, + ) + + chrome_started = time.time() + try: + page.goto(url, wait_until="load", timeout=50000) + page.wait_for_timeout(args.settle_ms) + page.screenshot( + path=str(chrome_path), full_page=False, timeout=50000 + ) + chrome_ok = chrome_path.is_file() and chrome_path.stat().st_size > 0 + chrome_status = 0 + except Exception as error: + chrome_messages.append(f"capture error: {error}") + chrome_ok = False + chrome_status = "error" + (out / f"{name}.chrome.log").write_text("\n".join(chrome_messages) + "\n") + page_result["chromium"] = { + "ok": chrome_ok, + "status": chrome_status, + "elapsed_s": round(time.time() - chrome_started, 3), + "title": page.title() if chrome_ok else None, + } + context.close() + page_result["obscura"] = ours_future.result() + if baseline_future: + page_result["baseline"] = baseline_future.result() + + if chrome_ok and page_result["obscura"]["ok"]: + chrome_rgb = load_rgb(chrome_path) + current_metrics = pair_metrics(load_rgb(ours_path), chrome_rgb) + page_result["metrics"] = current_metrics + if baseline_future and page_result["baseline"]["ok"]: + baseline_metrics = pair_metrics(load_rgb(baseline_path), chrome_rgb) + page_result["baseline_metrics"] = baseline_metrics + for key in ("rgb_mae", "pixels_gt_10", "pixels_gt_50"): + if key in current_metrics and key in baseline_metrics: + page_result.setdefault("delta_vs_baseline", {})[key] = round( + current_metrics[key] - baseline_metrics[key], 6 + ) + manifest["pages"].append(page_result) + write_results(results_path, manifest) + metric = page_result.get("metrics", {}).get("pixels_gt_50") + delta = page_result.get("delta_vs_baseline", {}).get("pixels_gt_50") + print( + f"{name:84} p>50={metric if metric is not None else 'capture-fail'} " + f"delta={delta if delta is not None else '-'}", + flush=True, + ) + browser.close() + + manifest["finished_utc"] = datetime.now(timezone.utc).isoformat() + write_results(results_path, manifest) + failed = [ + page["name"] + for page in manifest["pages"] + if not page.get("chromium", {}).get("ok") or not page.get("obscura", {}).get("ok") + ] + if failed: + print(f"capture failures: {', '.join(failed)}", file=sys.stderr) + raise SystemExit(1) + print(f"paired captures and raw diagnostics: {results_path}") + + +if __name__ == "__main__": + main() diff --git a/render-repros/run.sh b/render-repros/run.sh index 41244b4..7704124 100755 --- a/render-repros/run.sh +++ b/render-repros/run.sh @@ -5,6 +5,7 @@ set -uo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" BIN="${OBSCURA_BIN:-$ROOT/target/release/obscura}" CHROME="${CHROME_BIN:-chromium}" +PYTHON="${PYTHON_BIN:-python3}" DIR="$(cd "$(dirname "$0")" && pwd)" OUT="${1:-$DIR/out}" mkdir -p "$OUT" @@ -36,6 +37,11 @@ for f in "$DIR"/*.html; do mv "$chrome_shot" "$OUT/$n.chrome.png" echo "rendered $n" done +if [[ "$status" -eq 0 ]]; then + if ! PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "$DIR/check.py" "$OUT"; then + status=1 + fi +fi if [[ "$status" -eq 0 ]]; then echo "output in $OUT" else diff --git a/render-repros/spot-sites.txt b/render-repros/spot-sites.txt new file mode 100644 index 0000000..00f37b5 --- /dev/null +++ b/render-repros/spot-sites.txt @@ -0,0 +1,9 @@ +# Stable-ish pages for a quick real-world regression spot set. +https://example.com/ +https://news.ycombinator.com/ +https://www.sqlite.org/index.html +https://blog.rust-lang.org/ +https://docs.python.org/3/ +https://web.dev/ +https://jvns.ca/ +https://en.wikipedia.org/wiki/Rust_(programming_language)