test: add deterministic render parity checks

render-repros/check.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""Check deterministic repro behavior and report native-coordinate image deltas.
+
+Behavior assertions are based on solid-color component geometry, so font
+anti-aliasing does not decide pass/fail. Pair metrics are diagnostics only:
+they use the full equal-sized canvas with no registration, cropping, blank-page
+exclusion, or opaque aggregate "parity" verdict.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+import numpy as np
+from PIL import Image
+from scipy.ndimage import find_objects, label
+
+
+def rgb(value):
+    value = value.removeprefix("#")
+    return tuple(int(value[i : i + 2], 16) for i in (0, 2, 4))
+
+
+def components(array, color):
+    mask = np.all(array == color, axis=2)
+    labels, _ = label(mask)
+    found = []
+    for slices in find_objects(labels):
+        if slices is None:
+            continue
+        height = slices[0].stop - slices[0].start
+        width = slices[1].stop - slices[1].start
+        area = int(mask[slices].sum())
+        if area >= 20:
+            found.append({"width": width, "height": height, "area": area})
+    return found
+
+
+def bbox(mask):
+    ys, xs = np.nonzero(mask)
+    if not len(xs):
+        return None
+    return [int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1]
+
+
+def pair_metrics(ours, chromium):
+    if ours.shape != chromium.shape:
+        return {"size_mismatch": [list(ours.shape), list(chromium.shape)]}
+    delta = np.abs(ours.astype(np.int16) - chromium.astype(np.int16))
+    max_channel = delta.max(axis=2)
+    ours_ink = np.any(ours < 245, axis=2)
+    chromium_ink = np.any(chromium < 245, axis=2)
+    ours_bbox = bbox(ours_ink)
+    chromium_bbox = bbox(chromium_ink)
+    bbox_delta = None
+    if ours_bbox and chromium_bbox:
+        bbox_delta = max(abs(a - b) for a, b in zip(ours_bbox, chromium_bbox))
+    height, width = ours_ink.shape
+    row_projection = float(
+        np.abs(ours_ink.sum(axis=1) - chromium_ink.sum(axis=1)).mean() / width
+    )
+    col_projection = float(
+        np.abs(ours_ink.sum(axis=0) - chromium_ink.sum(axis=0)).mean() / height
+    )
+    return {
+        "rgb_mae": round(float(delta.mean() / 255.0), 6),
+        "pixels_gt_10": round(float((max_channel > 10).mean()), 6),
+        "pixels_gt_50": round(float((max_channel > 50).mean()), 6),
+        "ours_content_bbox": ours_bbox,
+        "chromium_content_bbox": chromium_bbox,
+        "content_bbox_max_delta": bbox_delta,
+        "row_projection_delta": round(row_projection, 6),
+        "column_projection_delta": round(col_projection, 6),
+    }
+
+
+def main():
+    if len(sys.argv) != 2:
+        raise SystemExit("usage: check.py OUTDIR")
+    out = Path(sys.argv[1])
+    checks = json.loads((Path(__file__).with_name("checks.json")).read_text())
+    failures = []
+    report = {"fixtures": {}}
+
+    for ours_path in sorted(out.glob("*.obscura.png")):
+        name = ours_path.name.removesuffix(".obscura.png")
+        chromium_path = out / f"{name}.chrome.png"
+        if not chromium_path.is_file():
+            failures.append(f"{name}: missing Chromium screenshot")
+            continue
+        ours = np.asarray(Image.open(ours_path).convert("RGB"))
+        chromium = np.asarray(Image.open(chromium_path).convert("RGB"))
+        fixture = {
+            "metrics": pair_metrics(ours, chromium),
+            "behavior": {"obscura": [], "chromium": []},
+        }
+        for engine, array in (("obscura", ours), ("chromium", chromium)):
+            for check in checks.get(name, []):
+                expected_count = check.get("count", 1)
+                matches = []
+                for component in components(array, rgb(check["color"])):
+                    width_ok = "width" not in check or abs(component["width"] - check["width"]) <= 1
+                    height_ok = "height" not in check or abs(component["height"] - check["height"]) <= 1
+                    if width_ok and height_ok:
+                        matches.append(component)
+                passed = len(matches) >= expected_count
+                fixture["behavior"][engine].append(
+                    {
+                        "name": check["name"],
+                        "passed": passed,
+                        "expected_count": expected_count,
+                        "matches": matches,
+                    }
+                )
+                if not passed:
+                    failures.append(
+                        f"{name} {engine}: {check['name']} expected {expected_count}, found {len(matches)}"
+                    )
+        report["fixtures"][name] = fixture
+        metrics = fixture["metrics"]
+        print(
+            f"{name:24} rgb_mae={metrics.get('rgb_mae', 'size-mismatch'):>8} "
+            f"p>50={metrics.get('pixels_gt_50', '-'):>8} "
+            f"bbox={metrics.get('content_bbox_max_delta', '-'):>3} "
+            f"row={metrics.get('row_projection_delta', '-'):>8} "
+            f"col={metrics.get('column_projection_delta', '-'):>8}"
+        )
+
+    report["failures"] = failures
+    (out / "analysis.json").write_text(json.dumps(report, indent=2) + "\n")
+    if failures:
+        for failure in failures:
+            print(f"FAILED behavior: {failure}", file=sys.stderr)
+        raise SystemExit(1)
+    print(f"behavior checks passed; diagnostics in {out / 'analysis.json'}")
+
+
+if __name__ == "__main__":
+    main()

render-repros/checks.json

@@ -0,0 +1,35 @@
+{
+  "abs-fill": [
+    {"name": "blue fill", "color": "#0000ff", "width": 400, "height": 100},
+    {"name": "teal fill", "color": "#008080", "width": 400, "height": 100},
+    {"name": "purple fill", "color": "#800080", "width": 400, "height": 100}
+  ],
+  "box-sizing": [
+    {"name": "content-box fixed", "color": "#22aa55", "width": 220, "height": 90},
+    {"name": "border-box fixed", "color": "#228866", "width": 190, "height": 60},
+    {"name": "content-box percent", "color": "#668822", "width": 220, "height": 70},
+    {"name": "border-box percent", "color": "#668822", "width": 190, "height": 40},
+    {"name": "inherited reset border-box", "color": "#aa3377", "width": 190, "height": 60}
+  ],
+  "carousel-clip": [
+    {"name": "visible translated slide", "color": "#33aa33", "width": 300, "height": 100},
+    {"name": "visible control slide", "color": "#ee3333", "width": 300, "height": 100}
+  ],
+  "cascade-priority": [
+    {"name": "important winners", "color": "#008000", "width": 320, "height": 36, "count": 6}
+  ],
+  "clear-float": [
+    {"name": "float", "color": "#88bbdd", "width": 120, "height": 80},
+    {"name": "cleared block", "color": "#ffcc66", "width": 500, "height": 40}
+  ],
+  "mixed-content-block": [
+    {"name": "full-width block child", "color": "#ccddff", "width": 400}
+  ],
+  "tables": [
+    {"name": "content-box fixed cell column", "color": "#ffcc99", "width": 208}
+  ],
+  "z-index": [
+    {"name": "positive layer", "color": "#008000", "width": 200, "height": 100},
+    {"name": "negative layer", "color": "#ffa500", "width": 260, "height": 110}
+  ]
+}

render-repros/paired-corpus.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+"""Capture Obscura and Chromium concurrently with the same real settle delay.
+
+Every run uses a new output directory, checks process status and non-empty
+screenshots, records browser versions and timings, and reports the raw
+full-canvas metrics from check.py. It deliberately emits no aggregate parity
+verdict. An optional pre-change Obscura binary can be captured concurrently so
+regressions are compared against the same live-page moment.
+"""
+
+import argparse
+import concurrent.futures
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+import numpy as np
+from PIL import Image
+from playwright.sync_api import sync_playwright
+
+from check import pair_metrics
+
+
+def slug(url):
+    value = re.sub(r"[^a-z0-9]+", "-", url.lower()).strip("-")
+    return value[:80] or "page"
+
+
+def binary_version(binary):
+    result = subprocess.run(
+        [binary, "--version"], capture_output=True, text=True, timeout=10
+    )
+    text = (result.stdout or result.stderr).strip()
+    return {"status": result.returncode, "text": text}
+
+
+def capture_obscura(binary, url, screenshot, log, width, height, settle_ms):
+    env = dict(
+        os.environ,
+        OBSCURA_SHOT_W=str(width),
+        OBSCURA_SHOT_H=str(height),
+        OBSCURA_ALLOW_PRIVATE_NETWORK="1",
+    )
+    command = [
+        binary,
+        "fetch",
+        url,
+        "--screenshot",
+        str(screenshot),
+        "--timeout",
+        "50000",
+        "--wait",
+        f"{settle_ms / 1000:g}",
+    ]
+    started = time.time()
+    try:
+        result = subprocess.run(
+            command, capture_output=True, text=True, timeout=75, env=env
+        )
+        log.write_text(result.stdout + result.stderr)
+        ok = result.returncode == 0 and screenshot.is_file() and screenshot.stat().st_size > 0
+        return {
+            "ok": ok,
+            "status": result.returncode,
+            "elapsed_s": round(time.time() - started, 3),
+        }
+    except subprocess.TimeoutExpired as error:
+        text = (error.stdout or "") + (error.stderr or "")
+        log.write_text(text if isinstance(text, str) else text.decode(errors="replace"))
+        return {"ok": False, "status": "timeout", "elapsed_s": round(time.time() - started, 3)}
+
+
+def load_rgb(path):
+    return np.asarray(Image.open(path).convert("RGB"))
+
+
+def write_results(path, manifest):
+    path.write_text(json.dumps(manifest, indent=2) + "\n")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("urls", help="one URL per line; # comments allowed")
+    parser.add_argument("--obscura-bin", required=True)
+    parser.add_argument("--baseline-bin")
+    parser.add_argument(
+        "--chromium-bin",
+        help="Chromium executable (default: Playwright's pinned Chromium)",
+    )
+    parser.add_argument("--out", required=True, help="must not already exist")
+    parser.add_argument("--width", type=int, default=1280)
+    parser.add_argument("--height", type=int, default=1400)
+    parser.add_argument("--settle-ms", type=int, default=3000)
+    args = parser.parse_args()
+
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=False)
+    urls = [
+        line.strip()
+        for line in Path(args.urls).read_text().splitlines()
+        if line.strip() and not line.lstrip().startswith("#")
+    ]
+    manifest = {
+        "started_utc": datetime.now(timezone.utc).isoformat(),
+        "viewport": {"width": args.width, "height": args.height, "dpr": 1},
+        "settle_ms_after_load": args.settle_ms,
+        "obscura": binary_version(args.obscura_bin),
+        "baseline": binary_version(args.baseline_bin) if args.baseline_bin else None,
+        "pages": [],
+    }
+    results_path = out / "results.json"
+    write_results(results_path, manifest)
+
+    with sync_playwright() as playwright:
+        chromium_executable = args.chromium_bin or playwright.chromium.executable_path
+        manifest["chromium_executable"] = chromium_executable
+        browser = playwright.chromium.launch(
+            executable_path=chromium_executable,
+            headless=True,
+            args=[
+                "--no-sandbox",
+                "--hide-scrollbars",
+                "--disable-background-networking",
+                "--force-device-scale-factor=1",
+            ],
+        )
+        manifest["chromium_version"] = browser.version
+        with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+            for index, url in enumerate(urls):
+                name = f"{index:03d}-{slug(url)}"
+                ours_path = out / f"{name}.obscura.png"
+                chrome_path = out / f"{name}.chrome.png"
+                baseline_path = out / f"{name}.baseline.png"
+                page_result = {"url": url, "name": name}
+                context = browser.new_context(
+                    viewport={"width": args.width, "height": args.height},
+                    device_scale_factor=1,
+                    color_scheme="light",
+                    reduced_motion="no-preference",
+                    locale="en-US",
+                    timezone_id="UTC",
+                )
+                page = context.new_page()
+                chrome_messages = []
+                page.on("console", lambda message: chrome_messages.append(f"console {message.type}: {message.text}"))
+                page.on("pageerror", lambda error: chrome_messages.append(f"pageerror: {error}"))
+
+                ours_future = executor.submit(
+                    capture_obscura,
+                    args.obscura_bin,
+                    url,
+                    ours_path,
+                    out / f"{name}.obscura.log",
+                    args.width,
+                    args.height,
+                    args.settle_ms,
+                )
+                baseline_future = None
+                if args.baseline_bin:
+                    baseline_future = executor.submit(
+                        capture_obscura,
+                        args.baseline_bin,
+                        url,
+                        baseline_path,
+                        out / f"{name}.baseline.log",
+                        args.width,
+                        args.height,
+                        args.settle_ms,
+                    )
+
+                chrome_started = time.time()
+                try:
+                    page.goto(url, wait_until="load", timeout=50000)
+                    page.wait_for_timeout(args.settle_ms)
+                    page.screenshot(
+                        path=str(chrome_path), full_page=False, timeout=50000
+                    )
+                    chrome_ok = chrome_path.is_file() and chrome_path.stat().st_size > 0
+                    chrome_status = 0
+                except Exception as error:
+                    chrome_messages.append(f"capture error: {error}")
+                    chrome_ok = False
+                    chrome_status = "error"
+                (out / f"{name}.chrome.log").write_text("\n".join(chrome_messages) + "\n")
+                page_result["chromium"] = {
+                    "ok": chrome_ok,
+                    "status": chrome_status,
+                    "elapsed_s": round(time.time() - chrome_started, 3),
+                    "title": page.title() if chrome_ok else None,
+                }
+                context.close()
+                page_result["obscura"] = ours_future.result()
+                if baseline_future:
+                    page_result["baseline"] = baseline_future.result()
+
+                if chrome_ok and page_result["obscura"]["ok"]:
+                    chrome_rgb = load_rgb(chrome_path)
+                    current_metrics = pair_metrics(load_rgb(ours_path), chrome_rgb)
+                    page_result["metrics"] = current_metrics
+                    if baseline_future and page_result["baseline"]["ok"]:
+                        baseline_metrics = pair_metrics(load_rgb(baseline_path), chrome_rgb)
+                        page_result["baseline_metrics"] = baseline_metrics
+                        for key in ("rgb_mae", "pixels_gt_10", "pixels_gt_50"):
+                            if key in current_metrics and key in baseline_metrics:
+                                page_result.setdefault("delta_vs_baseline", {})[key] = round(
+                                    current_metrics[key] - baseline_metrics[key], 6
+                                )
+                manifest["pages"].append(page_result)
+                write_results(results_path, manifest)
+                metric = page_result.get("metrics", {}).get("pixels_gt_50")
+                delta = page_result.get("delta_vs_baseline", {}).get("pixels_gt_50")
+                print(
+                    f"{name:84} p>50={metric if metric is not None else 'capture-fail'} "
+                    f"delta={delta if delta is not None else '-'}",
+                    flush=True,
+                )
+        browser.close()
+
+    manifest["finished_utc"] = datetime.now(timezone.utc).isoformat()
+    write_results(results_path, manifest)
+    failed = [
+        page["name"]
+        for page in manifest["pages"]
+        if not page.get("chromium", {}).get("ok") or not page.get("obscura", {}).get("ok")
+    ]
+    if failed:
+        print(f"capture failures: {', '.join(failed)}", file=sys.stderr)
+        raise SystemExit(1)
+    print(f"paired captures and raw diagnostics: {results_path}")
+
+
+if __name__ == "__main__":
+    main()

render-repros/run.sh

@@ -5,6 +5,7 @@ set -uo pipefail
 ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 BIN="${OBSCURA_BIN:-$ROOT/target/release/obscura}"
 CHROME="${CHROME_BIN:-chromium}"
+PYTHON="${PYTHON_BIN:-python3}"
 DIR="$(cd "$(dirname "$0")" && pwd)"
 OUT="${1:-$DIR/out}"
 mkdir -p "$OUT"
@@ -36,6 +37,11 @@ for f in "$DIR"/*.html; do
   mv "$chrome_shot" "$OUT/$n.chrome.png"
   echo "rendered $n"
 done
+if [[ "$status" -eq 0 ]]; then
+  if ! PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "$DIR/check.py" "$OUT"; then
+    status=1
+  fi
+fi
 if [[ "$status" -eq 0 ]]; then
   echo "output in $OUT"
 else

render-repros/spot-sites.txt

@@ -0,0 +1,9 @@
+# Stable-ish pages for a quick real-world regression spot set.
+https://example.com/
+https://news.ycombinator.com/
+https://www.sqlite.org/index.html
+https://blog.rust-lang.org/
+https://docs.python.org/3/
+https://web.dev/
+https://jvns.ca/
+https://en.wikipedia.org/wiki/Rust_(programming_language)