thebadking/obscura
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: add deterministic render parity checks

Browse Source
This commit is contained in:
SGavrl committed 2026-07-23 10:07:04 +00:00
1 parent 4979606a2f
commit 571c419e51
5 files changed
+427
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

render-repros/check.py
Executable
+139
View File
@@ -0,0 +1,139 @@
#!/usr/bin/env python3
"""Check deterministic repro behavior and report native-coordinate image deltas.
Behavior assertions are based on solid-color component geometry, so font
anti-aliasing does not decide pass/fail. Pair metrics are diagnostics only:
they use the full equal-sized canvas with no registration, cropping, blank-page
exclusion, or opaque aggregate "parity" verdict.
"""
import json
import sys
from pathlib import Path
import numpy as np
from PIL import Image
from scipy.ndimage import find_objects, label
def rgb(value):
value = value.removeprefix("#")
return tuple(int(value[i : i + 2], 16) for i in (0, 2, 4))
def components(array, color):
mask = np.all(array == color, axis=2)
labels, _ = label(mask)
found = []
for slices in find_objects(labels):
if slices is None:
continue
height = slices[0].stop - slices[0].start
width = slices[1].stop - slices[1].start
area = int(mask[slices].sum())
if area >= 20:
found.append({"width": width, "height": height, "area": area})
return found
def bbox(mask):
ys, xs = np.nonzero(mask)
if not len(xs):
return None
return [int(xs.min()), int(ys.min()), int(xs.max()) + 1, int(ys.max()) + 1]
def pair_metrics(ours, chromium):
if ours.shape != chromium.shape:
return {"size_mismatch": [list(ours.shape), list(chromium.shape)]}
delta = np.abs(ours.astype(np.int16) - chromium.astype(np.int16))
max_channel = delta.max(axis=2)
ours_ink = np.any(ours < 245, axis=2)
chromium_ink = np.any(chromium < 245, axis=2)
ours_bbox = bbox(ours_ink)
chromium_bbox = bbox(chromium_ink)
bbox_delta = None
if ours_bbox and chromium_bbox:
bbox_delta = max(abs(a - b) for a, b in zip(ours_bbox, chromium_bbox))
height, width = ours_ink.shape
row_projection = float(
np.abs(ours_ink.sum(axis=1) - chromium_ink.sum(axis=1)).mean() / width
)
col_projection = float(
np.abs(ours_ink.sum(axis=0) - chromium_ink.sum(axis=0)).mean() / height
)
return {
"rgb_mae": round(float(delta.mean() / 255.0), 6),
"pixels_gt_10": round(float((max_channel > 10).mean()), 6),
"pixels_gt_50": round(float((max_channel > 50).mean()), 6),
"ours_content_bbox": ours_bbox,
"chromium_content_bbox": chromium_bbox,
"content_bbox_max_delta": bbox_delta,
"row_projection_delta": round(row_projection, 6),
"column_projection_delta": round(col_projection, 6),
}
def main():
if len(sys.argv) != 2:
raise SystemExit("usage: check.py OUTDIR")
out = Path(sys.argv[1])
checks = json.loads((Path(__file__).with_name("checks.json")).read_text())
failures = []
report = {"fixtures": {}}
for ours_path in sorted(out.glob("*.obscura.png")):
name = ours_path.name.removesuffix(".obscura.png")
chromium_path = out / f"{name}.chrome.png"
if not chromium_path.is_file():
failures.append(f"{name}: missing Chromium screenshot")
continue
ours = np.asarray(Image.open(ours_path).convert("RGB"))
chromium = np.asarray(Image.open(chromium_path).convert("RGB"))
fixture = {
"metrics": pair_metrics(ours, chromium),
"behavior": {"obscura": [], "chromium": []},
}
for engine, array in (("obscura", ours), ("chromium", chromium)):
for check in checks.get(name, []):
expected_count = check.get("count", 1)
matches = []
for component in components(array, rgb(check["color"])):
width_ok = "width" not in check or abs(component["width"] - check["width"]) <= 1
height_ok = "height" not in check or abs(component["height"] - check["height"]) <= 1
if width_ok and height_ok:
matches.append(component)
passed = len(matches) >= expected_count
fixture["behavior"][engine].append(
{
"name": check["name"],
"passed": passed,
"expected_count": expected_count,
"matches": matches,
}
)
if not passed:
failures.append(
f"{name} {engine}: {check['name']} expected {expected_count}, found {len(matches)}"
)
report["fixtures"][name] = fixture
metrics = fixture["metrics"]
print(
f"{name:24} rgb_mae={metrics.get('rgb_mae', 'size-mismatch'):>8} "
f"p>50={metrics.get('pixels_gt_50', '-'):>8} "
f"bbox={metrics.get('content_bbox_max_delta', '-'):>3} "
f"row={metrics.get('row_projection_delta', '-'):>8} "
f"col={metrics.get('column_projection_delta', '-'):>8}"
)
report["failures"] = failures
(out / "analysis.json").write_text(json.dumps(report, indent=2) + "\n")
if failures:
for failure in failures:
print(f"FAILED behavior: {failure}", file=sys.stderr)
raise SystemExit(1)
print(f"behavior checks passed; diagnostics in {out / 'analysis.json'}")
if __name__ == "__main__":
main()
render-repros/checks.json
+35
View File
@@ -0,0 +1,35 @@
{
"abs-fill": [
{"name": "blue fill", "color": "#0000ff", "width": 400, "height": 100},
{"name": "teal fill", "color": "#008080", "width": 400, "height": 100},
{"name": "purple fill", "color": "#800080", "width": 400, "height": 100}
],
"box-sizing": [
{"name": "content-box fixed", "color": "#22aa55", "width": 220, "height": 90},
{"name": "border-box fixed", "color": "#228866", "width": 190, "height": 60},
{"name": "content-box percent", "color": "#668822", "width": 220, "height": 70},
{"name": "border-box percent", "color": "#668822", "width": 190, "height": 40},
{"name": "inherited reset border-box", "color": "#aa3377", "width": 190, "height": 60}
],
"carousel-clip": [
{"name": "visible translated slide", "color": "#33aa33", "width": 300, "height": 100},
{"name": "visible control slide", "color": "#ee3333", "width": 300, "height": 100}
],
"cascade-priority": [
{"name": "important winners", "color": "#008000", "width": 320, "height": 36, "count": 6}
],
"clear-float": [
{"name": "float", "color": "#88bbdd", "width": 120, "height": 80},
{"name": "cleared block", "color": "#ffcc66", "width": 500, "height": 40}
],
"mixed-content-block": [
{"name": "full-width block child", "color": "#ccddff", "width": 400}
],
"tables": [
{"name": "content-box fixed cell column", "color": "#ffcc99", "width": 208}
],
"z-index": [
{"name": "positive layer", "color": "#008000", "width": 200, "height": 100},
{"name": "negative layer", "color": "#ffa500", "width": 260, "height": 110}
]
}
render-repros/paired-corpus.py
Executable
+238
View File
@@ -0,0 +1,238 @@
#!/usr/bin/env python3
"""Capture Obscura and Chromium concurrently with the same real settle delay.
Every run uses a new output directory, checks process status and non-empty
screenshots, records browser versions and timings, and reports the raw
full-canvas metrics from check.py. It deliberately emits no aggregate parity
verdict. An optional pre-change Obscura binary can be captured concurrently so
regressions are compared against the same live-page moment.
"""
import argparse
import concurrent.futures
import json
import os
import re
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
import numpy as np
from PIL import Image
from playwright.sync_api import sync_playwright
from check import pair_metrics
def slug(url):
value = re.sub(r"[^a-z0-9]+", "-", url.lower()).strip("-")
return value[:80] or "page"
def binary_version(binary):
result = subprocess.run(
[binary, "--version"], capture_output=True, text=True, timeout=10
)
text = (result.stdout or result.stderr).strip()
return {"status": result.returncode, "text": text}
def capture_obscura(binary, url, screenshot, log, width, height, settle_ms):
env = dict(
os.environ,
OBSCURA_SHOT_W=str(width),
OBSCURA_SHOT_H=str(height),
OBSCURA_ALLOW_PRIVATE_NETWORK="1",
)
command = [
binary,
"fetch",
url,
"--screenshot",
str(screenshot),
"--timeout",
"50000",
"--wait",
f"{settle_ms / 1000:g}",
]
started = time.time()
try:
result = subprocess.run(
command, capture_output=True, text=True, timeout=75, env=env
)
log.write_text(result.stdout + result.stderr)
ok = result.returncode == 0 and screenshot.is_file() and screenshot.stat().st_size > 0
return {
"ok": ok,
"status": result.returncode,
"elapsed_s": round(time.time() - started, 3),
}
except subprocess.TimeoutExpired as error:
text = (error.stdout or "") + (error.stderr or "")
log.write_text(text if isinstance(text, str) else text.decode(errors="replace"))
return {"ok": False, "status": "timeout", "elapsed_s": round(time.time() - started, 3)}
def load_rgb(path):
return np.asarray(Image.open(path).convert("RGB"))
def write_results(path, manifest):
path.write_text(json.dumps(manifest, indent=2) + "\n")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("urls", help="one URL per line; # comments allowed")
parser.add_argument("--obscura-bin", required=True)
parser.add_argument("--baseline-bin")
parser.add_argument(
"--chromium-bin",
help="Chromium executable (default: Playwright's pinned Chromium)",
)
parser.add_argument("--out", required=True, help="must not already exist")
parser.add_argument("--width", type=int, default=1280)
parser.add_argument("--height", type=int, default=1400)
parser.add_argument("--settle-ms", type=int, default=3000)
args = parser.parse_args()
out = Path(args.out)
out.mkdir(parents=True, exist_ok=False)
urls = [
line.strip()
for line in Path(args.urls).read_text().splitlines()
if line.strip() and not line.lstrip().startswith("#")
]
manifest = {
"started_utc": datetime.now(timezone.utc).isoformat(),
"viewport": {"width": args.width, "height": args.height, "dpr": 1},
"settle_ms_after_load": args.settle_ms,
"obscura": binary_version(args.obscura_bin),
"baseline": binary_version(args.baseline_bin) if args.baseline_bin else None,
"pages": [],
}
results_path = out / "results.json"
write_results(results_path, manifest)
with sync_playwright() as playwright:
chromium_executable = args.chromium_bin or playwright.chromium.executable_path
manifest["chromium_executable"] = chromium_executable
browser = playwright.chromium.launch(
executable_path=chromium_executable,
headless=True,
args=[
"--no-sandbox",
"--hide-scrollbars",
"--disable-background-networking",
"--force-device-scale-factor=1",
],
)
manifest["chromium_version"] = browser.version
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
for index, url in enumerate(urls):
name = f"{index:03d}-{slug(url)}"
ours_path = out / f"{name}.obscura.png"
chrome_path = out / f"{name}.chrome.png"
baseline_path = out / f"{name}.baseline.png"
page_result = {"url": url, "name": name}
context = browser.new_context(
viewport={"width": args.width, "height": args.height},
device_scale_factor=1,
color_scheme="light",
reduced_motion="no-preference",
locale="en-US",
timezone_id="UTC",
)
page = context.new_page()
chrome_messages = []
page.on("console", lambda message: chrome_messages.append(f"console {message.type}: {message.text}"))
page.on("pageerror", lambda error: chrome_messages.append(f"pageerror: {error}"))
ours_future = executor.submit(
capture_obscura,
args.obscura_bin,
url,
ours_path,
out / f"{name}.obscura.log",
args.width,
args.height,
args.settle_ms,
)
baseline_future = None
if args.baseline_bin:
baseline_future = executor.submit(
capture_obscura,
args.baseline_bin,
url,
baseline_path,
out / f"{name}.baseline.log",
args.width,
args.height,
args.settle_ms,
)
chrome_started = time.time()
try:
page.goto(url, wait_until="load", timeout=50000)
page.wait_for_timeout(args.settle_ms)
page.screenshot(
path=str(chrome_path), full_page=False, timeout=50000
)
chrome_ok = chrome_path.is_file() and chrome_path.stat().st_size > 0
chrome_status = 0
except Exception as error:
chrome_messages.append(f"capture error: {error}")
chrome_ok = False
chrome_status = "error"
(out / f"{name}.chrome.log").write_text("\n".join(chrome_messages) + "\n")
page_result["chromium"] = {
"ok": chrome_ok,
"status": chrome_status,
"elapsed_s": round(time.time() - chrome_started, 3),
"title": page.title() if chrome_ok else None,
}
context.close()
page_result["obscura"] = ours_future.result()
if baseline_future:
page_result["baseline"] = baseline_future.result()
if chrome_ok and page_result["obscura"]["ok"]:
chrome_rgb = load_rgb(chrome_path)
current_metrics = pair_metrics(load_rgb(ours_path), chrome_rgb)
page_result["metrics"] = current_metrics
if baseline_future and page_result["baseline"]["ok"]:
baseline_metrics = pair_metrics(load_rgb(baseline_path), chrome_rgb)
page_result["baseline_metrics"] = baseline_metrics
for key in ("rgb_mae", "pixels_gt_10", "pixels_gt_50"):
if key in current_metrics and key in baseline_metrics:
page_result.setdefault("delta_vs_baseline", {})[key] = round(
current_metrics[key] - baseline_metrics[key], 6
)
manifest["pages"].append(page_result)
write_results(results_path, manifest)
metric = page_result.get("metrics", {}).get("pixels_gt_50")
delta = page_result.get("delta_vs_baseline", {}).get("pixels_gt_50")
print(
f"{name:84} p>50={metric if metric is not None else 'capture-fail'} "
f"delta={delta if delta is not None else '-'}",
flush=True,
)
browser.close()
manifest["finished_utc"] = datetime.now(timezone.utc).isoformat()
write_results(results_path, manifest)
failed = [
page["name"]
for page in manifest["pages"]
if not page.get("chromium", {}).get("ok") or not page.get("obscura", {}).get("ok")
]
if failed:
print(f"capture failures: {', '.join(failed)}", file=sys.stderr)
raise SystemExit(1)
print(f"paired captures and raw diagnostics: {results_path}")
if __name__ == "__main__":
main()
render-repros/run.sh
+6
View File
@@ -5,6 +5,7 @@ set -uo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
BIN="${OBSCURA_BIN:-$ROOT/target/release/obscura}"
CHROME="${CHROME_BIN:-chromium}"
PYTHON="${PYTHON_BIN:-python3}"
DIR="$(cd "$(dirname "$0")" && pwd)"
OUT="${1:-$DIR/out}"
mkdir -p "$OUT"
@@ -36,6 +37,11 @@ for f in "$DIR"/*.html; do
mv "$chrome_shot" "$OUT/$n.chrome.png"
echo "rendered $n"
done
if [[ "$status" -eq 0 ]]; then
if ! PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "$DIR/check.py" "$OUT"; then
status=1
fi
fi
if [[ "$status" -eq 0 ]]; then
echo "output in $OUT"
else
render-repros/spot-sites.txt
+9
View File
@@ -0,0 +1,9 @@
# Stable-ish pages for a quick real-world regression spot set.
https://example.com/
https://news.ycombinator.com/
https://www.sqlite.org/index.html
https://blog.rust-lang.org/
https://docs.python.org/3/
https://web.dev/
https://jvns.ca/
https://en.wikipedia.org/wiki/Rust_(programming_language)