Add representative live render suite

render-repros/representative-suite/run.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SUITE_DIR/../.." && pwd)"
+PAIRED_CORPUS="$REPO_ROOT/render-repros/paired-corpus.py"
+SITES="$SUITE_DIR/sites.txt"
+OBSCURA="${OBSCURA_BIN:-$REPO_ROOT/target/release/obscura}"
+PYTHON="${PYTHON_BIN:-python3}"
+
+usage() {
+  echo "usage: $0 OUT_DIR [SCROLL_Y]" >&2
+  echo "SCROLL_Y is an integer CSS-pixel offset or 'bottom'." >&2
+}
+
+if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
+  usage
+  exit 0
+fi
+if (( $# < 1 || $# > 2 )); then
+  usage
+  exit 2
+fi
+
+OUT="$1"
+if [[ -e "$OUT" ]]; then
+  echo "output path already exists: $OUT" >&2
+  exit 2
+fi
+if [[ ! -x "$OBSCURA" ]]; then
+  echo "Obscura binary is not executable: $OBSCURA" >&2
+  exit 2
+fi
+
+args=(
+  "$PAIRED_CORPUS"
+  "$SITES"
+  --obscura-bin "$OBSCURA"
+  --out "$OUT"
+  --width 1440
+  --height 1000
+  --settle-ms 3000
+  --animation-time-ms 0
+  --geometry-selector "header, nav, footer"
+  --geometry-selector "main, article"
+  --geometry-selector "section"
+  --geometry-selector "form, fieldset, input, button, select, textarea"
+  --geometry-selector "img, svg, video, canvas"
+  --geometry-selector "pre"
+  --geometry-selector "table"
+)
+
+if [[ -n "${CHROMIUM_BIN:-}" ]]; then
+  args+=(--chromium-bin "$CHROMIUM_BIN")
+fi
+if [[ -n "${BASELINE_BIN:-}" ]]; then
+  args+=(--baseline-bin "$BASELINE_BIN")
+fi
+if (( $# == 2 )); then
+  args+=(--scroll-y "$2")
+fi
+
+exec env PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "${args[@]}"

render-repros/representative-suite/sites.txt

@@ -0,0 +1,20 @@
+# Modern framework applications and documentation.
+https://react.dev/
+https://vuejs.org/
+https://angular.dev/
+https://emberjs.com/
+https://qwik.dev/
+https://www.solidjs.com/
+https://remix.run/
+
+# CSS systems and CSS-heavy documentation.
+https://tailwindcss.com/
+https://bulma.io/
+https://developer.mozilla.org/en-US/docs/Web/CSS
+
+# Commercial and marketing surfaces.
+https://brave.com/
+https://www.apple.com/
+
+# Native and framework-styled form controls.
+https://getbootstrap.com/docs/5.3/forms/overview/

render-repros/representative-suite/test_suite.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+import os
+import stat
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+SUITE_DIR = Path(__file__).resolve().parent
+RUNNER = SUITE_DIR / "run.sh"
+SITES = SUITE_DIR / "sites.txt"
+
+
+class RepresentativeSuiteTests(unittest.TestCase):
+    def run_wrapper(self, *arguments, extra_environment=None):
+        with tempfile.TemporaryDirectory() as temporary_directory:
+            temporary = Path(temporary_directory)
+            obscura = temporary / "obscura"
+            obscura.write_text("#!/usr/bin/env sh\nexit 0\n")
+            obscura.chmod(obscura.stat().st_mode | stat.S_IXUSR)
+
+            invocation = temporary / "invocation.txt"
+            python = temporary / "python"
+            python.write_text(
+                "#!/usr/bin/env sh\n"
+                ": \"${INVOCATION:?}\"\n"
+                "printf '%s\\n' \"$@\" > \"$INVOCATION\"\n"
+            )
+            python.chmod(python.stat().st_mode | stat.S_IXUSR)
+
+            environment = os.environ.copy()
+            environment.update(
+                {
+                    "INVOCATION": str(invocation),
+                    "OBSCURA_BIN": str(obscura),
+                    "PYTHON_BIN": str(python),
+                }
+            )
+            if extra_environment:
+                environment.update(extra_environment)
+
+            output = temporary / "output"
+            result = subprocess.run(
+                ["bash", str(RUNNER), str(output), *arguments],
+                capture_output=True,
+                text=True,
+                env=environment,
+                check=False,
+            )
+            invoked_arguments = (
+                invocation.read_text().splitlines()
+                if invocation.exists()
+                else []
+            )
+            return result, invoked_arguments, str(output), str(obscura)
+
+    def test_default_invocation_pins_capture_conditions_and_generic_probes(self):
+        result, arguments, output, obscura = self.run_wrapper()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(
+            arguments[:2],
+            [
+                str(SUITE_DIR.parent / "paired-corpus.py"),
+                str(SITES),
+            ],
+        )
+        self.assertEqual(
+            arguments[2:12],
+            [
+                "--obscura-bin",
+                obscura,
+                "--out",
+                output,
+                "--width",
+                "1440",
+                "--height",
+                "1000",
+                "--settle-ms",
+                "3000",
+            ],
+        )
+        self.assertIn("--animation-time-ms", arguments)
+        self.assertEqual(arguments[arguments.index("--animation-time-ms") + 1], "0")
+        self.assertEqual(arguments.count("--geometry-selector"), 7)
+        self.assertNotIn("--scroll-y", arguments)
+
+        selectors = [
+            arguments[index + 1]
+            for index, argument in enumerate(arguments)
+            if argument == "--geometry-selector"
+        ]
+        self.assertEqual(
+            selectors,
+            [
+                "header, nav, footer",
+                "main, article",
+                "section",
+                "form, fieldset, input, button, select, textarea",
+                "img, svg, video, canvas",
+                "pre",
+                "table",
+            ],
+        )
+
+    def test_scroll_and_optional_comparison_binaries_are_forwarded(self):
+        result, arguments, _, _ = self.run_wrapper(
+            "bottom",
+            extra_environment={
+                "CHROMIUM_BIN": "/opt/chromium",
+                "BASELINE_BIN": "/opt/obscura-baseline",
+            },
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(arguments[-6:], [
+            "--chromium-bin",
+            "/opt/chromium",
+            "--baseline-bin",
+            "/opt/obscura-baseline",
+            "--scroll-y",
+            "bottom",
+        ])
+
+    def test_site_corpus_is_https_and_has_no_duplicates(self):
+        urls = [
+            line.strip()
+            for line in SITES.read_text().splitlines()
+            if line.strip() and not line.lstrip().startswith("#")
+        ]
+        self.assertEqual(len(urls), 13)
+        self.assertEqual(len(urls), len(set(urls)))
+        self.assertTrue(all(url.startswith("https://") for url in urls))
+
+
+if __name__ == "__main__":
+    unittest.main()