diff --git a/render-repros/representative-suite/run.sh b/render-repros/representative-suite/run.sh new file mode 100755 index 0000000..a212c35 --- /dev/null +++ b/render-repros/representative-suite/run.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SUITE_DIR/../.." && pwd)" +PAIRED_CORPUS="$REPO_ROOT/render-repros/paired-corpus.py" +SITES="$SUITE_DIR/sites.txt" +OBSCURA="${OBSCURA_BIN:-$REPO_ROOT/target/release/obscura}" +PYTHON="${PYTHON_BIN:-python3}" + +usage() { + echo "usage: $0 OUT_DIR [SCROLL_Y]" >&2 + echo "SCROLL_Y is an integer CSS-pixel offset or 'bottom'." >&2 +} + +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + usage + exit 0 +fi +if (( $# < 1 || $# > 2 )); then + usage + exit 2 +fi + +OUT="$1" +if [[ -e "$OUT" ]]; then + echo "output path already exists: $OUT" >&2 + exit 2 +fi +if [[ ! -x "$OBSCURA" ]]; then + echo "Obscura binary is not executable: $OBSCURA" >&2 + exit 2 +fi + +args=( + "$PAIRED_CORPUS" + "$SITES" + --obscura-bin "$OBSCURA" + --out "$OUT" + --width 1440 + --height 1000 + --settle-ms 3000 + --animation-time-ms 0 + --geometry-selector "header, nav, footer" + --geometry-selector "main, article" + --geometry-selector "section" + --geometry-selector "form, fieldset, input, button, select, textarea" + --geometry-selector "img, svg, video, canvas" + --geometry-selector "pre" + --geometry-selector "table" +) + +if [[ -n "${CHROMIUM_BIN:-}" ]]; then + args+=(--chromium-bin "$CHROMIUM_BIN") +fi +if [[ -n "${BASELINE_BIN:-}" ]]; then + args+=(--baseline-bin "$BASELINE_BIN") +fi +if (( $# == 2 )); then + args+=(--scroll-y "$2") +fi + +exec env PYTHONDONTWRITEBYTECODE=1 "$PYTHON" "${args[@]}" diff --git a/render-repros/representative-suite/sites.txt b/render-repros/representative-suite/sites.txt new file mode 100644 index 0000000..68ba19c --- /dev/null +++ b/render-repros/representative-suite/sites.txt @@ -0,0 +1,20 @@ +# Modern framework applications and documentation. +https://react.dev/ +https://vuejs.org/ +https://angular.dev/ +https://emberjs.com/ +https://qwik.dev/ +https://www.solidjs.com/ +https://remix.run/ + +# CSS systems and CSS-heavy documentation. +https://tailwindcss.com/ +https://bulma.io/ +https://developer.mozilla.org/en-US/docs/Web/CSS + +# Commercial and marketing surfaces. +https://brave.com/ +https://www.apple.com/ + +# Native and framework-styled form controls. +https://getbootstrap.com/docs/5.3/forms/overview/ diff --git a/render-repros/representative-suite/test_suite.py b/render-repros/representative-suite/test_suite.py new file mode 100644 index 0000000..1fe6a00 --- /dev/null +++ b/render-repros/representative-suite/test_suite.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +import os +import stat +import subprocess +import tempfile +import unittest +from pathlib import Path + + +SUITE_DIR = Path(__file__).resolve().parent +RUNNER = SUITE_DIR / "run.sh" +SITES = SUITE_DIR / "sites.txt" + + +class RepresentativeSuiteTests(unittest.TestCase): + def run_wrapper(self, *arguments, extra_environment=None): + with tempfile.TemporaryDirectory() as temporary_directory: + temporary = Path(temporary_directory) + obscura = temporary / "obscura" + obscura.write_text("#!/usr/bin/env sh\nexit 0\n") + obscura.chmod(obscura.stat().st_mode | stat.S_IXUSR) + + invocation = temporary / "invocation.txt" + python = temporary / "python" + python.write_text( + "#!/usr/bin/env sh\n" + ": \"${INVOCATION:?}\"\n" + "printf '%s\\n' \"$@\" > \"$INVOCATION\"\n" + ) + python.chmod(python.stat().st_mode | stat.S_IXUSR) + + environment = os.environ.copy() + environment.update( + { + "INVOCATION": str(invocation), + "OBSCURA_BIN": str(obscura), + "PYTHON_BIN": str(python), + } + ) + if extra_environment: + environment.update(extra_environment) + + output = temporary / "output" + result = subprocess.run( + ["bash", str(RUNNER), str(output), *arguments], + capture_output=True, + text=True, + env=environment, + check=False, + ) + invoked_arguments = ( + invocation.read_text().splitlines() + if invocation.exists() + else [] + ) + return result, invoked_arguments, str(output), str(obscura) + + def test_default_invocation_pins_capture_conditions_and_generic_probes(self): + result, arguments, output, obscura = self.run_wrapper() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + arguments[:2], + [ + str(SUITE_DIR.parent / "paired-corpus.py"), + str(SITES), + ], + ) + self.assertEqual( + arguments[2:12], + [ + "--obscura-bin", + obscura, + "--out", + output, + "--width", + "1440", + "--height", + "1000", + "--settle-ms", + "3000", + ], + ) + self.assertIn("--animation-time-ms", arguments) + self.assertEqual(arguments[arguments.index("--animation-time-ms") + 1], "0") + self.assertEqual(arguments.count("--geometry-selector"), 7) + self.assertNotIn("--scroll-y", arguments) + + selectors = [ + arguments[index + 1] + for index, argument in enumerate(arguments) + if argument == "--geometry-selector" + ] + self.assertEqual( + selectors, + [ + "header, nav, footer", + "main, article", + "section", + "form, fieldset, input, button, select, textarea", + "img, svg, video, canvas", + "pre", + "table", + ], + ) + + def test_scroll_and_optional_comparison_binaries_are_forwarded(self): + result, arguments, _, _ = self.run_wrapper( + "bottom", + extra_environment={ + "CHROMIUM_BIN": "/opt/chromium", + "BASELINE_BIN": "/opt/obscura-baseline", + }, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(arguments[-6:], [ + "--chromium-bin", + "/opt/chromium", + "--baseline-bin", + "/opt/obscura-baseline", + "--scroll-y", + "bottom", + ]) + + def test_site_corpus_is_https_and_has_no_duplicates(self): + urls = [ + line.strip() + for line in SITES.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + self.assertEqual(len(urls), 13) + self.assertEqual(len(urls), len(set(urls))) + self.assertTrue(all(url.startswith("https://") for url in urls)) + + +if __name__ == "__main__": + unittest.main()