From a3b56f4f70313d8f1ddb49c60acd4f2139d4eee1 Mon Sep 17 00:00:00 2001 From: SGavrl <277843151+SGavrl@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:51:42 +0000 Subject: [PATCH] Exclude contentless screenshot pairs from fidelity --- render-repros/check.py | 17 +++++++++++++++++ render-repros/paired-corpus.py | 17 ++++++++++++++++- render-repros/test_check.py | 10 ++++++++++ render-repros/test_paired_corpus.py | 19 +++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/render-repros/check.py b/render-repros/check.py index 3e6fe0c..a21f4ed 100755 --- a/render-repros/check.py +++ b/render-repros/check.py @@ -130,6 +130,16 @@ def pair_metrics(ours, chromium): # low-amplitude texture, and font antialiasing differences. ours_edges = structural_edges(ours) chromium_edges = structural_edges(chromium) + ours_luminance = ( + ours[:, :, 0].astype(np.float32) * 0.2126 + + ours[:, :, 1].astype(np.float32) * 0.7152 + + ours[:, :, 2].astype(np.float32) * 0.0722 + ) + chromium_luminance = ( + chromium[:, :, 0].astype(np.float32) * 0.2126 + + chromium[:, :, 1].astype(np.float32) * 0.7152 + + chromium[:, :, 2].astype(np.float32) * 0.0722 + ) ours_edge_bbox = bbox(ours_edges) chromium_edge_bbox = bbox(chromium_edges) edge_bbox_delta = None @@ -162,6 +172,13 @@ def pair_metrics(ours, chromium): "edge_bbox_max_delta": edge_bbox_delta, "edge_row_projection_delta": round(edge_row_projection, 6), "edge_column_projection_delta": round(edge_col_projection, 6), + # A matching solid canvas is not evidence of rendering fidelity. Keep + # enough per-engine signal for the paired harness to exclude blank or + # otherwise contentless captures without discarding their raw deltas. + "ours_luminance_stddev": round(float(ours_luminance.std()), 6), + "chromium_luminance_stddev": round(float(chromium_luminance.std()), 6), + "ours_structural_edge_pixels": int(ours_edges.sum()), + "chromium_structural_edge_pixels": int(chromium_edges.sum()), } metrics.update(edge_distance) return metrics diff --git a/render-repros/paired-corpus.py b/render-repros/paired-corpus.py index bbcbc04..675d4c0 100755 --- a/render-repros/paired-corpus.py +++ b/render-repros/paired-corpus.py @@ -1645,7 +1645,9 @@ def classify_state_comparability( } -def classify_fidelity_metric(capture_purpose, state_comparable, metrics_present): +def classify_fidelity_metric( + capture_purpose, state_comparable, metrics_present, metrics=None +): """Return whether raw image metrics are valid representative evidence.""" reasons = [] if capture_purpose != "representative-fidelity": @@ -1654,6 +1656,17 @@ def classify_fidelity_metric(capture_purpose, state_comparable, metrics_present) reasons.append("page-state-not-comparable") if not metrics_present: reasons.append("image-metrics-unavailable") + metrics = metrics or {} + contentless_pair = ( + metrics.get("ours_structural_edge_pixels") == 0 + and metrics.get("chromium_structural_edge_pixels") == 0 + and isinstance(metrics.get("ours_luminance_stddev"), (int, float)) + and isinstance(metrics.get("chromium_luminance_stddev"), (int, float)) + and metrics["ours_luminance_stddev"] <= 0.5 + and metrics["chromium_luminance_stddev"] <= 0.5 + ) + if contentless_pair: + reasons.append("contentless-image-pair") return { "fidelity_metric_valid": not reasons, "exclusion_reasons": reasons, @@ -2734,6 +2747,7 @@ def main(): args.capture_purpose, page_result.get("state_comparable"), bool(page_result.get("metrics")), + page_result.get("metrics"), ) page_result["fidelity_metric_valid"] = fidelity_classification[ "fidelity_metric_valid" @@ -2746,6 +2760,7 @@ def main(): args.capture_purpose, page_result.get("baseline_state_comparable"), bool(page_result.get("baseline_metrics")), + page_result.get("baseline_metrics"), ) page_result["baseline_fidelity_metric_valid"] = ( baseline_fidelity_classification["fidelity_metric_valid"] diff --git a/render-repros/test_check.py b/render-repros/test_check.py index f6f9267..4ecac06 100644 --- a/render-repros/test_check.py +++ b/render-repros/test_check.py @@ -31,6 +31,16 @@ class PairMetricsTests(unittest.TestCase): self.assertEqual(metrics["edge_bidirectional_mean_distance_px"], 0.0) self.assertEqual(metrics["edge_bidirectional_p95_distance_px"], 0.0) + def test_solid_pair_records_no_visual_signal(self): + image = np.zeros((80, 80, 3), dtype=np.uint8) + + metrics = pair_metrics(image, image) + + self.assertEqual(metrics["ours_luminance_stddev"], 0.0) + self.assertEqual(metrics["chromium_luminance_stddev"], 0.0) + self.assertEqual(metrics["ours_structural_edge_pixels"], 0) + self.assertEqual(metrics["chromium_structural_edge_pixels"], 0) + if __name__ == "__main__": unittest.main() diff --git a/render-repros/test_paired_corpus.py b/render-repros/test_paired_corpus.py index 499e674..ff17b03 100644 --- a/render-repros/test_paired_corpus.py +++ b/render-repros/test_paired_corpus.py @@ -320,6 +320,25 @@ class StateComparabilityTests(unittest.TestCase): self.assertFalse(comparison["hashes_used_for_classification"]) self.assertTrue(fidelity["fidelity_metric_valid"]) + def test_matching_contentless_canvases_are_not_fidelity_evidence(self): + metrics = { + "pixels_gt_50": 0.0, + "ours_luminance_stddev": 0.0, + "chromium_luminance_stddev": 0.0, + "ours_structural_edge_pixels": 0, + "chromium_structural_edge_pixels": 0, + } + + fidelity = paired_corpus.classify_fidelity_metric( + "representative-fidelity", + True, + metrics_present=True, + metrics=metrics, + ) + + self.assertFalse(fidelity["fidelity_metric_valid"]) + self.assertIn("contentless-image-pair", fidelity["exclusion_reasons"]) + def test_live_and_deterministic_modes_with_same_dom_provenance_are_comparable(self): live = self.state( 210,