diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..1ee2980 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,8 @@ +# SGavrl is the primary maintainer and default owner for the repository. +* @SGavrl + +# Keep security-sensitive automation explicit even if more owners are added later. +/.github/CODEOWNERS @SGavrl +/.github/workflows/ @SGavrl +/scripts/ci/ @SGavrl +/deny.toml @SGavrl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7dae818 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,277 @@ +name: PR checks + +on: + pull_request: + merge_group: + push: + branches: [main] + +# Pull requests execute contributor-controlled build scripts, proc macros, +# tests, and binaries. They must never receive a write token or repository +# secrets. Publishing remains isolated in the tag-only release workflows. +permissions: + contents: read + +concurrency: + group: pr-checks-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + CARGO_BUILD_JOBS: "2" + CARGO_INCREMENTAL: "0" + CARGO_TERM_COLOR: always + OPENSSL_NO_VENDOR: "1" + +jobs: + policy: + name: PR / Policy + runs-on: ubuntu-22.04 + timeout-minutes: 20 + outputs: + base-sha: ${{ steps.base.outputs.sha }} + steps: + - name: Checkout merge result + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Resolve trusted base revision + id: base + shell: bash + env: + EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + run: | + set -euo pipefail + base_sha="$EVENT_BASE_SHA" + if [[ ! "$base_sha" =~ ^[0-9a-f]{40}$ ]] || [[ "$base_sha" == "0000000000000000000000000000000000000000" ]]; then + echo "GitHub did not provide a valid base revision" >&2 + exit 1 + fi + git cat-file -e "${base_sha}^{commit}" + + policy_root="$RUNNER_TEMP/obscura-policy" + mkdir -p "$policy_root" + if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then + install -m 0644 scripts/ci/pr_policy.py "$policy_root/pr_policy.py" + install -m 0644 deny.toml "$policy_root/deny.toml" + else + git show "$base_sha:scripts/ci/pr_policy.py" > "$policy_root/pr_policy.py" + git show "$base_sha:deny.toml" > "$policy_root/deny.toml" + fi + + # Container actions can only read paths mounted below the workspace. + # Replace any PR-supplied file before exposing the trusted config. + trusted_deny="$GITHUB_WORKSPACE/.github/.trusted-deny.toml" + rm -f -- "$trusted_deny" + install -m 0644 "$policy_root/deny.toml" "$trusted_deny" + printf 'sha=%s\n' "$base_sha" >> "$GITHUB_OUTPUT" + printf 'policy=%s\n' "$policy_root/pr_policy.py" >> "$GITHUB_OUTPUT" + + - name: Check PR policy + env: + BASE_SHA: ${{ steps.base.outputs.sha }} + run: python3 "${{ steps.base.outputs.policy }}" --base "$BASE_SHA" --head "$GITHUB_SHA" + + - name: Check dependency advisories, licenses, bans, and sources + uses: EmbarkStudios/cargo-deny-action@3c6349835b2b7b196a839186cb8b78e02f7b5f25 # v2 + with: + command: check + arguments: --all-features + command-arguments: --config .github/.trusted-deny.toml advisories bans licenses sources + + core: + name: PR / Core + needs: policy + runs-on: ubuntu-22.04 + timeout-minutes: 90 + steps: + - name: Checkout merge result + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + clang cmake libclang-dev libssl-dev llvm-dev pkg-config + + - name: Install Rust + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable + + - name: Install cargo-nextest + uses: taiki-e/install-action@b20dedce73af6905cdc30d6611090c9b67557c8d # v2 + with: + tool: nextest + fallback: none + + - name: Restore Rust dependency cache + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + # PR caches are restored but never saved. A contributor therefore + # cannot poison a cache later consumed by main or a release. + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Build render release + run: cargo build --release -p obscura-cli --bins --features render + + - name: Test render configuration + run: cargo nextest run --release --features render --no-fail-fast + + - name: Stage candidate binary for comparison + if: github.event_name != 'push' + run: install -D -m 0755 target/release/obscura "$RUNNER_TEMP/obscura-ci/obscura-pr" + + - name: Build render and stealth release + run: cargo build --release -p obscura-cli --bins --features render,stealth + + - name: Test the stealth transport + run: | + # The gzip fixture binds loopback. Keep the ordinary suite under the + # default SSRF policy, then opt in only for that single focused test. + cargo nextest run --release -p obscura-net --features stealth \ + --no-fail-fast -- --skip stealth_client_decodes_gzip_response + OBSCURA_ALLOW_PRIVATE_NETWORK=1 cargo nextest run --release \ + -p obscura-net --features stealth \ + stealth_client_decodes_gzip_response --no-fail-fast + + - name: Build no-render release + run: cargo build --release -p obscura-cli --bins --no-default-features + + - name: Test no-render configuration + run: | + # `obscura` uses its default `api` feature for the public library; an + # empty feature set is not a supported crate configuration. Test that + # no-render API normally and every non-render crate with defaults + # disabled. `obscura-render` has paint-only test fixtures, so verify + # its feature-minimal library separately without compiling those + # fixtures. + cargo nextest run --release -p obscura --no-fail-fast + cargo nextest run --release --workspace --exclude obscura \ + --exclude obscura-render \ + --no-default-features --no-fail-fast + cargo check --release -p obscura-render --no-default-features + + - name: Build no-render and stealth release + run: cargo build --release -p obscura-cli --bins --no-default-features --features stealth + + - name: Upload candidate binary + if: github.event_name != 'push' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: obscura-pr-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/obscura-ci/obscura-pr + if-no-files-found: error + retention-days: 1 + + performance: + name: PR / Performance + if: github.event_name != 'push' + needs: [policy, core] + runs-on: ubuntu-22.04 + timeout-minutes: 75 + steps: + - name: Checkout merge result + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Checkout pinned obstacle course + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: h4ckf0r0day/obscura-benchmark + ref: 6ebac8293d7477f59e837768bfd4e74173f04f1c + path: obscura-benchmark + persist-credentials: false + + - name: Download candidate binary + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: obscura-pr-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/obscura-ci + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + clang cmake libclang-dev libssl-dev llvm-dev pkg-config time + + - name: Install Rust + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable + + - name: Restore Rust dependency cache + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 + with: + save-if: false + + - name: Build base revision + shell: bash + env: + BASE_SHA: ${{ needs.policy.outputs.base-sha }} + run: | + set -euo pipefail + base_root="$RUNNER_TEMP/obscura-base" + script_root="$RUNNER_TEMP/obscura-ci-scripts" + git worktree add --detach "$base_root" "$BASE_SHA" + mkdir -p "$script_root" + git show "$BASE_SHA:scripts/ci/perf_smoke.py" > "$script_root/perf_smoke.py" + git show "$BASE_SHA:scripts/ci/compare_obstacle.py" > "$script_root/compare_obstacle.py" + # Run from the trusted worktree so a PR-supplied .cargo/config.toml in + # the merge checkout cannot alter the compiler or wrapper for the base. + ( + cd "$base_root" + CARGO_TARGET_DIR="$RUNNER_TEMP/obscura-base-target" \ + cargo build --release -p obscura-cli --bins --features render + ) + install -D -m 0755 \ + "$RUNNER_TEMP/obscura-base-target/release/obscura" \ + "$RUNNER_TEMP/obscura-ci/obscura-base" + chmod 0755 "$RUNNER_TEMP/obscura-ci/obscura-pr" + + - name: Run base and candidate obstacle courses + shell: bash + run: | + set -euo pipefail + results_root="$RUNNER_TEMP/obscura-ci-results" + mkdir -p "$results_root" + set +e + OBSCURA_BIN="$RUNNER_TEMP/obscura-ci/obscura-base" \ + python3 obscura-benchmark/obstacle-course/run.py \ + --json --runs 3 --warmup 1 > "$results_root/obstacle-base.json" + base_status=$? + OBSCURA_BIN="$RUNNER_TEMP/obscura-ci/obscura-pr" \ + python3 obscura-benchmark/obstacle-course/run.py \ + --json --runs 3 --warmup 1 > "$results_root/obstacle-pr.json" + pr_status=$? + set -e + printf 'Obstacle commands exited base=%s candidate=%s\n' "$base_status" "$pr_status" + python3 -m json.tool "$results_root/obstacle-base.json" >/dev/null + python3 -m json.tool "$results_root/obstacle-pr.json" >/dev/null + if [[ "$pr_status" -ne 0 ]]; then + echo "Candidate obstacle course exited with status $pr_status" >&2 + exit "$pr_status" + fi + + - name: Run interleaved latency and RSS smoke benchmark + run: | + python3 "$RUNNER_TEMP/obscura-ci-scripts/perf_smoke.py" \ + --base "$RUNNER_TEMP/obscura-ci/obscura-base" \ + --candidate "$RUNNER_TEMP/obscura-ci/obscura-pr" \ + --output "$RUNNER_TEMP/obscura-ci-results/perf-smoke.json" + + - name: Compare obstacle correctness and latency + run: | + python3 "$RUNNER_TEMP/obscura-ci-scripts/compare_obstacle.py" \ + --base "$RUNNER_TEMP/obscura-ci-results/obstacle-base.json" \ + --candidate "$RUNNER_TEMP/obscura-ci-results/obstacle-pr.json" + + - name: Upload comparison evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: pr-comparison-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/obscura-ci-results + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1db9926..e294f7f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,16 +12,18 @@ jobs: build-and-push: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Login to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -31,7 +33,7 @@ jobs: run: echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - name: Build and push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . push: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5ca36f..53fbc6d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,7 @@ on: - 'v*' permissions: - contents: write + contents: read jobs: build: @@ -44,9 +44,11 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable with: targets: ${{ matrix.target }} @@ -179,10 +181,11 @@ jobs: cd ../no-render-stealth 7z a ../../${{ matrix.name }}-no-render-stealth.zip obscura.exe obscura-worker.exe - - name: Upload - uses: softprops/action-gh-release@v2 + - name: Upload packaged binaries + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - files: | + name: release-${{ matrix.name }} + path: | ${{ matrix.name }}.tar.gz ${{ matrix.name }}-stealth.tar.gz ${{ matrix.name }}-no-render.tar.gz @@ -191,3 +194,25 @@ jobs: ${{ matrix.name }}-stealth.zip ${{ matrix.name }}-no-render.zip ${{ matrix.name }}-no-render-stealth.zip + if-no-files-found: error + retention-days: 1 + + publish: + needs: build + runs-on: ubuntu-22.04 + permissions: + contents: write + steps: + # This job never checks out or executes repository code. The write token is + # available only while collecting and publishing the completed artifacts. + - name: Download packaged binaries + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: release-* + path: dist + merge-multiple: true + + - name: Publish GitHub release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + files: dist/* diff --git a/Cargo.lock b/Cargo.lock index 7decb18..82d2e4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,9 +141,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arrayref" diff --git a/crates/obscura-browser/Cargo.toml b/crates/obscura-browser/Cargo.toml index ab87de7..e2869e4 100644 --- a/crates/obscura-browser/Cargo.toml +++ b/crates/obscura-browser/Cargo.toml @@ -2,6 +2,8 @@ name = "obscura-browser" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true [features] default = [] diff --git a/crates/obscura-cdp/Cargo.toml b/crates/obscura-cdp/Cargo.toml index ae2f989..e0d29ac 100644 --- a/crates/obscura-cdp/Cargo.toml +++ b/crates/obscura-cdp/Cargo.toml @@ -2,6 +2,8 @@ name = "obscura-cdp" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true [features] default = [] diff --git a/crates/obscura-cli/Cargo.toml b/crates/obscura-cli/Cargo.toml index 5bcd6a4..afd44b0 100644 --- a/crates/obscura-cli/Cargo.toml +++ b/crates/obscura-cli/Cargo.toml @@ -2,6 +2,8 @@ name = "obscura-cli" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true [[bin]] name = "obscura" diff --git a/crates/obscura-dom/Cargo.toml b/crates/obscura-dom/Cargo.toml index 58260db..c35a95e 100644 --- a/crates/obscura-dom/Cargo.toml +++ b/crates/obscura-dom/Cargo.toml @@ -2,6 +2,8 @@ name = "obscura-dom" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true [dependencies] html5ever = { workspace = true } diff --git a/crates/obscura-js/Cargo.toml b/crates/obscura-js/Cargo.toml index 28e8e13..ac4e3f8 100644 --- a/crates/obscura-js/Cargo.toml +++ b/crates/obscura-js/Cargo.toml @@ -2,6 +2,8 @@ name = "obscura-js" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true [features] default = [] diff --git a/crates/obscura-mcp/Cargo.toml b/crates/obscura-mcp/Cargo.toml index 87c3aa9..128290c 100644 --- a/crates/obscura-mcp/Cargo.toml +++ b/crates/obscura-mcp/Cargo.toml @@ -2,6 +2,8 @@ name = "obscura-mcp" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true [features] default = [] diff --git a/crates/obscura-net/Cargo.toml b/crates/obscura-net/Cargo.toml index f84435c..1d990c2 100644 --- a/crates/obscura-net/Cargo.toml +++ b/crates/obscura-net/Cargo.toml @@ -2,6 +2,8 @@ name = "obscura-net" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true [features] default = [] diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..dc28f6a --- /dev/null +++ b/deny.toml @@ -0,0 +1,45 @@ +# cargo-deny configuration: https://embarkstudios.github.io/cargo-deny/ +# Run: cargo deny --all-features check --config deny.toml advisories bans licenses sources + +[advisories] +# Real vulnerabilities fail the build. The advisories below are *unmaintained* +# notices for transitive crates with no available fix; they are explicitly +# accepted until the upstream parents drop them. +ignore = [ + "RUSTSEC-2025-0141", # bincode 1.x unmaintained, transitive via deno_core + "RUSTSEC-2025-0057", # fxhash unmaintained, transitive via selectors + "RUSTSEC-2024-0436", # paste unmaintained, transitive via v8 + # These rendering dependencies have no safe upgrade yet. Keep their + # unmaintained notices explicit so vulnerability advisories still fail. + "RUSTSEC-2026-0206", # rustybuzz, transitive via cosmic-text and usvg + "RUSTSEC-2026-0192", # ttf-parser, transitive via rendering dependencies +] + +[bans] +# Duplicate transitive versions are build-time bloat, not a security issue, and +# are largely outside our control (deno_core / v8 / reqwest trees). Surface them +# without failing the build. +multiple-versions = "warn" +wildcards = "allow" + +[licenses] +# Permissive allow-list enforced by the PR policy job. +allow = [ + "MIT", + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "Zlib", + "Unicode-3.0", + "MPL-2.0", + "BSL-1.0", + "CDLA-Permissive-2.0", +] +confidence-threshold = 0.8 + +[sources] +unknown-registry = "deny" +unknown-git = "deny" +allow-registry = ["https://github.com/rust-lang/crates.io-index"] diff --git a/scripts/ci/compare_obstacle.py b/scripts/ci/compare_obstacle.py new file mode 100755 index 0000000..cd994d2 --- /dev/null +++ b/scripts/ci/compare_obstacle.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Compare obstacle-course correctness and report latency changes.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import statistics +from pathlib import Path + + +MAX_RESULT_BYTES = 10 * 1024 * 1024 +EXPECTED_STAGE_COUNT = 33 + + +def load_result(path: Path) -> dict: + if path.stat().st_size > MAX_RESULT_BYTES: + raise ValueError(f"{path} is unexpectedly large") + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict) or not isinstance(value.get("results"), list): + raise ValueError(f"{path} is not an obstacle-course result") + return value + + +def result_map(value: dict) -> dict[str, dict]: + mapped: dict[str, dict] = {} + for result in value["results"]: + if not isinstance(result, dict) or not isinstance(result.get("name"), str): + raise ValueError("obstacle result has an invalid stage") + name = result["name"] + if name in mapped: + raise ValueError(f"duplicate obstacle stage: {name}") + median_ms = result.get("median_ms") + if ( + not isinstance(result.get("pass"), bool) + or isinstance(median_ms, bool) + or not isinstance(median_ms, (int, float)) + or not math.isfinite(median_ms) + or median_ms < 0 + ): + raise ValueError(f"invalid obstacle result for {name}") + mapped[name] = result + return mapped + + +def add_summary(markdown: str) -> None: + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as output: + output.write(markdown) + else: + print(markdown) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + args = parser.parse_args() + + base = result_map(load_result(args.base)) + candidate = result_map(load_result(args.candidate)) + if len(base) != EXPECTED_STAGE_COUNT: + raise SystemExit( + f"base obstacle result has {len(base)} stages; expected {EXPECTED_STAGE_COUNT}" + ) + if len(candidate) != EXPECTED_STAGE_COUNT: + raise SystemExit( + f"candidate obstacle result has {len(candidate)} stages; expected {EXPECTED_STAGE_COUNT}" + ) + if set(base) != set(candidate): + missing = sorted(set(base) - set(candidate)) + extra = sorted(set(candidate) - set(base)) + raise SystemExit(f"obstacle stage mismatch; missing={missing}, extra={extra}") + + regressions = sorted(name for name in base if base[name]["pass"] and not candidate[name]["pass"]) + improvements = sorted(name for name in base if not base[name]["pass"] and candidate[name]["pass"]) + ratios = [] + slow = [] + for name in sorted(base): + base_ms = float(base[name]["median_ms"]) + candidate_ms = float(candidate[name]["median_ms"]) + if base_ms > 0: + ratio = candidate_ms / base_ms + ratios.append(ratio) + if ratio > 1.20 and candidate_ms - base_ms > 10: + slow.append((name, base_ms, candidate_ms, ratio)) + + base_passed = sum(1 for result in base.values() if result["pass"]) + candidate_passed = sum(1 for result in candidate.values() if result["pass"]) + median_ratio = statistics.median(ratios) if ratios else 1.0 + lines = [ + "## Obstacle comparison\n", + "| Metric | Base | Candidate |\n", + "| --- | ---: | ---: |\n", + f"| Correct stages | {base_passed}/{len(base)} | {candidate_passed}/{len(candidate)} |\n", + f"| Median per-stage latency ratio | 1.00x | {median_ratio:.2f}x |\n", + ] + if improvements: + lines.append(f"\nImproved stages: {', '.join(improvements)}\n") + if regressions: + lines.append(f"\nRegressed stages: {', '.join(regressions)}\n") + if slow: + lines.append("\nStages above the reporting threshold:\n\n") + for name, base_ms, candidate_ms, ratio in slow: + lines.append(f"- `{name}`: {base_ms:.1f} ms to {candidate_ms:.1f} ms ({ratio:.2f}x)\n") + print(f"::warning::obstacle latency {name}: {ratio:.2f}x base") + add_summary("".join(lines)) + + if regressions: + raise SystemExit("candidate introduces new obstacle-course failures") + if candidate_passed != EXPECTED_STAGE_COUNT: + raise SystemExit( + f"candidate passes {candidate_passed}/{EXPECTED_STAGE_COUNT} obstacle stages" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/perf_smoke.py b/scripts/ci/perf_smoke.py new file mode 100755 index 0000000..46ad339 --- /dev/null +++ b/scripts/ci/perf_smoke.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Interleaved latency, peak-RSS, and result-correctness comparison.""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import statistics +import subprocess +import tempfile +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread + + +ROUNDS = 5 +PROCESS_TIMEOUT_SECONDS = 45 +LATENCY_FAILURE_RATIO = 1.20 +LATENCY_FAILURE_DELTA_MS = 10.0 +RSS_FAILURE_RATIO = 1.15 +RSS_FAILURE_DELTA_KIB = 8 * 1024 +HTML = b"
" +SCENARIOS = { + "dom-build": ( + "(function(){var r=document.getElementById('root');for(var i=0;i<5000;i++){var e=document.createElement('div');e.className='row';e.textContent='item-'+i;r.appendChild(e);}return r.children.length;})()", + 5000, + ), + "storage": ( + "(function(){for(var i=0;i<2000;i++)localStorage.setItem('k'+i,'value-'+i);var n=0;for(var j=0;j<2000;j++)n+=localStorage.getItem('k'+j).length;return n;})()", + 18890, + ), +} + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(HTML))) + self.end_headers() + self.wfile.write(HTML) + + def log_message(self, _format: str, *_args: object) -> None: + return + + +def run_once(binary: Path, url: str, expression: str, expected: int) -> tuple[float, int]: + with tempfile.NamedTemporaryFile(prefix="obscura-rss-", delete=False) as rss_file: + rss_path = Path(rss_file.name) + with tempfile.NamedTemporaryFile(prefix="obscura-stdout-", delete=False) as stdout_file: + stdout_path = Path(stdout_file.name) + with tempfile.NamedTemporaryFile(prefix="obscura-stderr-", delete=False) as stderr_file: + stderr_path = Path(stderr_file.name) + command = [ + "prlimit", + "--core=0", + "--fsize=2097152", + "--nofile=256", + "--", + "/usr/bin/time", + "-f", + "%M", + "-o", + str(rss_path), + str(binary), + "fetch", + url, + "--allow-private-network", + "--quiet", + "--timeout", + "20", + "--wait", + "0", + "--eval", + expression, + ] + started = time.perf_counter() + try: + with stdout_path.open("wb") as stdout_handle, stderr_path.open("wb") as stderr_handle: + process = subprocess.Popen( + command, + stdout=stdout_handle, + stderr=stderr_handle, + start_new_session=True, + ) + try: + process.wait(timeout=PROCESS_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise RuntimeError(f"{binary} timed out") + finally: + # Remove background descendants left by an untrusted candidate binary. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + elapsed_ms = (time.perf_counter() - started) * 1000 + rss_kib = int(rss_path.read_text(encoding="ascii").strip()) + if process.returncode != 0: + raise RuntimeError(f"{binary} exited with status {process.returncode}") + try: + result = json.loads(stdout_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise RuntimeError(f"{binary} produced an invalid evaluation result") from error + if result != expected: + raise RuntimeError( + f"{binary} returned {result!r}; expected {expected!r}" + ) + return elapsed_ms, rss_kib + finally: + rss_path.unlink(missing_ok=True) + stdout_path.unlink(missing_ok=True) + stderr_path.unlink(missing_ok=True) + + +def median(values: list[float | int]) -> float: + return float(statistics.median(values)) + + +def add_summary(markdown: str) -> None: + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as output: + output.write(markdown) + else: + print(markdown) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + for binary in (args.base, args.candidate): + if not binary.is_file(): + raise SystemExit(f"missing binary: {binary}") + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + url = f"http://127.0.0.1:{server.server_port}/" + samples: dict[str, dict[str, dict[str, list[float | int]]]] = {} + try: + for scenario, (expression, expected) in SCENARIOS.items(): + samples[scenario] = { + "base": {"latency_ms": [], "rss_kib": []}, + "candidate": {"latency_ms": [], "rss_kib": []}, + } + for round_number in range(ROUNDS): + order = ( + (("base", args.base), ("candidate", args.candidate)) + if round_number % 2 == 0 + else (("candidate", args.candidate), ("base", args.base)) + ) + for label, binary in order: + latency_ms, rss_kib = run_once(binary, url, expression, expected) + samples[scenario][label]["latency_ms"].append(latency_ms) + samples[scenario][label]["rss_kib"].append(rss_kib) + finally: + server.shutdown() + server.server_close() + + report: dict[str, object] = {"rounds": ROUNDS, "scenarios": {}} + lines = [ + "## Performance smoke comparison\n\n", + "Fails above 20% and 10 ms for latency, or above 15% and 8 MiB for RSS.\n\n", + "| Scenario | Base latency | PR latency | Change | Base RSS | PR RSS | Change |\n", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |\n", + ] + failures = [] + for scenario in SCENARIOS: + base_latency = median(samples[scenario]["base"]["latency_ms"]) + pr_latency = median(samples[scenario]["candidate"]["latency_ms"]) + base_rss = median(samples[scenario]["base"]["rss_kib"]) + pr_rss = median(samples[scenario]["candidate"]["rss_kib"]) + latency_ratio = pr_latency / base_latency + rss_ratio = pr_rss / base_rss + report["scenarios"][scenario] = { + "base_latency_ms": base_latency, + "candidate_latency_ms": pr_latency, + "latency_ratio": latency_ratio, + "base_rss_kib": base_rss, + "candidate_rss_kib": pr_rss, + "rss_ratio": rss_ratio, + "samples": samples[scenario], + } + lines.append( + f"| {scenario} | {base_latency:.1f} ms | {pr_latency:.1f} ms | " + f"{(latency_ratio - 1) * 100:+.1f}% | {base_rss / 1024:.1f} MiB | " + f"{pr_rss / 1024:.1f} MiB | {(rss_ratio - 1) * 100:+.1f}% |\n" + ) + latency_delta = pr_latency - base_latency + rss_delta = pr_rss - base_rss + if ( + latency_ratio > LATENCY_FAILURE_RATIO + and latency_delta > LATENCY_FAILURE_DELTA_MS + ): + failures.append(f"{scenario} latency is {latency_ratio:.2f}x the base") + if rss_ratio > RSS_FAILURE_RATIO and rss_delta > RSS_FAILURE_DELTA_KIB: + failures.append(f"{scenario} RSS is {rss_ratio:.2f}x the base") + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + add_summary("".join(lines)) + for failure in failures: + print(f"::error::{failure}") + if failures: + raise SystemExit("candidate exceeds the performance regression threshold") + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/pr_policy.py b/scripts/ci/pr_policy.py new file mode 100755 index 0000000..3c5d6a6 --- /dev/null +++ b/scripts/ci/pr_policy.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Fast, deterministic PR policy checks with no shell evaluation of PR data.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path, PurePosixPath + + +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +USES_RE = re.compile(r"^\s*-?\s*uses\s*:\s*(.+?)\s*$") +FULL_ACTION_REF_RE = re.compile(r"^[^@]+@[0-9a-f]{40}$") +PINNED_DOCKER_REF_RE = re.compile(r"^docker://[^@]+@sha256:[0-9a-f]{64}$") +MAX_CONTROL_FILE_BYTES = 2 * 1024 * 1024 +REQUIRED_SECTIONS = ("What changed", "Validation", "Rendering", "Performance", "Checklist") +ALWAYS_BLOCKED_WORKFLOW_ADDITIONS = ( + "pull_request_target", + "workflow_run", +) +WRITE_PERMISSION_RE = re.compile( + r"\b(?:actions|checks|contents|id-token|packages|pull-requests)\s*:\s*['\"]?write['\"]?(?:\s|[,}]|$)" +) +WRITE_ALL_RE = re.compile(r"\bpermissions\s*:\s*['\"]?write-all['\"]?(?:\s|$)") +SECRET_REFERENCE_RE = re.compile(r"\$\{\{\s*secrets\s*(?:\.|\[)") +MAINTAINER_PRIVILEGED_WORKFLOWS = { + ".github/workflows/docker.yml", + ".github/workflows/release.yml", +} +UNTRUSTED_CONTEXT_RE = re.compile( + r"\$\{\{\s*github\.event\.(?:pull_request\.(?:title|body|head\.ref)|" + r"issue\.(?:title|body)|comment\.body|head_commit\.message)" +) +BLOCKED_SUFFIXES = {".dll", ".dylib", ".exe", ".o", ".obj", ".pyc", ".so"} + + +def git(*args: str, text: bool = False) -> bytes | str: + result = subprocess.run( + ["git", *args], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=text, + ) + return result.stdout + + +def annotation(kind: str, message: str) -> None: + safe = message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + print(f"::{kind}::{safe}") + + +def changed_files(base: str, head: str) -> list[str]: + raw = git("diff", "--name-only", "-z", "--diff-filter=ACMR", f"{base}...{head}") + assert isinstance(raw, bytes) + return [part.decode("utf-8", "surrogateescape") for part in raw.split(b"\0") if part] + + +def event_body() -> str | None: + if os.environ.get("GITHUB_EVENT_NAME") != "pull_request": + return None + event_path = os.environ.get("GITHUB_EVENT_PATH") + if not event_path: + return "" + try: + event = json.loads(Path(event_path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return "" + body = event.get("pull_request", {}).get("body") + return body if isinstance(body, str) else "" + + +def check_template(errors: list[str]) -> None: + body = event_body() + if body is None: + return + for heading in REQUIRED_SECTIONS: + pattern = re.compile(rf"(?mi)^##\s+{re.escape(heading)}\s*$") + if not pattern.search(body): + errors.append(f"PR description is missing the '## {heading}' section") + + +def check_artifacts(files: list[str], errors: list[str], warnings: list[str]) -> None: + for name in files: + path = PurePosixPath(name) + parts = set(path.parts) + if "target" in parts or "__pycache__" in parts or path.suffix.lower() in BLOCKED_SUFFIXES: + errors.append(f"generated build artifact is not allowed: {name}") + elif path.suffix.lower() in {".png", ".jpg", ".jpeg", ".webp"} and not name.startswith( + ("crates/obscura-render/assets/", ".github/") + ): + warnings.append(f"review newly committed image evidence: {name}") + + +def check_control_symlinks(files: list[str], errors: list[str]) -> None: + protected = [name for name in files if name.startswith((".github/", "scripts/ci/"))] + if not protected: + return + raw = git("ls-files", "-s", "-z", "--", *protected) + assert isinstance(raw, bytes) + for record in raw.split(b"\0"): + if record.startswith(b"120000 "): + errors.append("symlinks are not allowed in CI control paths") + return + + +def check_new_vendor(base: str, files: list[str], errors: list[str]) -> None: + roots = sorted( + {"/".join(PurePosixPath(name).parts[:2]) for name in files if name.startswith("vendor/")} + ) + for root in roots: + exists = subprocess.run( + ["git", "cat-file", "-e", f"{base}:{root}"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if exists.returncode != 0: + errors.append(f"new vendored dependency requires explicit maintainer handling: {root}") + + +def workflow_patch(base: str, head: str, files: list[str]) -> list[tuple[str, str]]: + workflows = [name for name in files if name.startswith(".github/workflows/")] + additions: list[tuple[str, str]] = [] + for name in workflows: + patch = git("diff", "--unified=0", f"{base}...{head}", "--", name, text=True) + assert isinstance(patch, str) + additions.extend( + (name, line[1:]) + for line in patch.splitlines() + if line.startswith("+") and not line.startswith("+++") + ) + return additions + + +def check_workflows( + base: str, + head: str, + files: list[str], + errors: list[str], + warnings: list[str], +) -> None: + workflows = [name for name in files if name.startswith(".github/workflows/")] + for name in workflows: + path = Path(name) + if path.is_symlink(): + # check_control_symlinks reports the policy violation. Do not follow + # the link: it may target an unbounded device or a runner-local file. + continue + try: + if path.stat().st_size > MAX_CONTROL_FILE_BYTES: + errors.append(f"workflow is unexpectedly large: {name}") + continue + content = path.read_text(encoding="utf-8") + except OSError as error: + errors.append(f"cannot read workflow {name}: {error}") + continue + for line_number, line in enumerate(content.splitlines(), 1): + match = USES_RE.match(line) + if not match: + continue + action = match.group(1).split("#", 1)[0].strip().strip("'\"") + if action.startswith("./"): + continue + if action.startswith("docker://"): + if not PINNED_DOCKER_REF_RE.fullmatch(action): + errors.append( + f"{name}:{line_number} container action is not pinned to a sha256 digest" + ) + continue + if not FULL_ACTION_REF_RE.fullmatch(action): + errors.append(f"{name}:{line_number} action is not pinned to a full commit SHA") + + for name, line in workflow_patch(base, head, files): + compact = line.strip().lower() + if not compact or compact.startswith("#"): + continue + if any(token in compact for token in ALWAYS_BLOCKED_WORKFLOW_ADDITIONS): + errors.append(f"privileged workflow addition requires a separate maintainer-reviewed change: {line.strip()}") + privileged = ( + WRITE_PERMISSION_RE.search(compact) + or WRITE_ALL_RE.search(compact) + or SECRET_REFERENCE_RE.search(compact) + ) + if privileged: + if name in MAINTAINER_PRIVILEGED_WORKFLOWS: + warnings.append(f"CODEOWNER must review privileged publishing change in {name}") + else: + errors.append( + f"privileged workflow addition is not allowed outside publishing workflows: {line.strip()}" + ) + if UNTRUSTED_CONTEXT_RE.search(line): + errors.append("untrusted GitHub metadata must not be interpolated into workflow commands") + + +def check_scope(base: str, head: str, files: list[str], warnings: list[str]) -> None: + shortstat = git("diff", "--shortstat", f"{base}...{head}", text=True) + assert isinstance(shortstat, str) + numbers = [int(value) for value in re.findall(r"\d+", shortstat)] + changed_count = numbers[0] if numbers else len(files) + added = numbers[1] if len(numbers) > 1 else 0 + if changed_count > 50: + warnings.append(f"large PR changes {changed_count} files; verify that the scope is focused") + if added > 5000: + warnings.append(f"large PR adds {added} lines; inspect generated or vendored content") + + runtime = any(name.startswith("crates/") and "/src/" in name for name in files) + tests = any("/tests/" in name or name.startswith("render-repros/") for name in files) + if runtime and not tests: + warnings.append("runtime code changed without a focused integration-test file") + rendering = any(name.startswith("crates/obscura-render/") for name in files) + render_validation = any( + name.startswith("render-repros/") or name.startswith("crates/obscura-render/tests/") + for name in files + ) + if rendering and not render_validation: + warnings.append("rendering code changed without a deterministic rendering fixture") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base", required=True) + parser.add_argument("--head", required=True) + args = parser.parse_args() + if not SHA_RE.fullmatch(args.base) or not SHA_RE.fullmatch(args.head): + raise SystemExit("base and head must be full lowercase commit SHAs") + + files = changed_files(args.base, args.head) + errors: list[str] = [] + warnings: list[str] = [] + check_template(errors) + check_artifacts(files, errors, warnings) + check_control_symlinks(files, errors) + check_new_vendor(args.base, files, errors) + check_workflows(args.base, args.head, files, errors, warnings) + check_scope(args.base, args.head, files, warnings) + + for message in sorted(set(warnings)): + annotation("warning", message) + for message in sorted(set(errors)): + annotation("error", message) + + print(f"Policy inspected {len(files)} changed files: {len(errors)} errors, {len(warnings)} warnings") + if errors: + raise SystemExit(1) + + +if __name__ == "__main__": + main()