test: merge pr-300
This commit is contained in:
SGavrl committed
commit
b27318dc78
18 files changed
+1291
-425
No files matched your search
@@ -20,30 +20,26 @@ jobs:
|
||||
# newer glibc shipped by ubuntu-latest.
|
||||
os: ubuntu-22.04
|
||||
name: obscura-x86_64-linux
|
||||
cross: false
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
# Native build on GitHub's ARM runner.
|
||||
os: ubuntu-22.04-arm
|
||||
name: obscura-aarch64-linux
|
||||
cross: false
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
# Cross-compile fallback from x86_64 Ubuntu 22.04. Same glibc
|
||||
# 2.35 floor, useful when ARM runners are unavailable.
|
||||
os: ubuntu-22.04
|
||||
name: obscura-aarch64-linux-cross
|
||||
cross: true
|
||||
- target: aarch64-apple-darwin
|
||||
os: macos-latest
|
||||
name: obscura-aarch64-macos
|
||||
cross: false
|
||||
- target: x86_64-apple-darwin
|
||||
os: macos-latest
|
||||
# Must run on a native Intel runner. obscura-js bakes a V8 startup
|
||||
# snapshot at build time, and that snapshot is architecture specific.
|
||||
# macos-latest is Apple Silicon, so building x86_64 there embeds an
|
||||
# arm64 snapshot into the Intel binary and V8 crashes deserializing
|
||||
# it at isolate init (issue #290). macos-15-intel is the last x86_64
|
||||
# image (available until Aug 2027); there is no cross-build that
|
||||
# produces a correct snapshot.
|
||||
os: macos-15-intel
|
||||
name: obscura-x86_64-macos
|
||||
cross: false
|
||||
- target: x86_64-pc-windows-msvc
|
||||
os: windows-latest
|
||||
name: obscura-x86_64-windows
|
||||
cross: false
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
@@ -54,17 +50,6 @@ jobs:
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install cross-compilation toolchain (aarch64-linux)
|
||||
if: matrix.cross == true
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
mkdir -p .cargo
|
||||
cat >> .cargo/config.toml << 'CARGO_EOF'
|
||||
[target.aarch64-unknown-linux-gnu]
|
||||
linker = "aarch64-linux-gnu-gcc"
|
||||
CARGO_EOF
|
||||
|
||||
- name: Build
|
||||
run: cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
@@ -72,6 +57,17 @@ jobs:
|
||||
run: cargo build --release --target ${{ matrix.target }} --features stealth
|
||||
continue-on-error: true
|
||||
|
||||
- name: Smoke test (V8 snapshot)
|
||||
# Run the V8 path once so a release never ships a binary that crashes at
|
||||
# isolate init from a mismatched snapshot (issue #290). The data: URL
|
||||
# keeps it offline. Every target here is native, so the runner can
|
||||
# execute what it just built.
|
||||
shell: bash
|
||||
run: |
|
||||
BIN=target/${{ matrix.target }}/release/obscura
|
||||
[ -f "$BIN.exe" ] && BIN="$BIN.exe"
|
||||
"$BIN" fetch "data:text/html,<title>ok</title>" --eval "1+1"
|
||||
|
||||
- name: Package (Unix)
|
||||
if: runner.os != 'Windows'
|
||||
run: |
|
||||
|
||||
Generated
+140
-229
@@ -8,18 +8,6 @@ version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.4"
|
||||
@@ -44,6 +32,12 @@ dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "allocator-api2"
|
||||
version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
@@ -221,9 +215,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.12.1"
|
||||
version = "2.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a"
|
||||
checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
|
||||
|
||||
[[package]]
|
||||
name = "bitvec"
|
||||
@@ -246,31 +240,6 @@ dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "boring-sys2"
|
||||
version = "5.0.0-alpha.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "455d79965f5155dcc88a7abce112c3590883889131b799beda10bf9a813ed669"
|
||||
dependencies = [
|
||||
"bindgen 0.72.1",
|
||||
"cmake",
|
||||
"fs_extra",
|
||||
"fslock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "boring2"
|
||||
version = "5.0.0-alpha.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "183ccc3854411c035410dcdbffafca62084f3a6c33f013c77e83c025d2a08a28"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"boring-sys2",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
"openssl-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli"
|
||||
version = "8.0.3"
|
||||
@@ -292,6 +261,31 @@ dependencies = [
|
||||
"alloc-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "btls"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c5e60b8c8d282c86360cab651ded04ab0335a7b5390c8d34145cbeab8cacf5f"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"btls-sys",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
"openssl-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "btls-sys"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b1b8638a2e1c38a5ae4efa90ae57e643baec35a30d03fc5b399b893adc4954b"
|
||||
dependencies = [
|
||||
"bindgen 0.72.1",
|
||||
"cmake",
|
||||
"fs_extra",
|
||||
"fslock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
@@ -332,9 +326,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.63"
|
||||
version = "1.2.64"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
|
||||
checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -453,35 +447,6 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "147be55d677052dabc6b22252d5dd0fd4c29c8c27aa4f2fbef0f94aa003b406f"
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
version = "0.18.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
"time",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cookie_store"
|
||||
version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206"
|
||||
dependencies = [
|
||||
"cookie",
|
||||
"document-features",
|
||||
"idna",
|
||||
"log",
|
||||
"publicsuffix",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"time",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
@@ -659,15 +624,6 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
|
||||
dependencies = [
|
||||
"powerfmt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "0.99.20"
|
||||
@@ -700,15 +656,6 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "document-features"
|
||||
version = "0.2.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"
|
||||
dependencies = [
|
||||
"litrs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dtoa"
|
||||
version = "1.0.11"
|
||||
@@ -789,6 +736,12 @@ version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.5.0"
|
||||
@@ -1019,19 +972,13 @@ dependencies = [
|
||||
"crc32fast",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.15.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
|
||||
dependencies = [
|
||||
"foldhash",
|
||||
"foldhash 0.1.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1039,6 +986,11 @@ name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"equivalent",
|
||||
"foldhash 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
@@ -1069,9 +1021,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.4.1"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0"
|
||||
checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"itoa",
|
||||
@@ -1102,9 +1054,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "http2"
|
||||
version = "0.5.17"
|
||||
version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "569ef7a780e853c4e1768f58a3c8168193b82cdcbab66638a0b1c6583ec5995e"
|
||||
checksum = "9d8c910a959d6fd27dd6f8fdb26509ac0f6561465cc8c5f875d45a203ab97911"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
@@ -1113,7 +1065,6 @@ dependencies = [
|
||||
"futures-sink",
|
||||
"http",
|
||||
"indexmap",
|
||||
"parking_lot",
|
||||
"slab",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
@@ -1351,13 +1302,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.99"
|
||||
version = "0.3.102"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
|
||||
checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
"once_cell",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
@@ -1407,12 +1357,6 @@ version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
|
||||
|
||||
[[package]]
|
||||
name = "litrs"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.14"
|
||||
@@ -1428,6 +1372,15 @@ version = "0.4.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a"
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9"
|
||||
dependencies = [
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru-slab"
|
||||
version = "0.1.2"
|
||||
@@ -1476,9 +1429,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.1"
|
||||
version = "2.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
|
||||
checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
|
||||
|
||||
[[package]]
|
||||
name = "minimal-lexical"
|
||||
@@ -1543,12 +1496,6 @@ dependencies = [
|
||||
"rand 0.8.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.46"
|
||||
@@ -1864,12 +1811,6 @@ dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.21"
|
||||
@@ -1927,22 +1868,6 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psl-types"
|
||||
version = "2.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
|
||||
|
||||
[[package]]
|
||||
name = "publicsuffix"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf"
|
||||
dependencies = [
|
||||
"idna",
|
||||
"psl-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
@@ -2080,9 +2005,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.12.3"
|
||||
version = "1.12.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
|
||||
checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -2103,9 +2028,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.10"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
@@ -2115,8 +2040,6 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes",
|
||||
"cookie",
|
||||
"cookie_store",
|
||||
"futures-core",
|
||||
"http",
|
||||
"http-body",
|
||||
@@ -2240,17 +2163,6 @@ version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
|
||||
[[package]]
|
||||
name = "schnellru"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "356285bbf17bea63d9e52e96bd18f039672ac92b55b8cb997d6162a2a37d1649"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"cfg-if",
|
||||
"hashbrown 0.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
@@ -2434,9 +2346,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.15.1"
|
||||
version = "1.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
||||
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
@@ -2672,37 +2584,6 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"itoa",
|
||||
"num-conv",
|
||||
"powerfmt",
|
||||
"serde_core",
|
||||
"time-core",
|
||||
"time-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time-core"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
|
||||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
|
||||
dependencies = [
|
||||
"num-conv",
|
||||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.8.3"
|
||||
@@ -2746,12 +2627,12 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-boring2"
|
||||
version = "5.0.0-alpha.13"
|
||||
name = "tokio-btls"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f81df1210d791f31d72d840de8fbd80b9c3cb324956523048b1413e2bd55756"
|
||||
checksum = "2e1fd638ec35427faf3b8f412e0fdd6fae76591d79dba40f38fa667d22bc44dd"
|
||||
dependencies = [
|
||||
"boring2",
|
||||
"btls",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
@@ -3030,9 +2911,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.23.2"
|
||||
version = "1.23.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7"
|
||||
checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7"
|
||||
dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"js-sys",
|
||||
@@ -3090,9 +2971,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
|
||||
|
||||
[[package]]
|
||||
name = "wasip2"
|
||||
version = "1.0.3+wasi-0.2.9"
|
||||
version = "1.0.4+wasi-0.2.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
|
||||
checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
|
||||
dependencies = [
|
||||
"wit-bindgen 0.57.1",
|
||||
]
|
||||
@@ -3108,9 +2989,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.122"
|
||||
version = "0.2.125"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
|
||||
checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
@@ -3121,9 +3002,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.72"
|
||||
version = "0.4.75"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"
|
||||
checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
@@ -3131,9 +3012,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.122"
|
||||
version = "0.2.125"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
|
||||
checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
@@ -3141,9 +3022,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.122"
|
||||
version = "0.2.125"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
|
||||
checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
@@ -3154,9 +3035,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.122"
|
||||
version = "0.2.125"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
|
||||
checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@@ -3207,9 +3088,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.99"
|
||||
version = "0.3.102"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
|
||||
checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
@@ -3550,16 +3431,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wreq"
|
||||
version = "6.0.0-rc.28"
|
||||
version = "6.0.0-rc.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f79937f6c4df65b3f6f78715b9de2977afe9ee3b3436483c7949a24511e25935"
|
||||
checksum = "3f0eba5f5814a94e5f1a99156f187133464e525b66bdbc69a9627d46530af2e1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"boring2",
|
||||
"brotli",
|
||||
"btls",
|
||||
"btls-sys",
|
||||
"bytes",
|
||||
"flate2",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
@@ -3568,29 +3446,62 @@ dependencies = [
|
||||
"httparse",
|
||||
"ipnet",
|
||||
"libc",
|
||||
"lru",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"schnellru",
|
||||
"smallvec",
|
||||
"socket2",
|
||||
"tokio",
|
||||
"tokio-boring2",
|
||||
"tokio-btls",
|
||||
"tokio-socks",
|
||||
"tower",
|
||||
"url",
|
||||
"want",
|
||||
"webpki-root-certs",
|
||||
"zstd",
|
||||
"wreq-proto",
|
||||
"wreq-rt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wreq-proto"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a43942f024bb303f1042c9aa3c87fa1d9149f507c65db6e5220a11ccdb207387"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http2",
|
||||
"httparse",
|
||||
"pin-project-lite",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"want",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wreq-rt"
|
||||
version = "0.2.2-rc.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99e9bce67a3fa3dd3f1503f066d86661c9caf399a763d3bd184da7afaf886c8b"
|
||||
dependencies = [
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"wreq-proto",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wreq-util"
|
||||
version = "3.0.0-rc.10"
|
||||
version = "3.0.0-rc.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c6bbe24d28beb9ceb58b514bd6a613c759d3b706f768b9d2950d5d35b543c04"
|
||||
checksum = "baa5d2ab72139256916ca352a3d05c53d74e1dd360052eb5ba7691033c417c65"
|
||||
dependencies = [
|
||||
"brotli",
|
||||
"flate2",
|
||||
"typed-builder",
|
||||
"wreq",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3633,18 +3544,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.50"
|
||||
version = "0.8.52"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
|
||||
checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.50"
|
||||
version = "0.8.52"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
|
||||
checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -3674,9 +3585,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.8.2"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
|
||||
checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
|
||||
|
||||
[[package]]
|
||||
name = "zerotrie"
|
||||
|
||||
+5
-1
@@ -32,7 +32,11 @@ servo_arc = "0.4"
|
||||
cssparser = "0.34"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-tungstenite = "0.26"
|
||||
reqwest = { version = "0.12", features = ["cookies", "gzip", "brotli", "deflate", "rustls-tls", "socks"], default-features = false }
|
||||
# No "cookies" feature: obscura manages cookies through its own CookieJar
|
||||
# (obscura-net/src/cookies.rs) and never uses reqwest's cookie store. Pulling
|
||||
# it in only added the cookie/cookie_store crates, and cookie 0.18.1's blanket
|
||||
# From impl fails to compile under coherence on some toolchains (issue #295).
|
||||
reqwest = { version = "0.12", features = ["gzip", "brotli", "deflate", "rustls-tls", "socks"], default-features = false }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
url = "2"
|
||||
|
||||
@@ -140,6 +140,12 @@ pub struct NetworkEvent {
|
||||
pub timestamp: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct StoredResponseBody {
|
||||
pub body: String,
|
||||
pub base64_encoded: bool,
|
||||
}
|
||||
|
||||
pub struct Page {
|
||||
pub id: String,
|
||||
pub frame_id: String,
|
||||
@@ -161,6 +167,8 @@ pub struct Page {
|
||||
pub history: Vec<String>,
|
||||
pub history_index: usize,
|
||||
pub network_events: Vec<NetworkEvent>,
|
||||
response_bodies: std::collections::HashMap<String, StoredResponseBody>,
|
||||
response_body_order: std::collections::VecDeque<String>,
|
||||
network_event_counter: u32,
|
||||
pub intercept_enabled: bool,
|
||||
pub intercept_block_patterns: Vec<String>,
|
||||
@@ -213,6 +221,8 @@ impl Page {
|
||||
history: Vec::new(),
|
||||
history_index: 0,
|
||||
network_events: Vec::new(),
|
||||
response_bodies: std::collections::HashMap::new(),
|
||||
response_body_order: std::collections::VecDeque::new(),
|
||||
network_event_counter: 0,
|
||||
intercept_enabled: false,
|
||||
intercept_block_patterns: Vec::new(),
|
||||
@@ -275,19 +285,32 @@ impl Page {
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
if self.stealth_client.is_some() {
|
||||
rt.set_stealth(true);
|
||||
rt.set_user_agent(obscura_net::STEALTH_USER_AGENT);
|
||||
} else if let Ok(ua) = self.http_client.user_agent.try_read() {
|
||||
rt.set_user_agent(&ua);
|
||||
// Match platform to the Linux UA so navigator.platform and
|
||||
// userAgentData.platform are consistent with navigator.userAgent.
|
||||
rt.set_platform("Linux x86_64", "Linux", "");
|
||||
} else {
|
||||
if let Ok(ua) = self.http_client.user_agent.try_read() {
|
||||
rt.set_user_agent(&ua);
|
||||
}
|
||||
rt.set_platform(
|
||||
&self.context.platform,
|
||||
&self.context.ua_platform,
|
||||
&self.context.ua_platform_version,
|
||||
);
|
||||
}
|
||||
#[cfg(not(feature = "stealth"))]
|
||||
if let Ok(ua) = self.http_client.user_agent.try_read() {
|
||||
rt.set_user_agent(&ua);
|
||||
{
|
||||
if let Ok(ua) = self.http_client.user_agent.try_read() {
|
||||
rt.set_user_agent(&ua);
|
||||
}
|
||||
rt.set_platform(
|
||||
&self.context.platform,
|
||||
&self.context.ua_platform,
|
||||
&self.context.ua_platform_version,
|
||||
);
|
||||
}
|
||||
rt.set_platform(
|
||||
&self.context.platform,
|
||||
&self.context.ua_platform,
|
||||
&self.context.ua_platform_version,
|
||||
);
|
||||
if let Some((lat, lon)) = env_geolocation() {
|
||||
rt.set_geolocation(lat, lon);
|
||||
}
|
||||
@@ -303,6 +326,8 @@ impl Page {
|
||||
rt.set_dom(dom);
|
||||
}
|
||||
|
||||
rt.run_page_init();
|
||||
|
||||
self.js = Some(rt);
|
||||
}
|
||||
|
||||
@@ -576,7 +601,7 @@ impl Page {
|
||||
if script.src.is_some() {
|
||||
if let Some((url, code, resp)) = fetched.remove(&i) {
|
||||
tracing::info!("Executing script ({} bytes): {}", code.len(), url);
|
||||
self.record_network_event(&url, "GET", "Script", resp.status, &resp.headers, resp.body.len());
|
||||
self.record_network_event_with_body(&url, "GET", "Script", resp.status, &resp.headers, &resp.body, false);
|
||||
if let Some(js) = &mut self.js {
|
||||
let _ = js.execute_script("<current-script>", &format!("globalThis.__currentScriptNid={};", script.nid));
|
||||
if let Err(e) = js.execute_script_guarded(&url, &code) {
|
||||
@@ -948,13 +973,14 @@ impl Page {
|
||||
PageError::NetworkError(e.to_string())
|
||||
})?;
|
||||
|
||||
self.record_network_event(
|
||||
self.record_network_event_with_body(
|
||||
url.as_str(),
|
||||
"GET",
|
||||
"Document",
|
||||
response.status,
|
||||
&response.headers,
|
||||
response.body.len(),
|
||||
&response.body,
|
||||
false,
|
||||
);
|
||||
|
||||
if !response.redirected_from.is_empty() {
|
||||
@@ -1048,7 +1074,7 @@ impl Page {
|
||||
// CSS bodies: honor the Content-Type charset; CSS @charset is
|
||||
// out of scope for the current scrape-focused pipeline.
|
||||
let css = obscura_net::decode_non_html(&resp.body, resp.content_type());
|
||||
self.record_network_event(&url_str, "GET", "Stylesheet", resp.status, &resp.headers, resp.body.len());
|
||||
self.record_network_event_with_body(&url_str, "GET", "Stylesheet", resp.status, &resp.headers, &resp.body, false);
|
||||
css_sources.push(css);
|
||||
}
|
||||
}
|
||||
@@ -1341,13 +1367,47 @@ impl Page {
|
||||
response_headers: &std::collections::HashMap<String, String>,
|
||||
body_size: usize,
|
||||
) {
|
||||
self.record_network_event_inner(url, method, resource_type, status, response_headers, body_size);
|
||||
}
|
||||
|
||||
fn record_network_event_with_body(
|
||||
&mut self,
|
||||
url: &str,
|
||||
method: &str,
|
||||
resource_type: &str,
|
||||
status: u16,
|
||||
response_headers: &std::collections::HashMap<String, String>,
|
||||
body: &[u8],
|
||||
base64_encoded: bool,
|
||||
) {
|
||||
let request_id = self.record_network_event_inner(
|
||||
url,
|
||||
method,
|
||||
resource_type,
|
||||
status,
|
||||
response_headers,
|
||||
body.len(),
|
||||
);
|
||||
self.store_response_body(request_id, body, base64_encoded);
|
||||
}
|
||||
|
||||
fn record_network_event_inner(
|
||||
&mut self,
|
||||
url: &str,
|
||||
method: &str,
|
||||
resource_type: &str,
|
||||
status: u16,
|
||||
response_headers: &std::collections::HashMap<String, String>,
|
||||
body_size: usize,
|
||||
) -> String {
|
||||
self.network_event_counter += 1;
|
||||
let request_id = format!("{}.{}", self.id, self.network_event_counter);
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs_f64();
|
||||
self.network_events.push(NetworkEvent {
|
||||
request_id: format!("{}.{}", self.id, self.network_event_counter),
|
||||
request_id: request_id.clone(),
|
||||
url: url.to_string(),
|
||||
method: method.to_string(),
|
||||
resource_type: resource_type.to_string(),
|
||||
@@ -1357,6 +1417,46 @@ impl Page {
|
||||
body_size,
|
||||
timestamp,
|
||||
});
|
||||
request_id
|
||||
}
|
||||
|
||||
fn store_response_body(&mut self, request_id: String, body: &[u8], base64_encoded: bool) {
|
||||
let max_entries = response_body_entry_limit();
|
||||
let max_bytes = response_body_byte_limit();
|
||||
if max_entries == 0 || max_bytes == 0 || body.len() > max_bytes {
|
||||
return;
|
||||
}
|
||||
let body = if base64_encoded {
|
||||
BASE64.encode(body)
|
||||
} else {
|
||||
String::from_utf8_lossy(body).to_string()
|
||||
};
|
||||
self.response_bodies.insert(request_id.clone(), StoredResponseBody { body, base64_encoded });
|
||||
self.response_body_order.push_back(request_id);
|
||||
while self.response_body_order.len() > max_entries {
|
||||
if let Some(oldest) = self.response_body_order.pop_front() {
|
||||
self.response_bodies.remove(&oldest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_response_body(&self, request_id: &str) -> Option<StoredResponseBody> {
|
||||
self.response_bodies.get(request_id).cloned().or_else(|| {
|
||||
self.js.as_ref()?.get_network_response_body(request_id).map(|body| {
|
||||
StoredResponseBody {
|
||||
body: body.body,
|
||||
base64_encoded: body.base64_encoded,
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn clear_response_bodies(&mut self) {
|
||||
self.response_bodies.clear();
|
||||
self.response_body_order.clear();
|
||||
if let Some(js) = &self.js {
|
||||
js.clear_network_response_bodies();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn execute_preload_script(&mut self, source: &str) -> Result<(), String> {
|
||||
@@ -1463,3 +1563,17 @@ impl From<ObscuraNetError> for PageError {
|
||||
PageError::NetworkError(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn response_body_entry_limit() -> usize {
|
||||
std::env::var("OBSCURA_NETWORK_BODY_BUFFER_ENTRIES")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(128)
|
||||
}
|
||||
|
||||
fn response_body_byte_limit() -> usize {
|
||||
std::env::var("OBSCURA_NETWORK_BODY_BUFFER_BYTES")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(2 * 1024 * 1024)
|
||||
}
|
||||
@@ -238,6 +238,7 @@ fn is_v8_free_method(method: &str) -> bool {
|
||||
| "Network.getCookies" | "Network.getAllCookies"
|
||||
| "Network.setCookie" | "Network.setCookies"
|
||||
| "Network.deleteCookies" | "Network.clearBrowserCookies"
|
||||
| "Network.getResponseBody"
|
||||
| "Fetch.continueRequest" | "Fetch.fulfillRequest"
|
||||
| "Fetch.failRequest" | "Fetch.getResponseBody"
|
||||
| "Storage.getCookies" | "Storage.setCookies" | "Storage.deleteCookies"
|
||||
|
||||
@@ -32,6 +32,16 @@ pub async fn handle(
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => Ok(json!({})),
|
||||
"disable" => {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.clear_response_bodies();
|
||||
} else {
|
||||
for page in &mut ctx.pages {
|
||||
page.clear_response_bodies();
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"setExtraHTTPHeaders" => {
|
||||
let headers = params.get("headers").and_then(|v| v.as_object());
|
||||
if let Some(page) = ctx.get_session_page(session_id) {
|
||||
@@ -86,6 +96,26 @@ pub async fn handle(
|
||||
}
|
||||
"setCacheDisabled" => Ok(json!({})),
|
||||
"setRequestInterception" => Ok(json!({})),
|
||||
"getResponseBody" => {
|
||||
let request_id = params
|
||||
.get("requestId")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("Network.getResponseBody requires requestId")?;
|
||||
|
||||
let body = if let Some(page) = ctx.get_session_page(session_id) {
|
||||
page.get_response_body(request_id)
|
||||
} else {
|
||||
ctx.pages.iter().find_map(|page| page.get_response_body(request_id))
|
||||
};
|
||||
|
||||
match body {
|
||||
Some(body) => Ok(json!({
|
||||
"body": body.body,
|
||||
"base64Encoded": body.base64_encoded,
|
||||
})),
|
||||
None => Err(format!("No response body found for requestId {}", request_id)),
|
||||
}
|
||||
}
|
||||
_ => Err(format!("Unknown Network method: {}", method)),
|
||||
}
|
||||
}
|
||||
@@ -193,6 +223,75 @@ mod tests {
|
||||
.expect("clearBrowserCookies must succeed");
|
||||
assert!(ctx.default_context.cookie_jar.get_all_cookies().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_response_body_returns_stored_document_body() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = Some("session-1".to_string());
|
||||
ctx.sessions.insert(session_id.clone().unwrap(), page_id.clone());
|
||||
|
||||
let page = ctx.get_page_mut(&page_id).unwrap();
|
||||
page.navigate("data:text/html,<html><body>hello body</body></html>")
|
||||
.await
|
||||
.unwrap();
|
||||
let request_id = page.network_events[0].request_id.clone();
|
||||
|
||||
let result = handle(
|
||||
"getResponseBody",
|
||||
&json!({ "requestId": request_id }),
|
||||
&mut ctx,
|
||||
&session_id,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result["body"], "<html><body>hello body</body></html>");
|
||||
assert_eq!(result["base64Encoded"], false);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_response_body_errors_for_unknown_request_id() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let err = handle(
|
||||
"getResponseBody",
|
||||
&json!({ "requestId": "missing" }),
|
||||
&mut ctx,
|
||||
&None,
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(err.contains("missing"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn network_disable_clears_stored_response_bodies() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = Some("session-1".to_string());
|
||||
ctx.sessions.insert(session_id.clone().unwrap(), page_id.clone());
|
||||
|
||||
let page = ctx.get_page_mut(&page_id).unwrap();
|
||||
page.navigate("data:text/html,<html><body>temporary body</body></html>")
|
||||
.await
|
||||
.unwrap();
|
||||
let request_id = page.network_events[0].request_id.clone();
|
||||
|
||||
handle("disable", &json!({}), &mut ctx, &session_id)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let err = handle(
|
||||
"getResponseBody",
|
||||
&json!({ "requestId": request_id }),
|
||||
&mut ctx,
|
||||
&session_id,
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(err.contains("No response body found"));
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn cookie_info_to_cdp_json(c: &obscura_net::CookieInfo) -> Value {
|
||||
|
||||
@@ -565,11 +565,11 @@ impl DomTree {
|
||||
// querySelector matches strict descendants of root only, so the
|
||||
// indexed element must have root among its ancestors.
|
||||
Some(nid) if self.ancestors(nid).contains(&root) => return Ok(Some(nid)),
|
||||
// No element has this id at all: a bare id selector cannot match.
|
||||
None => return Ok(None),
|
||||
// Indexed (first) element is not under a scoped root; a later
|
||||
// duplicate could still be a descendant, so fall through to scan.
|
||||
Some(_) => {}
|
||||
// Index miss or stale entry (detached node) — fall through to full scan.
|
||||
// The id_index is best-effort: it only registers nodes at creation time
|
||||
// and does not update on reparent, so it can point to a detached clone
|
||||
// while the live node (with the same id) is elsewhere in the tree.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let selector_list = parse_selector(selector)?;
|
||||
|
||||
Vendored
+649
-124
File diff suppressed because it is too large.
Load diff
@@ -1,5 +1,5 @@
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::rc::Rc;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -38,6 +38,12 @@ pub struct InterceptedRequest {
|
||||
pub resolver: tokio::sync::oneshot::Sender<InterceptResolution>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct StoredNetworkResponseBody {
|
||||
pub body: String,
|
||||
pub base64_encoded: bool,
|
||||
}
|
||||
|
||||
pub struct ObscuraState {
|
||||
pub dom: Option<DomTree>,
|
||||
pub url: String,
|
||||
@@ -57,6 +63,9 @@ pub struct ObscuraState {
|
||||
// `op_binding_called` op. Drained by the CDP layer after each dispatch
|
||||
// and emitted as `Runtime.bindingCalled` events.
|
||||
pub pending_binding_calls: Vec<(String, String)>,
|
||||
pub network_response_bodies: HashMap<String, StoredNetworkResponseBody>,
|
||||
pub network_response_body_order: VecDeque<String>,
|
||||
pub network_response_body_counter: u64,
|
||||
}
|
||||
|
||||
impl ObscuraState {
|
||||
@@ -74,10 +83,27 @@ impl ObscuraState {
|
||||
intercept_counter: 0,
|
||||
intercept_enabled: false,
|
||||
pending_binding_calls: Vec::new(),
|
||||
network_response_bodies: HashMap::new(),
|
||||
network_response_body_order: VecDeque::new(),
|
||||
network_response_body_counter: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn response_body_entry_limit() -> usize {
|
||||
std::env::var("OBSCURA_NETWORK_BODY_BUFFER_ENTRIES")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(128)
|
||||
}
|
||||
|
||||
fn response_body_byte_limit() -> usize {
|
||||
std::env::var("OBSCURA_NETWORK_BODY_BUFFER_BYTES")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(2 * 1024 * 1024)
|
||||
}
|
||||
|
||||
pub type SharedState = Rc<RefCell<ObscuraState>>;
|
||||
|
||||
#[op2]
|
||||
@@ -137,7 +163,21 @@ fn op_dom_inner(state: &OpState, cmd: String, arg1: String, arg2: String) -> Str
|
||||
"null".into()
|
||||
}
|
||||
"get_element_by_id" => {
|
||||
dom.get_element_by_id(&arg1).map(|id| id.index().to_string()).unwrap_or("-1".into())
|
||||
// Verify the indexed node is in the live document. The id_index is best-effort:
|
||||
// it only registers nodes at creation time and doesn't update on reparent, so
|
||||
// it can point to a detached clone while the live node is elsewhere in the tree.
|
||||
let doc = dom.document();
|
||||
let nid = dom.get_element_by_id(&arg1);
|
||||
let live = nid.filter(|&n| dom.ancestors(n).contains(&doc));
|
||||
match live {
|
||||
Some(n) => n.index().to_string(),
|
||||
None => {
|
||||
// Fall back to full scan for the live document.
|
||||
let sel = format!("[id=\"{}\"]", arg1.replace('\\', "\\\\").replace('"', "\\\""));
|
||||
dom.query_selector(&sel).ok().flatten()
|
||||
.map(|id| id.index().to_string()).unwrap_or("-1".into())
|
||||
}
|
||||
}
|
||||
}
|
||||
"query_selector" => {
|
||||
dom.query_selector(&arg1).ok().flatten().map(|id| id.index().to_string()).unwrap_or("-1".into())
|
||||
@@ -235,19 +275,22 @@ fn op_dom_inner(state: &OpState, cmd: String, arg1: String, arg2: String) -> Str
|
||||
serde_json::to_string(&dom.outer_html(NodeId::new(nid))).unwrap_or("\"\"".into())
|
||||
}
|
||||
"append_child" => {
|
||||
let parent = arg1.parse::<u32>().unwrap_or(0);
|
||||
let child = arg2.parse::<u32>().unwrap_or(0);
|
||||
// Reject if either nid failed to parse (was "undefined"/empty) — those
|
||||
// default to 0 which is the document root, and silently operating on it
|
||||
// corrupts the tree. Require both args to be valid positive integers.
|
||||
let parent = match arg1.parse::<u32>() { Ok(n) => n, Err(_) => return "false".into() };
|
||||
let child = match arg2.parse::<u32>() { Ok(n) => n, Err(_) => return "false".into() };
|
||||
dom.append_child(NodeId::new(parent), NodeId::new(child));
|
||||
"true".into()
|
||||
}
|
||||
"remove_child" => {
|
||||
let child = arg1.parse::<u32>().unwrap_or(0);
|
||||
let child = match arg1.parse::<u32>() { Ok(n) => n, Err(_) => return "false".into() };
|
||||
dom.remove_child(NodeId::new(child));
|
||||
"true".into()
|
||||
}
|
||||
"insert_before" => {
|
||||
let new_node = arg1.parse::<u32>().unwrap_or(0);
|
||||
let ref_node = arg2.parse::<u32>().unwrap_or(0);
|
||||
let new_node = match arg1.parse::<u32>() { Ok(n) => n, Err(_) => return "false".into() };
|
||||
let ref_node = match arg2.parse::<u32>() { Ok(n) => n, Err(_) => return "false".into() };
|
||||
dom.insert_before(NodeId::new(ref_node), NodeId::new(new_node));
|
||||
"true".into()
|
||||
}
|
||||
@@ -261,7 +304,12 @@ fn op_dom_inner(state: &OpState, cmd: String, arg1: String, arg2: String) -> Str
|
||||
"true".into()
|
||||
}
|
||||
"set_inner_html" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
let nid = match arg1.parse::<u32>() {
|
||||
Ok(n) if n > 0 => n,
|
||||
// nid=0 is the document root; never allow innerHTML to clear it.
|
||||
// nid parse failure (e.g. "undefined") also falls here.
|
||||
_ => return "false".into(),
|
||||
};
|
||||
let target = NodeId::new(nid);
|
||||
let children = dom.children(target);
|
||||
for child in children {
|
||||
@@ -850,6 +898,31 @@ async fn op_fetch_url(
|
||||
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?;
|
||||
let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
|
||||
let resp_body_base64 = BASE64.encode(&resp_bytes);
|
||||
let response_request_id = {
|
||||
let state_borrow = state.borrow();
|
||||
let gs = state_borrow.borrow::<SharedState>().clone();
|
||||
let mut gs = gs.borrow_mut();
|
||||
gs.network_response_body_counter += 1;
|
||||
let request_id = format!("fetch-{}", gs.network_response_body_counter);
|
||||
let max_entries = response_body_entry_limit();
|
||||
let max_bytes = response_body_byte_limit();
|
||||
if max_entries > 0 && max_bytes > 0 && resp_bytes.len() <= max_bytes {
|
||||
gs.network_response_bodies.insert(
|
||||
request_id.clone(),
|
||||
StoredNetworkResponseBody {
|
||||
body: resp_body.clone(),
|
||||
base64_encoded: false,
|
||||
},
|
||||
);
|
||||
gs.network_response_body_order.push_back(request_id.clone());
|
||||
while gs.network_response_body_order.len() > max_entries {
|
||||
if let Some(oldest) = gs.network_response_body_order.pop_front() {
|
||||
gs.network_response_bodies.remove(&oldest);
|
||||
}
|
||||
}
|
||||
}
|
||||
request_id
|
||||
};
|
||||
|
||||
tracing::debug!("op_fetch_url completed: {} {} ({} bytes)", method, url, resp_body.len());
|
||||
|
||||
@@ -857,6 +930,7 @@ async fn op_fetch_url(
|
||||
"status": status,
|
||||
"body": resp_body,
|
||||
"bodyBase64": resp_body_base64,
|
||||
"requestId": response_request_id,
|
||||
"url": url,
|
||||
"headers": resp_headers,
|
||||
})
|
||||
|
||||
@@ -10,7 +10,7 @@ use obscura_dom::DomTree;
|
||||
pub use deno_core::v8::IsolateHandle;
|
||||
|
||||
use crate::module_loader::ObscuraModuleLoader;
|
||||
use crate::ops::{build_extension, ObscuraState};
|
||||
use crate::ops::{build_extension, ObscuraState, StoredNetworkResponseBody};
|
||||
|
||||
static SNAPSHOT: &[u8] = include_bytes!(env!("OBSCURA_SNAPSHOT_PATH"));
|
||||
|
||||
@@ -130,7 +130,7 @@ impl ObscuraJsRuntime {
|
||||
runtime
|
||||
.execute_script(
|
||||
"<obscura:init>",
|
||||
"globalThis.__obscura_objects = {}; globalThis.__obscura_oid = 0; globalThis.__obscura_init();".to_string(),
|
||||
"globalThis.__obscura_objects = {}; globalThis.__obscura_oid = 0;".to_string(),
|
||||
)
|
||||
.expect("init should not fail");
|
||||
|
||||
@@ -184,6 +184,16 @@ impl ObscuraJsRuntime {
|
||||
std::mem::take(&mut self.state.borrow_mut().pending_binding_calls)
|
||||
}
|
||||
|
||||
pub fn get_network_response_body(&self, request_id: &str) -> Option<StoredNetworkResponseBody> {
|
||||
self.state.borrow().network_response_bodies.get(request_id).cloned()
|
||||
}
|
||||
|
||||
pub fn clear_network_response_bodies(&self) {
|
||||
let mut state = self.state.borrow_mut();
|
||||
state.network_response_bodies.clear();
|
||||
state.network_response_body_order.clear();
|
||||
}
|
||||
|
||||
/// Wire up the interception channel without enabling interception.
|
||||
/// Use set_intercept_enabled separately. The two were entangled before
|
||||
/// and every navigation auto-enabled interception, which made
|
||||
@@ -220,6 +230,22 @@ impl ObscuraJsRuntime {
|
||||
);
|
||||
}
|
||||
|
||||
pub fn set_stealth(&mut self, enabled: bool) {
|
||||
let _ = self.runtime.execute_script(
|
||||
"<set-stealth>",
|
||||
format!("globalThis.__obscura_stealth = {};", enabled),
|
||||
);
|
||||
}
|
||||
|
||||
/// Run __obscura_init() after all per-page properties (UA, platform, stealth, etc.)
|
||||
/// have been set. Must be called once per page setup, after all set_* methods.
|
||||
pub fn run_page_init(&mut self) {
|
||||
let _ = self.runtime.execute_script(
|
||||
"<obscura:page-init>",
|
||||
"globalThis.__obscura_init();".to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
/// Override the coordinates the navigator.geolocation shim reports. The
|
||||
/// values are injected as numeric globals the bootstrap reads; when unset it
|
||||
/// keeps the built-in default. Callers validate the range before calling.
|
||||
@@ -2452,6 +2478,21 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn console_log_error_does_not_trigger_prepare_stack_trace() {
|
||||
let mut rt = setup_runtime("<div></div>");
|
||||
let result = rt.evaluate(r#"
|
||||
let called = false;
|
||||
const saved = Error.prepareStackTrace;
|
||||
Error.prepareStackTrace = function() { called = true; return saved; };
|
||||
const e = new Error("test");
|
||||
console.log(e);
|
||||
Error.prepareStackTrace = saved;
|
||||
return called;
|
||||
"#).unwrap();
|
||||
assert_eq!(result, serde_json::json!(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn element_aria_reflection_setters_write_through() {
|
||||
let mut rt = setup_runtime(r#"<div id="d"></div>"#);
|
||||
|
||||
@@ -23,14 +23,14 @@ tempfile = "3"
|
||||
# Emulation builder reshuffled). A caret requirement let a fresh resolve pull a
|
||||
# newer, incompatible rc and broke `cargo build --features stealth` (issue #234).
|
||||
# Pin exact versions so the build always resolves to the API wreq_client.rs targets.
|
||||
wreq-util = { version = "=3.0.0-rc.10", optional = true }
|
||||
wreq-util = { version = "=3.0.0-rc.12", optional = true }
|
||||
|
||||
# `prefix-symbols` renames BoringSSL exports to avoid clashes with system OpenSSL,
|
||||
# but the renaming is only correct on Linux and Android (per wreq's README).
|
||||
# Enabling it on other platforms produces unresolved `build_script_main_*`
|
||||
# symbols at link time (see issue #39).
|
||||
[target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies]
|
||||
wreq = { version = "=6.0.0-rc.28", optional = true, features = ["prefix-symbols", "socks"] }
|
||||
wreq = { version = "=6.0.0-rc.29", optional = true, features = ["prefix-symbols", "socks"] }
|
||||
|
||||
[target.'cfg(not(any(target_os = "linux", target_os = "android")))'.dependencies]
|
||||
wreq = { version = "=6.0.0-rc.28", optional = true, features = ["socks"] }
|
||||
wreq = { version = "=6.0.0-rc.29", optional = true, features = ["socks"] }
|
||||
@@ -36,21 +36,13 @@ impl StealthHttpClient {
|
||||
}
|
||||
|
||||
pub fn with_proxy(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
|
||||
// Issue #184: `set_default_paths()` reads OpenSSL's compile-time CA
|
||||
// paths, which only resolve on Linux. On Windows the store ends up
|
||||
// empty and every TLS handshake fails with CERTIFICATE_VERIFY_FAILED.
|
||||
// `CertStore::default()` uses wreq's bundled Mozilla roots
|
||||
// (`webpki-root-certs`), which works the same on every platform.
|
||||
let cert_store = wreq::tls::CertStore::default();
|
||||
|
||||
let emulation_opts = wreq_util::EmulationOption::builder()
|
||||
.emulation(wreq_util::Emulation::Chrome145)
|
||||
.emulation_os(wreq_util::EmulationOS::Linux)
|
||||
let emulation_opts = wreq_util::Emulation::builder()
|
||||
.profile(wreq_util::Profile::Chrome145)
|
||||
.platform(wreq_util::Platform::Linux)
|
||||
.build();
|
||||
|
||||
let mut builder = wreq::Client::builder()
|
||||
.emulation(emulation_opts)
|
||||
.cert_store(cert_store)
|
||||
.timeout(Duration::from_secs(30))
|
||||
.redirect(wreq::redirect::Policy::none());
|
||||
|
||||
|
||||
@@ -61,6 +61,7 @@ Run the CDP server. Puppeteer and Playwright connect over WebSocket.
|
||||
--allow-file-access Permit CDP clients to navigate to file:// URLs
|
||||
--storage-dir <DIR> Persistent cookies and localStorage
|
||||
--allow-private-network Permit loopback / RFC1918 / link-local
|
||||
-q, --quiet Suppress info logging
|
||||
-v, --verbose Enable info logging
|
||||
```
|
||||
|
||||
@@ -95,6 +96,7 @@ Run obscura as an MCP server.
|
||||
|
||||
```
|
||||
--http HTTP transport instead of stdio
|
||||
--host <HOST> HTTP bind host (default 127.0.0.1)
|
||||
--port <PORT> HTTP port (default 3000)
|
||||
--proxy <URL> HTTP or SOCKS5 proxy
|
||||
--user-agent <UA> Override the User-Agent
|
||||
@@ -103,4 +105,6 @@ Run obscura as an MCP server.
|
||||
-v, --verbose Enable info logging
|
||||
```
|
||||
|
||||
`--host` only applies with `--http`. The default `127.0.0.1` keeps the server loopback-only; set `0.0.0.0` to bind all interfaces (for example a Docker Compose sidecar) and pair it with `OBSCURA_MCP_ALLOWED_ORIGINS`.
|
||||
|
||||
Default transport is stdio. See [Use the MCP server](Use-the-MCP-server.md).
|
||||
@@ -60,6 +60,31 @@ obscura serve --user-agent "Mozilla/5.0 (...) ..."
|
||||
|
||||
Default UA matches a recent Chrome on the build platform.
|
||||
|
||||
## Browser profile, timezone, and geolocation
|
||||
|
||||
The engine presents one of a built-in pool of realistic browser profiles (a mix of Windows and macOS, recent Chrome versions). Each profile keeps `navigator.platform`, `navigator.userAgentData` (platform and platform version), the UA string, and the WebGL/GPU renderer internally consistent, so the surfaces a site fingerprints agree with each other. Windows profiles report ANGLE Direct3D11 renderers, macOS profiles report ANGLE Metal renderers.
|
||||
|
||||
A single stable profile is used by default. One IP cycling through different identities is itself a signal, so rotation is opt-in:
|
||||
|
||||
```bash
|
||||
OBSCURA_PROFILE=2 obscura serve # pin a specific profile by index
|
||||
OBSCURA_ROTATE_PROFILE=1 obscura serve # random profile per browser context
|
||||
```
|
||||
|
||||
Timezone is driven by the process zone so `Date` (`getTimezoneOffset`, `toString`) and `Intl.DateTimeFormat` report the same region. Default is `Europe/Berlin`; set it to match the exit IP:
|
||||
|
||||
```bash
|
||||
OBSCURA_TIMEZONE=America/New_York obscura serve
|
||||
```
|
||||
|
||||
`navigator.geolocation` reports configurable coordinates. Set them as `lat,lon` and keep them consistent with the timezone and proxy region:
|
||||
|
||||
```bash
|
||||
OBSCURA_GEOLOCATION="40.7128,-74.0060" obscura serve
|
||||
```
|
||||
|
||||
Keep these aligned. A rotated or mismatched profile carries no matching TLS or timezone fingerprint, so when you pin a proxy region or TLS fingerprint, leave rotation off and set the timezone and geolocation to the same region. See [Environment variables](Environment-variables.md) for the full list.
|
||||
|
||||
## Combine
|
||||
|
||||
```bash
|
||||
|
||||
@@ -72,13 +72,15 @@ await page.goto('https://example.com', { waitUntil: 'load' });
|
||||
|
||||
- `page.goto`, `page.reload`, `page.goBack`, `page.goForward`
|
||||
- `page.evaluate`, `page.evaluateHandle`
|
||||
- `page.click`, `page.type`, `page.fill`
|
||||
- `page.click`, `page.type`, `page.fill`, `page.focus`
|
||||
- `page.waitForSelector`, `page.waitForFunction`, `page.waitForNavigation`
|
||||
- `page.cookies`, `page.setCookie`, `context.cookies`
|
||||
- `page.setRequestInterception`, block / modify
|
||||
- `page.exposeFunction`
|
||||
- `page.content`, `page.title`, `page.url`
|
||||
|
||||
DOM-agent frameworks such as browser-use also connect: obscura implements `DOMSnapshot.captureSnapshot` and `Target.targetInfoChanged` for perception, and `DOM.focus` so a focused field receives `Input.dispatchKeyEvent` keystrokes.
|
||||
|
||||
## Not supported
|
||||
|
||||
- `page.screenshot`: obscura doesn't render pixels.
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
### `OBSCURA_ALLOW_PRIVATE_NETWORK`
|
||||
|
||||
Allow fetches to loopback (`127.0.0.0/8`), RFC1918 (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`), and link-local (`169.254.0.0/16`) addresses. Off by default to block SSRF.
|
||||
Allow fetches to loopback (`127.0.0.0/8`), RFC1918 (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`), and link-local (`169.254.0.0/16`, including the `169.254.169.254` cloud-metadata endpoint) addresses. The deny-set also covers the unspecified address (`0.0.0.0` / `::`), IPv6 unique-local (`fc00::/7`), and any IPv4-mapped form of the above. Off by default to block SSRF.
|
||||
|
||||
The guard validates at DNS-resolution time as well as on literal hosts, so a public hostname that resolves to a forbidden address is rejected at connect time (DNS-rebinding safe), not just hosts written as raw IPs.
|
||||
|
||||
Truthy values: `1`, `true`, `yes`, `on`.
|
||||
|
||||
@@ -44,6 +46,52 @@ Default proxy URL used by `obscura-worker` for the parallel `scrape` command whe
|
||||
OBSCURA_PROXY=http://proxy.example.com:8080 obscura scrape - < urls.txt
|
||||
```
|
||||
|
||||
## Stealth and identity
|
||||
|
||||
These tune the browser identity the engine presents so it stays internally consistent. See [Configure stealth and proxies](Configure-stealth-and-proxies.md) for the full picture.
|
||||
|
||||
### `OBSCURA_TIMEZONE`
|
||||
|
||||
Pins the process timezone before V8/ICU reads it, so `Date` (`getTimezoneOffset`, `toString`) and `Intl.DateTimeFormat` report one consistent zone. Default `Europe/Berlin`. Set it to match the exit IP's region.
|
||||
|
||||
```bash
|
||||
OBSCURA_TIMEZONE=America/New_York obscura serve
|
||||
```
|
||||
|
||||
### `OBSCURA_GEOLOCATION`
|
||||
|
||||
Override the coordinates the `navigator.geolocation` shim reports, as `lat,lon`. Without it the shim reports a fixed default. Keep it consistent with `OBSCURA_TIMEZONE` and the proxy region.
|
||||
|
||||
```bash
|
||||
OBSCURA_GEOLOCATION="40.7128,-74.0060" obscura serve
|
||||
```
|
||||
|
||||
### `OBSCURA_PROFILE`
|
||||
|
||||
Pin a specific browser profile from the built-in pool by index (`0`-based). Each profile keeps `navigator.platform`, `userAgentData`, the UA string, and the GPU renderer internally consistent. Without it a single stable profile is used.
|
||||
|
||||
```bash
|
||||
OBSCURA_PROFILE=2 obscura serve
|
||||
```
|
||||
|
||||
### `OBSCURA_ROTATE_PROFILE`
|
||||
|
||||
Opt into picking a random profile per browser context instead of the stable default. Leave it off when you pin a TLS fingerprint, proxy region, or timezone, since a rotated profile would no longer match those.
|
||||
|
||||
```bash
|
||||
OBSCURA_ROTATE_PROFILE=1 obscura serve
|
||||
```
|
||||
|
||||
## MCP
|
||||
|
||||
### `OBSCURA_MCP_ALLOWED_ORIGINS`
|
||||
|
||||
Comma-separated `Origin` allowlist for the HTTP MCP transport (`obscura mcp --http`). Off by default, which keeps the permissive behavior. When set, a browser request whose `Origin` is not listed is refused with `403` before it can drive the server; native, non-browser MCP clients (which send no `Origin`) are always allowed. Use it to stop cross-origin pages from reaching a loopback MCP port.
|
||||
|
||||
```bash
|
||||
OBSCURA_MCP_ALLOWED_ORIGINS="https://app.example.com" obscura mcp --http --host 0.0.0.0
|
||||
```
|
||||
|
||||
## Logging
|
||||
|
||||
### `RUST_LOG`
|
||||
@@ -76,7 +124,7 @@ V8 flags are passed via `--v8-flags`, not environment variables:
|
||||
obscura serve --v8-flags "--max-old-space-size=2048 --expose-gc"
|
||||
```
|
||||
|
||||
Default heap is 4 GB on 64-bit systems.
|
||||
Defaults are `--max-old-space-size=4096 --max-semi-space-size=4 --optimize-for-size` on 64-bit systems (a 4 GB old-space ceiling, a capped young generation, and codegen tuned for a smaller footprint to cut RSS). Anything you pass with `--v8-flags` is appended after these, and V8 uses the last value for a repeated flag, so your value wins for that flag while the other defaults stay in effect.
|
||||
|
||||
## HTTP proxy environment
|
||||
|
||||
|
||||
@@ -50,13 +50,13 @@ Use one worker per CPU core. Each worker handles its own pool of pages. Sessions
|
||||
|
||||
## V8 heap
|
||||
|
||||
Default V8 heap is 4 GB on 64-bit systems. Override:
|
||||
Default V8 heap is 4 GB on 64-bit systems. The defaults also cap the young generation (`--max-semi-space-size=4`) and pass `--optimize-for-size` to hold RSS down. Override:
|
||||
|
||||
```bash
|
||||
obscura serve --v8-flags "--max-old-space-size=2048"
|
||||
```
|
||||
|
||||
Lower for memory-constrained hosts, raise for heavy SPAs.
|
||||
Flags you pass are appended after the defaults, and V8 uses the last value for a repeated flag, so your `--max-old-space-size` wins while the memory-tuning defaults stay in effect. Lower for memory-constrained hosts, raise for heavy SPAs.
|
||||
|
||||
## Parallel scrape
|
||||
|
||||
@@ -120,6 +120,17 @@ Obscura's CDP server has no built-in auth. Anyone who can reach the port can dri
|
||||
|
||||
Never bind `0.0.0.0` on a public IP without one of the above.
|
||||
|
||||
## MCP HTTP transport
|
||||
|
||||
`obscura mcp --http` binds `127.0.0.1` by default. To reach it from another container, bind with `--host 0.0.0.0` and set an `Origin` allowlist so a browser page cannot drive it cross-origin:
|
||||
|
||||
```bash
|
||||
OBSCURA_MCP_ALLOWED_ORIGINS="https://app.example.com" \
|
||||
obscura mcp --http --host 0.0.0.0 --port 3000
|
||||
```
|
||||
|
||||
Request bodies are capped at 16 MiB. Like the CDP server it has no built-in auth, so keep it on an internal network or behind an authenticating proxy. See [Use the MCP server](Use-the-MCP-server.md).
|
||||
|
||||
## Observability
|
||||
|
||||
```bash
|
||||
|
||||
@@ -14,12 +14,31 @@ HTTP (for remote or shared use):
|
||||
obscura mcp --http --port 3000
|
||||
```
|
||||
|
||||
The HTTP transport binds `127.0.0.1` by default. Bind all interfaces with `--host` for a container or sidecar deployment:
|
||||
|
||||
```bash
|
||||
obscura mcp --http --host 0.0.0.0 --port 3000
|
||||
```
|
||||
|
||||
With stealth and proxy:
|
||||
|
||||
```bash
|
||||
obscura mcp --stealth --proxy http://proxy.example.com:8080
|
||||
```
|
||||
|
||||
## Security
|
||||
|
||||
The HTTP transport has no built-in auth, so anyone who can reach the port can drive the browser. Two guards ship for the HTTP transport:
|
||||
|
||||
- **Origin allowlist.** Set `OBSCURA_MCP_ALLOWED_ORIGINS` to a comma-separated list of allowed `Origin` values. When set, a browser request from an unlisted origin is refused with `403` before it can drive the server, which blocks a malicious page from POSTing to a loopback MCP port. Native, non-browser clients send no `Origin` and are always allowed. Unset (the default) keeps the permissive behavior.
|
||||
- **Body cap.** A single request body is capped at 16 MiB, so an unauthenticated caller cannot force a large allocation with an oversized `Content-Length`.
|
||||
|
||||
```bash
|
||||
OBSCURA_MCP_ALLOWED_ORIGINS="https://app.example.com" obscura mcp --http --host 0.0.0.0
|
||||
```
|
||||
|
||||
When you expose the HTTP transport beyond loopback, set the allowlist and put it behind a reverse proxy or network isolation that enforces auth.
|
||||
|
||||
## Tools exposed
|
||||
|
||||
The server keeps a live browser session, so tools operate on the current page rather than taking a URL each call. Navigate first, then read or act.
|
||||
|
||||
Reference in new issue
Block a user