Merge remote-tracking branch 'origin/main' into maintainer/pr-494-review

This commit is contained in:
SGavrl committed 2026-08-01 15:40:11 +00:00
commit c327354911
18 files changed
+1248 -150

No files matched your search

-10
View File
@@ -1,12 +1,2 @@
[env] [env]
RUST_TEST_THREADS = "1" RUST_TEST_THREADS = "1"
# macOS 26 (Tahoe) moved libc++ headers inside the SDK; Apple clang 17 no longer
# finds them at the old search path. This causes boring-sys's cmake step to fail
# when building with --features stealth. Setting CXXFLAGS and SDKROOT here makes
# them visible to all build scripts (including boring-sys's) so cmake can locate
# the headers. force=false lets CI or developers override via their shell env.
# Non-macOS platforms ignore SDKROOT; the -isystem path is silently skipped if
# it doesn't exist. See: https://github.com/h4ckf0r0day/obscura/issues/136
CXXFLAGS = { value = "-isystem /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/c++/v1", force = false }
SDKROOT = { value = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk", force = false }
+67 -15
View File
@@ -50,41 +50,93 @@ jobs:
with: with:
targets: ${{ matrix.target }} targets: ${{ matrix.target }}
- name: Build - name: Install stealth build dependencies (Linux)
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y cmake clang libclang-dev llvm-dev
- name: Configure libclang (macOS)
if: runner.os == 'macOS'
shell: bash
run: |
# Use one coherent SDK. A hard-coded CommandLineTools include path can
# mix libc++ headers with the selected Xcode SDK and break BoringSSL.
XCODE_PATH="$(xcode-select -p)"
SDK_PATH="$(xcrun --show-sdk-path)"
echo "SDKROOT=$SDK_PATH" >> "$GITHUB_ENV"
echo "CXXFLAGS=-isystem $SDK_PATH/usr/include/c++/v1" >> "$GITHUB_ENV"
echo "LIBCLANG_PATH=$XCODE_PATH/Toolchains/XcodeDefault.xctoolchain/usr/lib" >> "$GITHUB_ENV"
- name: Configure libclang (Windows)
if: runner.os == 'Windows'
shell: pwsh
run: |
"LIBCLANG_PATH=C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_ENV -Append
- name: Build default release binary
run: cargo build --release --target ${{ matrix.target }} run: cargo build --release --target ${{ matrix.target }}
- name: Build stealth - name: Stage default release binary
run: cargo build --release --target ${{ matrix.target }} --features stealth shell: bash
continue-on-error: true run: |
BIN_DIR=target/${{ matrix.target }}/release
mkdir -p dist/default
if [ -f "$BIN_DIR/obscura.exe" ]; then
cp "$BIN_DIR/obscura.exe" "$BIN_DIR/obscura-worker.exe" dist/default/
else
cp "$BIN_DIR/obscura" "$BIN_DIR/obscura-worker" dist/default/
fi
- name: Smoke test (V8 snapshot) # Build and package stealth separately so the default download keeps its
# lower RSS while the advertised wreq/BoringSSL variant cannot silently
# fall back to a non-stealth binary.
- name: Build stealth release binary
run: cargo build --release --target ${{ matrix.target }} --features stealth
- name: Stage stealth release binary
shell: bash
run: |
BIN_DIR=target/${{ matrix.target }}/release
mkdir -p dist/stealth
if [ -f "$BIN_DIR/obscura.exe" ]; then
cp "$BIN_DIR/obscura.exe" "$BIN_DIR/obscura-worker.exe" dist/stealth/
else
cp "$BIN_DIR/obscura" "$BIN_DIR/obscura-worker" dist/stealth/
fi
- name: Smoke test release binaries (V8 snapshot)
# Run the V8 path once so a release never ships a binary that crashes at # Run the V8 path once so a release never ships a binary that crashes at
# isolate init from a mismatched snapshot (issue #290). The data: URL # isolate init from a mismatched snapshot (issue #290). The data: URL
# keeps it offline. Every target here is native, so the runner can # keeps it offline. Every target here is native, so the runner can
# execute what it just built. # execute both variants it just built.
shell: bash shell: bash
run: | run: |
BIN=target/${{ matrix.target }}/release/obscura for VARIANT in default stealth; do
[ -f "$BIN.exe" ] && BIN="$BIN.exe" BIN=dist/$VARIANT/obscura
"$BIN" fetch "data:text/html,<title>ok</title>" --eval "1+1" [ -f "$BIN.exe" ] && BIN="$BIN.exe"
"$BIN" fetch "data:text/html,<title>ok</title>" --eval "1+1"
done
- name: Package (Unix) - name: Package (Unix)
if: runner.os != 'Windows' if: runner.os != 'Windows'
run: | run: |
cd target/${{ matrix.target }}/release tar czf ${{ matrix.name }}.tar.gz -C dist/default obscura obscura-worker
tar czf ../../../${{ matrix.name }}.tar.gz obscura obscura-worker tar czf ${{ matrix.name }}-stealth.tar.gz -C dist/stealth obscura obscura-worker
cd ../../..
- name: Package (Windows) - name: Package (Windows)
if: runner.os == 'Windows' if: runner.os == 'Windows'
run: | run: |
cd target/${{ matrix.target }}/release cd dist/default
7z a ../../../${{ matrix.name }}.zip obscura.exe obscura-worker.exe 7z a ../../${{ matrix.name }}.zip obscura.exe obscura-worker.exe
cd ../../.. cd ../stealth
7z a ../../${{ matrix.name }}-stealth.zip obscura.exe obscura-worker.exe
- name: Upload - name: Upload
uses: softprops/action-gh-release@v2 uses: softprops/action-gh-release@v2
with: with:
files: | files: |
${{ matrix.name }}.tar.gz ${{ matrix.name }}.tar.gz
${{ matrix.name }}-stealth.tar.gz
${{ matrix.name }}.zip ${{ matrix.name }}.zip
${{ matrix.name }}-stealth.zip
Generated
+74 -2
View File
@@ -697,6 +697,12 @@ dependencies = [
"tokio", "tokio",
] ]
[[package]]
name = "deranged"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
[[package]] [[package]]
name = "derive_more" name = "derive_more"
version = "0.99.20" version = "0.99.20"
@@ -1608,6 +1614,12 @@ dependencies = [
"rand 0.8.6", "rand 0.8.6",
] ]
[[package]]
name = "num-conv"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
[[package]] [[package]]
name = "num-integer" name = "num-integer"
version = "0.1.46" version = "0.1.46"
@@ -1764,12 +1776,14 @@ version = "0.1.0"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"encoding_rs", "encoding_rs",
"rcgen",
"reqwest", "reqwest",
"serde", "serde",
"serde_json", "serde_json",
"tempfile", "tempfile",
"thiserror 2.0.18", "thiserror 2.0.18",
"tokio", "tokio",
"tokio-rustls",
"tracing", "tracing",
"url", "url",
"wreq", "wreq",
@@ -1850,6 +1864,16 @@ dependencies = [
"hmac", "hmac",
] ]
[[package]]
name = "pem"
version = "3.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be"
dependencies = [
"base64",
"serde_core",
]
[[package]] [[package]]
name = "percent-encoding" name = "percent-encoding"
version = "2.3.2" version = "2.3.2"
@@ -1961,6 +1985,12 @@ dependencies = [
"zerovec", "zerovec",
] ]
[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]] [[package]]
name = "ppv-lite86" name = "ppv-lite86"
version = "0.2.21" version = "0.2.21"
@@ -2040,9 +2070,9 @@ dependencies = [
[[package]] [[package]]
name = "quinn-proto" name = "quinn-proto"
version = "0.11.14" version = "0.11.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e"
dependencies = [ dependencies = [
"bytes", "bytes",
"getrandom 0.3.4", "getrandom 0.3.4",
@@ -2147,6 +2177,19 @@ dependencies = [
"getrandom 0.3.4", "getrandom 0.3.4",
] ]
[[package]]
name = "rcgen"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2"
dependencies = [
"pem",
"ring",
"rustls-pki-types",
"time",
"yasna",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.5.18" version = "0.5.18"
@@ -2275,6 +2318,7 @@ version = "0.23.40"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
dependencies = [ dependencies = [
"log",
"once_cell", "once_cell",
"ring", "ring",
"rustls-pki-types", "rustls-pki-types",
@@ -2737,6 +2781,25 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "time"
version = "0.3.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244"
dependencies = [
"deranged",
"num-conv",
"powerfmt",
"serde_core",
"time-core",
]
[[package]]
name = "time-core"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
[[package]] [[package]]
name = "tinystr" name = "tinystr"
version = "0.8.3" version = "0.8.3"
@@ -3682,6 +3745,15 @@ dependencies = [
"tap", "tap",
] ]
[[package]]
name = "yasna"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd"
dependencies = [
"time",
]
[[package]] [[package]]
name = "yoke" name = "yoke"
version = "0.8.3" version = "0.8.3"
+38 -3
View File
@@ -1,16 +1,16 @@
<p align="center"> <p align="center">
<img src="https://raw.githubusercontent.com/h4ckf0r0day/obscura/main/assets/icon.png" alt="Obscura" width="80" /> <img src="https://raw.githubusercontent.com/h4ckf0r0day/obscura/main/assets/icon.png" alt="Obscura" width="80" />
</p> </p>
<h2 align="center">Obscura</h2> <h2 align="center">Obscura</h2>
<p align="center">
<a href="https://trendshift.io/repositories/25837?utm_source=trendshift-badge&amp;utm_medium=badge&amp;utm_campaign=badge-trendshift-25837" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/25837/daily" alt="h4ckf0r0day%2Fobscura | Trendshift" width="250" height="55"/></a>
</p>
<p align="center"> <p align="center">
<a href="https://docs.obscura.sh"><img src="https://img.shields.io/badge/Docs-1a1a1a?style=for-the-badge&logo=gitbook&logoColor=white" alt="Documentation" /></a> <a href="https://docs.obscura.sh"><img src="https://img.shields.io/badge/Docs-1a1a1a?style=for-the-badge&logo=gitbook&logoColor=white" alt="Documentation" /></a>
<a href="https://obscura.sh"><img src="https://img.shields.io/badge/Website-1a1a1a?style=for-the-badge&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjE5MCAxNzAgNDIwIDQyMCI+PHBhdGggZmlsbD0iI0ZGRkZGRiIgZmlsbC1ydWxlPSJldmVub2RkIiBkPSJNNDA2LDE3NS4wMDA0MTIgQzQ0MC40OTg5MzIsMTc1LjAwMDM1MSA0NzQuNDk4MTA4LDE3NS4wODA5OTQgNTA4LjQ5NjczNSwxNzQuOTc1NjYyIEM1MzcuNzMzNjQzLDE3NC44ODUwODYgNTYxLjI3MDY5MSwxODcuMjIwNTUxIDU3OS45Mzk2MzYsMjA4Ljc4NjYyMSBDNTkwLjg0MTA2NCwyMjEuMzc5Nzc2IDU5Ny44ODM1NDUsMjM2LjM1MjY2MSA1OTkuOTgxODczLDI1My4yMjAzNjcgQzYwMC40NDU1NTcsMjU2Ljk0NzU0MCA2MDAuOTUxMTExLDI2MC43MDQ0MzcgNjAwLjk1NjYwNCwyNjQuNDQ4MzY0IEM2MDEuMDIyNjQ0LDMwOS42MTM1MjUgNjAxLjA3NTgwNiwzNTQuNzc5MDIyIDYwMC45NTE3ODIsMzk5Ljk0MzkzOSBDNjAwLjkxMzE0Nyw0MTQuMDMyMzQ5IDYwMC42NjMyNjksNDI4LjE0MDEwNiA1OTkuNzgyMjg4LDQ0Mi4xOTMzNTkgQzU5OS41NDEwMTYsNDQ2LjA0MDg5NCA1OTcuNTYwNzkxLDQ1MC42MTU2MDEgNTk0Ljg1MzIxMCw0NTMuMzQxMzA5IEM1NzEuMTc3NTUxLDQ3Ny4xNzUzMjMgNTQ3LjE4MDcyNSw1MDAuNjkwNzk2IDUyMy4yMjc5NjYsNTI0LjI0ODc3OSBDNTA2Ljc5OTgzNSw1NDAuNDA2MTI4IDQ5MC4yMDk5OTEsNTU2LjM5OTkwMiA0NzMuODY0NTYzLDU3Mi42NDAxMzcgQzQ2OC44MDA4NDIsNTc3LjY3MTIwNCA0NjMuMDgyODg2LDU4MC4wNTQxOTkgNDU1Ljk3MDYxMiw1ODAuMDQzODIzIEM0MDAuOTcyNDQzLDU3OS45NjM1MDEgMzQ1Ljk3MzY2Myw1ODAuMTMyMDE5IDI5MC45NzYwNDQsNTc5LjkzNjc2OCBDMjY3LjU4MDQxNCw1NzkuODUzNjk5IDI0Ni41NTEyMDgsNTcyLjA0MzU3OSAyMjguOTM5NzQzLDU1Ni44Mjc1MTUgQzIxMi44NDk2MjUsNTQyLjkyNTg0MiAyMDIuNzUxOTY4LDUyNC45MDIxMDAgMTk4LjE1NjE1OCw1MDQuMDM5NjczIEMxOTcuMjgyMTk2LDUwMC4wNzIyOTYgMTk3LjA1MjQxNCw0OTUuODk1Mzg2IDE5Ny4wNDczMzMsNDkxLjgxNDQ4NCBDMTk2Ljk3ODE0OSw0MzYuMTQ5NjU4IDE5Ny4wNTA3MjAsMzgwLjQ4NDYxOSAxOTYuOTMyNjE3LDMyNC44MTk5NDYgQzE5Ni45MTkxNDQsMzE4LjQ3MDkxNyAxOTkuMTg0ODQ1LDMxMy40MTYxOTkgMjAzLjQ5NTU3NSwzMDkuMTAwNTI1IEMyNDAuNTg3OTk3LDI3MS45NjU0MjQgMjc3LjY4MjQ5NSwyMzQuODMyMzA2IDMxNC44MzQ1NjQsMTk3Ljc1Njk1OCBDMzIwLjk1NjQyMSwxOTEuNjQ3Nzk3IDMyNy4yNjQ0MzUsMTg1LjcxNTI1NiAzMzMuNjczOTIwLDE3OS45MDgwODEgQzMzNy4zNzYwMzgsMTc2LjU1MzgzMyAzNDEuNzIxNDY2LDE3NC44NTMzMTcgMzQ3LjAwNTc5OCwxNzQuOTEyODcyIEMzNjYuNTAxNzA5LDE3NS4xMzI2MTQgMzg2LjAwMTYxNywxNzUuMDAwMzIwIDQwNiwxNzUuMDAwNDEyIFogTTUwMy4zNDQ2NjYsMjczLjg0MDE0OSBDNTA0LjEwMjcyMiwyNzYuMTY3NTcyIDUwNC45NDA5NDgsMjc4LjIxMTA5MCA1MDQuOTQzMjY4LDI4MC4yNTU1MjQgQzUwNS4wMTI4NDgsMzQxLjc0MjI0OSA1MDQuOTY5MTQ3LDQwMy4yMjkwNjUgNTA1LjAzMDc5Miw0NjQuNzE1NzkwIEM1MDUuMDQwODAyLDQ3NC42ODY0OTMgNDk2LjExNzQ2Miw0ODMuOTUzMTg2IDQ4NS43Nzc5MjQsNDgzLjk2NTI3MSBDNDI0LjYyNDQ1MSw0ODQuMDM2ODk2IDM2My40NzA5MTcsNDg0LjAwMzIzNSAzMDIuMzE3MzgzLDQ4My45OTY3MzUgQzI5NS41ODc3NjksNDgzLjk5NjAzMyAyOTMuMDAxMjUxLDQ4MS4zMDE2MDUgMjkzLjAwMDkxNiw0NzQuMzM4NTYyIEMyOTIuOTk3ODY0LDQxMy4zNTE2NTQgMjkzLjIwOTYyNSwzNTIuMzYzMzEyIDI5Mi43ODA3MDEsMjkxLjM3OTM5NSBDMjkyLjcxMzUzMSwyODEuODI4MDk0IDMwMy4yMTMwNDMsMjcwLjgyNDE1OCAzMTMuNDc4MjcxLDI3MC44OTQzMTggQzM1MS45NjgyMDEsMjcxLjE1NzQ0MCAzOTAuNDYwOTk5LDI3MC45OTk3ODYgNDI4Ljk1MjcyOCwyNzAuOTk5Nzg2IEM0NTEuMTE0NjI0LDI3MC45OTk3ODYgNDczLjI3NzI4MywyNzAuOTE3NDgwIDQ5NS40Mzc0NjksMjcxLjExMjE1MiBDNDk3Ljk3NzAyMCwyNzEuMTM0NDkxIDUwMC41MDI5MzAsMjcyLjcwNDM3NiA1MDMuMzQ0NjY2LDI3My44NDAxNDkgWiI+PC9wYXRoPjwvc3ZnPgo=&logoColor=white" alt="Website" /></a> <a href="https://obscura.sh"><img src="https://img.shields.io/badge/Website-1a1a1a?style=for-the-badge&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjE5MCAxNzAgNDIwIDQyMCI+PHBhdGggZmlsbD0iI0ZGRkZGRiIgZmlsbC1ydWxlPSJldmVub2RkIiBkPSJNNDA2LDE3NS4wMDA0MTIgQzQ0MC40OTg5MzIsMTc1LjAwMDM1MSA0NzQuNDk4MTA4LDE3NS4wODA5OTQgNTA4LjQ5NjczNSwxNzQuOTc1NjYyIEM1MzcuNzMzNjQzLDE3NC44ODUwODYgNTYxLjI3MDY5MSwxODcuMjIwNTUxIDU3OS45Mzk2MzYsMjA4Ljc4NjYyMSBDNTkwLjg0MTA2NCwyMjEuMzc5Nzc2IDU5Ny44ODM1NDUsMjM2LjM1MjY2MSA1OTkuOTgxODczLDI1My4yMjAzNjcgQzYwMC40NDU1NTcsMjU2Ljk0NzU0MCA2MDAuOTUxMTExLDI2MC43MDQ0MzcgNjAwLjk1NjYwNCwyNjQuNDQ4MzY0IEM2MDEuMDIyNjQ0LDMwOS42MTM1MjUgNjAxLjA3NTgwNiwzNTQuNzc5MDIyIDYwMC45NTE3ODIsMzk5Ljk0MzkzOSBDNjAwLjkxMzE0Nyw0MTQuMDMyMzQ5IDYwMC42NjMyNjksNDI4LjE0MDEwNiA1OTkuNzgyMjg4LDQ0Mi4xOTMzNTkgQzU5OS41NDEwMTYsNDQ2LjA0MDg5NCA1OTcuNTYwNzkxLDQ1MC42MTU2MDEgNTk0Ljg1MzIxMCw0NTMuMzQxMzA5IEM1NzEuMTc3NTUxLDQ3Ny4xNzUzMjMgNTQ3LjE4MDcyNSw1MDAuNjkwNzk2IDUyMy4yMjc5NjYsNTI0LjI0ODc3OSBDNTA2Ljc5OTgzNSw1NDAuNDA2MTI4IDQ5MC4yMDk5OTEsNTU2LjM5OTkwMiA0NzMuODY0NTYzLDU3Mi42NDAxMzcgQzQ2OC44MDA4NDIsNTc3LjY3MTIwNCA0NjMuMDgyODg2LDU4MC4wNTQxOTkgNDU1Ljk3MDYxMiw1ODAuMDQzODIzIEM0MDAuOTcyNDQzLDU3OS45NjM1MDEgMzQ1Ljk3MzY2Myw1ODAuMTMyMDE5IDI5MC45NzYwNDQsNTc5LjkzNjc2OCBDMjY3LjU4MDQxNCw1NzkuODUzNjk5IDI0Ni41NTEyMDgsNTcyLjA0MzU3OSAyMjguOTM5NzQzLDU1Ni44Mjc1MTUgQzIxMi44NDk2MjUsNTQyLjkyNTg0MiAyMDIuNzUxOTY4LDUyNC45MDIxMDAgMTk4LjE1NjE1OCw1MDQuMDM5NjczIEMxOTcuMjgyMTk2LDUwMC4wNzIyOTYgMTk3LjA1MjQxNCw0OTUuODk1Mzg2IDE5Ny4wNDczMzMsNDkxLjgxNDQ4NCBDMTk2Ljk3ODE0OSw0MzYuMTQ5NjU4IDE5Ny4wNTA3MjAsMzgwLjQ4NDYxOSAxOTYuOTMyNjE3LDMyNC44MTk5NDYgQzE5Ni45MTkxNDQsMzE4LjQ3MDkxNyAxOTkuMTg0ODQ1LDMxMy40MTYxOTkgMjAzLjQ5NTU3NSwzMDkuMTAwNTI1IEMyNDAuNTg3OTk3LDI3MS45NjU0MjQgMjc3LjY4MjQ5NSwyMzQuODMyMzA2IDMxNC44MzQ1NjQsMTk3Ljc1Njk1OCBDMzIwLjk1NjQyMSwxOTEuNjQ3Nzk3IDMyNy4yNjQ0MzUsMTg1LjcxNTI1NiAzMzMuNjczOTIwLDE3OS45MDgwODEgQzMzNy4zNzYwMzgsMTc2LjU1MzgzMyAzNDEuNzIxNDY2LDE3NC44NTMzMTcgMzQ3LjAwNTc5OCwxNzQuOTEyODcyIEMzNjYuNTAxNzA5LDE3NS4xMzI2MTQgMzg2LjAwMTYxNywxNzUuMDAwMzIwIDQwNiwxNzUuMDAwNDEyIFogTTUwMy4zNDQ2NjYsMjczLjg0MDE0OSBDNTA0LjEwMjcyMiwyNzYuMTY3NTcyIDUwNC45NDA5NDgsMjc4LjIxMTA5MCA1MDQuOTQzMjY4LDI4MC4yNTU1MjQgQzUwNS4wMTI4NDgsMzQxLjc0MjI0OSA1MDQuOTY5MTQ3LDQwMy4yMjkwNjUgNTA1LjAzMDc5Miw0NjQuNzE1NzkwIEM1MDUuMDQwODAyLDQ3NC42ODY0OTMgNDk2LjExNzQ2Miw0ODMuOTUzMTg2IDQ4NS43Nzc5MjQsNDgzLjk2NTI3MSBDNDI0LjYyNDQ1MSw0ODQuMDM2ODk2IDM2My40NzA5MTcsNDg0LjAwMzIzNSAzMDIuMzE3MzgzLDQ4My45OTY3MzUgQzI5NS41ODc3NjksNDgzLjk5NjAzMyAyOTMuMDAxMjUxLDQ4MS4zMDE2MDUgMjkzLjAwMDkxNiw0NzQuMzM4NTYyIEMyOTIuOTk3ODY0LDQxMy4zNTE2NTQgMjkzLjIwOTYyNSwzNTIuMzYzMzEyIDI5Mi43ODA3MDEsMjkxLjM3OTM5NSBDMjkyLjcxMzUzMSwyODEuODI4MDk0IDMwMy4yMTMwNDMsMjcwLjgyNDE1OCAzMTMuNDc4MjcxLDI3MC44OTQzMTggQzM1MS45NjgyMDEsMjcxLjE1NzQ0MCAzOTAuNDYwOTk5LDI3MC45OTk3ODYgNDI4Ljk1MjcyOCwyNzAuOTk5Nzg2IEM0NTEuMTE0NjI0LDI3MC45OTk3ODYgNDczLjI3NzI4MywyNzAuOTE3NDgwIDQ5NS40Mzc0NjksMjcxLjExMjE1MiBDNDk3Ljk3NzAyMCwyNzEuMTM0NDkxIDUwMC41MDI5MzAsMjcyLjcwNDM3NiA1MDMuMzQ0NjY2LDI3My44NDAxNDkgWiI+PC9wYXRoPjwvc3ZnPgo=&logoColor=white" alt="Website" /></a>
<a href="https://cal.com/obscura/quick-chat"><img src="https://img.shields.io/badge/Book_a_Demo-1a1a1a?style=for-the-badge&logo=googlecalendar&logoColor=white" alt="Book a demo" /></a> <a href="https://cal.com/obscura/quick-chat"><img src="https://img.shields.io/badge/Book_a_Demo-1a1a1a?style=for-the-badge&logo=googlecalendar&logoColor=white" alt="Book a demo" /></a>
<a href="https://github.com/h4ckf0r0day/obscura/releases"><img src="https://img.shields.io/badge/Releases-1a1a1a?style=for-the-badge&logo=github&logoColor=white" alt="Releases" /></a> <a href="https://github.com/h4ckf0r0day/obscura/releases"><img src="https://img.shields.io/badge/Releases-1a1a1a?style=for-the-badge&logo=github&logoColor=white" alt="Releases" /></a>
</p> </p>
<p align="center"> <p align="center">
<strong>The open-source headless browser for AI agents and web scraping.</strong><br> <strong>The open-source headless browser for AI agents and web scraping.</strong><br>
Lightweight, stealthy, and built in Rust. Lightweight, stealthy, and built in Rust.
@@ -132,6 +132,26 @@ Want to sponsor? Email [[email protected]](mailto:[email protected]).
🎁 Use discount code <b>RAPID10</b> to get <b>10% off</b>.<br><br></b> 🎁 Use discount code <b>RAPID10</b> to get <b>10% off</b>.<br><br></b>
</td> </td>
</tr> </tr>
<tr>
<td width="200" align="center" valign="middle">
<a href="https://www.thordata.com/?ls=dob&lk=dob" target="_blank">
<img alt="Thordata" src="/assets/sponsors/thordatalogo.png" width="180"/>
</a>
</td>
<td valign="middle">
🚀 <b>Obscura × Thordata</b><br>
Need more stable proxies for automation, public web scraping, SEO, or ad verification? Power your workflow with <a href="https://www.thordata.com/?ls=dob&lk=dob"><b>Thordata</b></a>.<br><br>
<b>🌍 Residential Proxies: 100M+ real residential IPs in 190+ countries<br>
♾️ Unlimited Proxies: for heavy traffic and high concurrency<br>
✨ Recently upgraded residential IP pool<br>
✨ Unlimited Concurrent Sessions<br>
🔁 Rotating & Sticky Sessions<br>
📍 Flexible GEO targeting<br>
⚡ Stable HTTP(S) connections<br><br>
🎁 Use code <b>obscura</b> for <b>10% off</b>.<br><br></b>
Get started with a 3-day free trial and test Thordata with your own workflow.
</td>
</tr>
</table> </table>
## Install ## Install
@@ -153,6 +173,9 @@ tar xzf obscura-aarch64-linux.tar.gz
# Arch Linux (AUR) # Arch Linux (AUR)
yay -S obscura-browser yay -S obscura-browser
# NixOS
nix-env -iA nixpkgs.obscura
# macOS Apple Silicon # macOS Apple Silicon
curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-aarch64-macos.tar.gz curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-aarch64-macos.tar.gz
tar xzf obscura-aarch64-macos.tar.gz tar xzf obscura-aarch64-macos.tar.gz
@@ -169,6 +192,10 @@ No Chrome, no Node.js, no dependencies. Release archives include both
`obscura` and `obscura-worker`; keep them in the same directory for the `obscura` and `obscura-worker`; keep them in the same directory for the
parallel `scrape` command. parallel `scrape` command.
Default archive names contain the lean binary. Archives ending in
`-stealth.tar.gz` or `-stealth.zip` additionally include the wreq/BoringSSL
transport used for TLS impersonation.
Linux release builds target Ubuntu 22.04 so the downloaded binary remains Linux release builds target Ubuntu 22.04 so the downloaded binary remains
usable on common LTS servers with glibc 2.35+. usable on common LTS servers with glibc 2.35+.
@@ -192,6 +219,14 @@ cargo build --release --features stealth
``` ```
Requires Rust 1.75+ ([rustup.rs](https://rustup.rs)). First build takes ~5 min (V8 compiles from source, cached after). Requires Rust 1.75+ ([rustup.rs](https://rustup.rs)). First build takes ~5 min (V8 compiles from source, cached after).
The stealth build also compiles BoringSSL and generates bindings, so it needs
CMake, Clang, and the libclang/LLVM development libraries. On Ubuntu/Debian:
```bash
sudo apt-get install build-essential cmake clang libclang-dev llvm-dev
```
The default build does not require these additional stealth dependencies.
## Quick Start ## Quick Start
Binary file not shown.

After

Width:  |  Height:  |  Size: 3.5 KiB

@@ -117,3 +117,134 @@ async fn dynamic_external_scripts_execute_and_fire_load() {
"dynamic scripts must execute and fire load before navigation settles" "dynamic scripts must execute and fire load before navigation settles"
); );
} }
#[tokio::test(flavor = "current_thread")]
async fn dynamic_data_scripts_execute_before_chained_load_handlers() {
let mut ctx = CdpContext::new();
let page_id = ctx.create_page();
let session_id = "session-1";
ctx.sessions.insert(session_id.to_string(), page_id);
cdp(
&mut ctx,
1,
"Page.navigate",
json!({
"url": "data:text/html,<html><head></head><body></body></html>",
"waitUntil": "load"
}),
session_id,
)
.await;
cdp(
&mut ctx,
2,
"Runtime.evaluate",
json!({
"expression": r#"(function () {
var state = {
aExec: false,
aLoad: false,
bExec: false,
bLoad: false,
cExec: false,
cLoad: false
};
window.__dataScriptState = state;
var a = document.createElement('script');
a.src = 'data:,window.__dataScriptState.aExec=true';
a.onload = function () {
state.aLoad = true;
var b = document.createElement('script');
b.src = 'data:text/html,' + encodeURIComponent('window.__dataScriptState.bExec=true');
b.onload = function () {
state.bLoad = true;
var c = document.createElement('script');
c.src = 'data:text/javascript;base64,' +
btoa('window.__dataScriptState.cExec=true').replace(/=+$/, '') +
'#ignored-fragment';
c.onload = function () { state.cLoad = true; };
document.head.appendChild(c);
};
document.head.appendChild(b);
};
document.head.appendChild(a);
return 'kicked';
})()"#,
"returnByValue": true,
}),
session_id,
)
.await;
ctx.pages[0].settle(500).await;
let result = cdp(
&mut ctx,
3,
"Runtime.evaluate",
json!({
"expression": "JSON.stringify(window.__dataScriptState)",
"returnByValue": true,
}),
session_id,
)
.await;
assert_eq!(
result["result"]["value"],
r#"{"aExec":true,"aLoad":true,"bExec":true,"bLoad":true,"cExec":true,"cLoad":true}"#,
"data URL script bodies must execute before each chained load handler"
);
}
#[tokio::test(flavor = "current_thread")]
async fn invalid_dynamic_data_script_fires_error_not_load() {
let mut ctx = CdpContext::new();
let page_id = ctx.create_page();
let session_id = "session-1";
ctx.sessions.insert(session_id.to_string(), page_id);
cdp(
&mut ctx,
1,
"Page.navigate",
json!({"url": "data:text/html,<html><head></head><body></body></html>", "waitUntil": "load"}),
session_id,
)
.await;
cdp(
&mut ctx,
2,
"Runtime.callFunctionOn",
json!({
"functionDeclaration": r#"function () {
window.__invalidDataScript = { error: false, load: false };
var script = document.createElement('script');
script.src = 'data:text/javascript;base64,!';
script.onerror = function () { window.__invalidDataScript.error = true; };
script.onload = function () { window.__invalidDataScript.load = true; };
document.head.appendChild(script);
}"#,
"awaitPromise": true,
}),
session_id,
)
.await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let result = cdp(
&mut ctx,
3,
"Runtime.evaluate",
json!({
"expression": "JSON.stringify(window.__invalidDataScript)",
"returnByValue": true,
}),
session_id,
)
.await;
assert_eq!(result["result"]["value"], r#"{"error":true,"load":false}"#);
}
+112 -10
View File
@@ -18,7 +18,7 @@
'__obscura_hw', '__obscura_mem', '__obscura_hw', '__obscura_mem',
'__documentReadyState__', '__currentUrl', '__documentReadyState__', '__currentUrl',
// internal helpers (var-declared throughout the file) // internal helpers (var-declared throughout the file)
'__processDynScriptQueue', '_markNative', '_fpRand', '_fpNoise', '__processDynScriptQueue', '_decodeDataScriptUrl', '_markNative', '_fpRand', '_fpNoise',
'_fpCache', '_getFp', '_fp', '_splitAsciiWhitespace', '_fpCache', '_getFp', '_fp', '_splitAsciiWhitespace',
'_getElementsByClassName', '_docEncoding', '_docIsUtf8', '_getElementsByClassName', '_docEncoding', '_docIsUtf8',
'_isSpecialScheme', '_applyDocQueryEncoding', '_anchorBase', '_isSpecialScheme', '_applyDocQueryEncoding', '_anchorBase',
@@ -163,6 +163,51 @@ let _fpSeed = 0;
// when SPAs dynamically insert multiple <script module> tags at once. // when SPAs dynamically insert multiple <script module> tags at once.
let __dynScriptQueue = []; let __dynScriptQueue = [];
let __dynScriptBusy = false; let __dynScriptBusy = false;
function _decodeDataScriptUrl(url) {
const comma = url.indexOf(',');
if (!url.startsWith('data:') || comma < 5) {
throw new TypeError('Invalid dynamic script data URL');
}
const meta = url.slice(5, comma);
const fragment = url.indexOf('#', comma + 1);
const payload = url.slice(comma + 1, fragment < 0 ? url.length : fragment);
if (meta.split(';').some(part => part.toLowerCase() === 'base64')) {
let encoded = payload.replace(/[\r\n\t\f ]/g, '');
const remainder = encoded.length % 4;
if (remainder === 1 || !/^[A-Za-z0-9+/]*={0,2}$/.test(encoded) || /=/.test(encoded.slice(0, -2))) {
throw new TypeError('Invalid dynamic script data URL base64');
}
if (remainder > 0) encoded += '='.repeat(4 - remainder);
if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(encoded)) {
throw new TypeError('Invalid dynamic script data URL base64');
}
return new TextDecoder().decode(_base64ToUint8Array(encoded));
}
const bytes = [];
for (let i = 0; i < payload.length; i++) {
const code = payload.charCodeAt(i);
if (code === 0x25 && i + 2 < payload.length) {
const hi = _hexv(payload.charCodeAt(i + 1));
const lo = _hexv(payload.charCodeAt(i + 2));
if (hi >= 0 && lo >= 0) {
bytes.push(hi * 16 + lo);
i += 2;
continue;
}
}
if (code < 0x80) {
bytes.push(code);
} else {
const character = String.fromCodePoint(payload.codePointAt(i));
if (character.length === 2) i++;
const encoded = new TextEncoder().encode(character);
for (let j = 0; j < encoded.length; j++) bytes.push(encoded[j]);
}
}
return new TextDecoder().decode(new Uint8Array(bytes));
}
async function __processDynScriptQueue() { async function __processDynScriptQueue() {
if (__dynScriptBusy) return; if (__dynScriptBusy) return;
__dynScriptBusy = true; __dynScriptBusy = true;
@@ -176,11 +221,16 @@ async function __processDynScriptQueue() {
if (task.isModule) { if (task.isModule) {
await import(task.url); await import(task.url);
} else { } else {
const raw = await Deno.core.ops.op_fetch_url(task.url, "GET", "{}", "", task.pageOrigin, "no-cors"); let body;
const parsed = JSON.parse(raw); if (task.url.startsWith('data:')) {
if (parsed.body) { body = _decodeDataScriptUrl(task.url);
} else {
const raw = await Deno.core.ops.op_fetch_url(task.url, "GET", "{}", "", task.pageOrigin, "no-cors");
body = JSON.parse(raw).body;
}
if (body) {
globalThis.__currentScriptNid = task.nid; globalThis.__currentScriptNid = task.nid;
try { (0, eval)(parsed.body); } try { (0, eval)(body); }
catch(e) { console.error('Dynamic script error (' + task.url + '):', e.message); } catch(e) { console.error('Dynamic script error (' + task.url + '):', e.message); }
finally { globalThis.__currentScriptNid = task.prevNid || 0; } finally { globalThis.__currentScriptNid = task.prevNid || 0; }
} }
@@ -1239,11 +1289,21 @@ class Element extends Node {
set className(v) { this.setAttribute("class", v); } set className(v) { this.setAttribute("class", v); }
get namespaceURI() { get namespaceURI() {
// createElementNS records the requested namespace on _ns; an empty string // createElementNS records the requested namespace on _ns; an empty string
// maps to the null namespace per spec. Elements made via createElement (or // maps to the null namespace per spec.
// parsed) have no _ns: default to XHTML, except <svg> which is SVG.
if (this._ns !== undefined) return this._ns === "" ? null : this._ns; if (this._ns !== undefined) return this._ns === "" ? null : this._ns;
if (this.localName === "svg") return "http://www.w3.org/2000/svg"; // Otherwise use the namespace the HTML tree builder assigned. Foreign
return "http://www.w3.org/1999/xhtml"; // content puts the WHOLE <svg>/<math> subtree in that namespace, not just
// the root, so deriving it from the tag name (the old `localName === "svg"`
// check) left every descendant looking like HTML and skipped the SVG-only
// reflections -- notably `get href()`, which then returned a plain string
// instead of an SVGAnimatedString. An element's namespace never changes,
// so cache it like _lname.
if (this._nsCache !== undefined) return this._nsCache;
let ns = _domParse("namespace_uri", this._nid) || "";
// Nodes with no element name recorded fall back to the previous heuristic.
if (!ns) ns = this.localName === "svg" ? "http://www.w3.org/2000/svg" : "http://www.w3.org/1999/xhtml";
this._nsCache = ns;
return ns;
} }
// `inner_html` resolves a <template> to its contents document on the Rust // `inner_html` resolves a <template> to its contents document on the Rust
// side (issue #463), so this needs no template special case. // side (issue #463), so this needs no template special case.
@@ -2632,6 +2692,13 @@ class Document extends Node {
// returned a generic Event for every type, which broke libraries that call // returned a generic Event for every type, which broke libraries that call
// createEvent('CustomEvent').initCustomEvent(...) — see issue #41. // createEvent('CustomEvent').initCustomEvent(...) — see issue #41.
createEvent(type) { createEvent(type) {
const normalized = String(type || '').toLowerCase();
if (normalized === 'promiserejectionevent') {
throw new DOMException(
"The provided event type ('PromiseRejectionEvent') is invalid",
'NotSupportedError'
);
}
const map = { const map = {
'customevent': CustomEvent, 'customevents': CustomEvent, 'customevent': CustomEvent, 'customevents': CustomEvent,
'mouseevent': MouseEvent, 'mouseevents': MouseEvent, 'mouseevent': MouseEvent, 'mouseevents': MouseEvent,
@@ -2646,8 +2713,9 @@ class Document extends Node {
'popstateevent': PopStateEvent, 'popstateevent': PopStateEvent,
'animationevent': AnimationEvent, 'animationevent': AnimationEvent,
'transitionevent': TransitionEvent, 'transitionevent': TransitionEvent,
'storageevent': StorageEvent,
}; };
const Cls = map[String(type || '').toLowerCase()] || Event; const Cls = map[normalized] || Event;
return new Cls(''); return new Cls('');
} }
createRange() { return new Range(); } createRange() { return new Range(); }
@@ -5026,6 +5094,40 @@ globalThis.ToggleEvent = class ToggleEvent extends Event {
}; };
_markNative(globalThis.ToggleEvent); _markNative(globalThis.ToggleEvent);
globalThis.PromiseRejectionEvent = class PromiseRejectionEvent extends Event {
constructor(type, init) {
if (arguments.length < 2 || init == null || !('promise' in Object(init))) {
throw new TypeError(
"Failed to construct 'PromiseRejectionEvent': required member promise is undefined."
);
}
super(type, init);
this.promise = init.promise;
this.reason = init.reason;
}
};
_markNative(globalThis.PromiseRejectionEvent);
globalThis.StorageEvent = class StorageEvent extends Event {
constructor(type, init = {}) {
super(type, init);
this.key = init.key !== undefined ? init.key : null;
this.oldValue = init.oldValue !== undefined ? init.oldValue : null;
this.newValue = init.newValue !== undefined ? init.newValue : null;
this.url = init.url || "";
this.storageArea = init.storageArea || null;
}
initStorageEvent(type, bubbles, cancelable, key, oldValue, newValue, url, storageArea) {
this.initEvent(type, bubbles, cancelable);
this.key = key !== undefined ? key : null;
this.oldValue = oldValue !== undefined ? oldValue : null;
this.newValue = newValue !== undefined ? newValue : null;
this.url = url || "";
this.storageArea = storageArea || null;
}
};
_markNative(globalThis.StorageEvent);
// AbortController / AbortSignal. AbortSignal is a real constructor with a // AbortController / AbortSignal. AbortSignal is a real constructor with a
// prototype, so feature-detection and `AbortSignal.prototype` access work. It // prototype, so feature-detection and `AbortSignal.prototype` access work. It
// carries aborted/reason, supports throwIfAborted(), and fires "abort" to // carries aborted/reason, supports throwIfAborted(), and fires "abort" to
+38 -28
View File
@@ -303,6 +303,14 @@ fn op_dom_inner(state: &OpState, cmd: String, arg1: String, arg2: String) -> Str
let name = dom.with_node(NodeId::new(nid), |n| n.as_element().map(|name| name.local.as_ref().to_ascii_uppercase())).flatten().unwrap_or_default(); let name = dom.with_node(NodeId::new(nid), |n| n.as_element().map(|name| name.local.as_ref().to_ascii_uppercase())).flatten().unwrap_or_default();
serde_json::to_string(&name).unwrap_or("\"\"".into()) serde_json::to_string(&name).unwrap_or("\"\"".into())
} }
// The tree builder already assigns foreign content (an <svg>/<math>
// subtree) its own namespace; expose it so JS does not have to guess
// the namespace from the tag name.
"namespace_uri" => {
let nid = arg1.parse::<u32>().unwrap_or(0);
let ns = dom.with_node(NodeId::new(nid), |n| n.as_element().map(|name| name.ns.as_ref().to_string())).flatten().unwrap_or_default();
serde_json::to_string(&ns).unwrap_or("\"\"".into())
}
"get_attribute" => { "get_attribute" => {
let nid = arg1.parse::<u32>().unwrap_or(0); let nid = arg1.parse::<u32>().unwrap_or(0);
let val = dom.with_node(NodeId::new(nid), |n| n.get_attribute(&arg2).map(|s| s.to_string())).flatten(); let val = dom.with_node(NodeId::new(nid), |n| n.get_attribute(&arg2).map(|s| s.to_string())).flatten();
@@ -570,24 +578,9 @@ fn op_console_msg(state: &OpState, #[string] level: &str, #[string] msg: &str) {
} }
} }
// op_fetch_url backs JS-level `fetch()` and XHR. Pre-#139 it used a // Fallback cache for runtimes that have no owning ObscuraHttpClient, such as
// process-wide `OnceLock<reqwest::Client>` initialised with no proxy, so // a standalone module loader. Browser pages use their context-scoped client
// every JS network call bypassed the configured upstream proxy. We now // below so sequential V8 runtimes never share an async network pool (#453).
// build a client per request, threading whatever `proxy_url` the page's
// ObscuraHttpClient was configured with.
//
// The per-request build cost is negligible (≪1ms) compared with the actual
// network round-trip; the simplification is worth not having to invalidate
// a cache when the proxy is reconfigured between fetches.
//
// Process-wide cache keyed by proxy URL. Previously we built a fresh
// reqwest::Client on every op_fetch_url call (every JS fetch(), XHR,
// dynamic script load). Each build re-initialised TLS roots and a
// fresh connection pool with zero reuse, costing ~5ms per fetch on top
// of any real network work. On an asset-heavy page with 30+ subresources
// that adds ~150ms of pure waste. With the cache, the first fetch on a
// given proxy pays the build cost once and every subsequent fetch reuses
// the same connection pool.
static FETCH_CLIENT_CACHE: std::sync::OnceLock< static FETCH_CLIENT_CACHE: std::sync::OnceLock<
std::sync::RwLock<std::collections::HashMap<String, reqwest::Client>>, std::sync::RwLock<std::collections::HashMap<String, reqwest::Client>>,
> = std::sync::OnceLock::new(); > = std::sync::OnceLock::new();
@@ -626,13 +619,9 @@ fn build_request_client(proxy_url: Option<&str>) -> Result<reqwest::Client, Stri
// send().await errors, which op_fetch_url propagates and the fetch shim turns // send().await errors, which op_fetch_url propagates and the fetch shim turns
// into an XHR `error`/`loadend`. 30s matches the other clients in the // into an XHR `error`/`loadend`. 30s matches the other clients in the
// workspace; OBSCURA_FETCH_TIMEOUT_MS overrides it for tighter cloud limits. // workspace; OBSCURA_FETCH_TIMEOUT_MS overrides it for tighter cloud limits.
let timeout_ms: u64 = std::env::var("OBSCURA_FETCH_TIMEOUT_MS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(30_000);
let mut builder = reqwest::Client::builder() let mut builder = reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none()) .redirect(reqwest::redirect::Policy::none())
.timeout(std::time::Duration::from_millis(timeout_ms)) .timeout(fetch_timeout())
// SSRF guard: also reject hostnames that resolve to a private/loopback IP. // SSRF guard: also reject hostnames that resolve to a private/loopback IP.
.dns_resolver(std::sync::Arc::new(obscura_net::SsrfGuardResolver::new(false))) .dns_resolver(std::sync::Arc::new(obscura_net::SsrfGuardResolver::new(false)))
// Be explicit about pool size: default is unbounded which is fine, // Be explicit about pool size: default is unbounded which is fine,
@@ -651,6 +640,14 @@ fn build_request_client(proxy_url: Option<&str>) -> Result<reqwest::Client, Stri
.map_err(|e| format!("failed to build reqwest::Client: {}", e)) .map_err(|e| format!("failed to build reqwest::Client: {}", e))
} }
fn fetch_timeout() -> std::time::Duration {
let timeout_ms = std::env::var("OBSCURA_FETCH_TIMEOUT_MS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(30_000);
std::time::Duration::from_millis(timeout_ms)
}
/// Cap on the number of redirect hops op_fetch_url will follow. /// Cap on the number of redirect hops op_fetch_url will follow.
/// Matches reqwest's default policy of 10. /// Matches reqwest's default policy of 10.
const FETCH_REDIRECT_LIMIT: usize = 10; const FETCH_REDIRECT_LIMIT: usize = 10;
@@ -681,7 +678,7 @@ async fn op_fetch_url(
} }
} }
let (cookie_jar, in_flight, intercept_tx, proxy_url, callbacks) = { let (cookie_jar, in_flight, intercept_tx, proxy_url, callbacks, http_client) = {
let state_borrow = state.borrow(); let state_borrow = state.borrow();
let gs = state_borrow.borrow::<SharedState>().clone(); let gs = state_borrow.borrow::<SharedState>().clone();
let mut gs = gs.borrow_mut(); let mut gs = gs.borrow_mut();
@@ -713,7 +710,14 @@ async fn op_fetch_url(
} else { } else {
None None
}; };
(jar, in_flight, itx, proxy_url, gs.callbacks.clone()) (
jar,
in_flight,
itx,
proxy_url,
gs.callbacks.clone(),
gs.http_client.clone(),
)
}; };
// Slots the interception channel can override via Continue so a consumer // Slots the interception channel can override via Continue so a consumer
@@ -796,8 +800,11 @@ async fn op_fetch_url(
let method = override_method.unwrap_or(method); let method = override_method.unwrap_or(method);
let body = override_body.unwrap_or(body); let body = override_body.unwrap_or(body);
let client = cached_request_client(proxy_url.as_deref()) let client = match &http_client {
.map_err(deno_error::JsErrorBox::generic)?; Some(client) => client.request_client().await,
None => cached_request_client(proxy_url.as_deref())
.map_err(deno_error::JsErrorBox::generic)?,
};
let request_origin = url::Url::parse(&url) let request_origin = url::Url::parse(&url)
.ok() .ok()
@@ -878,6 +885,7 @@ async fn op_fetch_url(
if needs_preflight { if needs_preflight {
let preflight = client let preflight = client
.request(reqwest::Method::OPTIONS, &url) .request(reqwest::Method::OPTIONS, &url)
.timeout(fetch_timeout())
.header("Origin", &page_origin) .header("Origin", &page_origin)
.header("Access-Control-Request-Method", method.as_str()) .header("Access-Control-Request-Method", method.as_str())
.header( .header(
@@ -911,7 +919,9 @@ async fn op_fetch_url(
let mut current_body = body; let mut current_body = body;
let mut redirects_followed: usize = 0; let mut redirects_followed: usize = 0;
let response = loop { let response = loop {
let mut req = client.request(current_method.clone(), &current_url); let mut req = client
.request(current_method.clone(), &current_url)
.timeout(fetch_timeout());
if is_cross_origin { if is_cross_origin {
req = req.header("Origin", &page_origin); req = req.header("Origin", &page_origin);
+95
View File
@@ -3137,6 +3137,101 @@ mod tests {
assert_eq!(kind, serde_json::json!(true)); assert_eq!(kind, serde_json::json!(true));
} }
#[test]
fn test_promise_rejection_event_requires_promise() {
let mut rt = setup_runtime("<html><body></body></html>");
let result = rt
.evaluate(
r#"(() => {
const promise = Promise.resolve(1);
const event = new PromiseRejectionEvent('unhandledrejection', {
promise,
reason: 'failed'
});
let missingPromiseThrows = false;
try {
new PromiseRejectionEvent('unhandledrejection');
} catch (error) {
missingPromiseThrows = error instanceof TypeError;
}
return [
event instanceof Event,
event.promise === promise,
event.reason === 'failed',
missingPromiseThrows
];
})()"#,
)
.unwrap();
assert_eq!(result, serde_json::json!([true, true, true, true]));
}
#[test]
fn test_create_event_rejects_promise_rejection_event() {
let mut rt = setup_runtime("<html><body></body></html>");
let result = rt
.evaluate(
r#"(() => {
try {
document.createEvent('PromiseRejectionEvent');
return null;
} catch (error) {
return [error.name, error instanceof DOMException];
}
})()"#,
)
.unwrap();
assert_eq!(
result,
serde_json::json!(["NotSupportedError", true])
);
}
#[test]
fn test_storage_event_constructor_and_legacy_factory() {
let mut rt = setup_runtime("<html><body></body></html>");
let result = rt
.evaluate(
r#"(() => {
const event = new StorageEvent('storage', {
key: 'theme',
oldValue: 'light',
newValue: 'dark',
url: 'https://example.test/'
});
const legacy = document.createEvent('StorageEvent');
legacy.initStorageEvent(
'storage', false, false, 'count', '1', '2',
'https://example.test/', null
);
return [
event instanceof Event,
event.key,
event.oldValue,
event.newValue,
event.url,
legacy instanceof StorageEvent,
legacy.key,
legacy.newValue
];
})()"#,
)
.unwrap();
assert_eq!(
result,
serde_json::json!([
true,
"theme",
"light",
"dark",
"https://example.test/",
true,
"count",
"2"
])
);
}
#[test] #[test]
fn test_html_to_markdown_headings() { fn test_html_to_markdown_headings() {
let mut rt = setup_runtime("<html><body><h1>Title</h1><h2>Sub</h2><p>Body</p></body></html>"); let mut rt = setup_runtime("<html><body><h1>Title</h1><h2>Sub</h2><p>Body</p></body></html>");
+210 -71
View File
@@ -1,7 +1,8 @@
use anyhow::Result; use anyhow::Result;
use serde_json::{json, Value}; use serde_json::{json, Value};
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; use tokio::io::{AsyncBufRead, AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
use tokio::net::TcpListener; use tokio::net::TcpListener;
use tokio::time::Duration;
use crate::{dispatch, BrowserState}; use crate::{dispatch, BrowserState};
@@ -12,6 +13,122 @@ use crate::{dispatch, BrowserState};
/// is far above any real JSON-RPC tool call. /// is far above any real JSON-RPC tool call.
const MAX_BODY_BYTES: usize = 16 * 1024 * 1024; const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
/// Maximum time allowed to receive one complete HTTP request (request line,
/// headers, and body). The deadline is shared across all reads so a client
/// cannot keep the sequential MCP server occupied by slowly dribbling data.
const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(30);
enum RequestBody {
NotRead,
MissingLength,
TooLarge,
Read(Vec<u8>),
}
struct HttpRequest {
method: String,
path: String,
accept_sse: bool,
keep_alive: bool,
origin: Option<String>,
body: RequestBody,
}
enum RequestRead {
Closed,
Invalid,
Request(HttpRequest),
}
async fn read_request(
reader: &mut (impl AsyncBufRead + Unpin),
allowed_origins: Option<&str>,
) -> Result<RequestRead> {
let mut request_line = String::new();
if reader.read_line(&mut request_line).await? == 0 {
return Ok(RequestRead::Closed);
}
let request_line = request_line.trim();
if request_line.is_empty() {
return Ok(RequestRead::Closed);
}
let parts: Vec<&str> = request_line.splitn(3, ' ').collect();
if parts.len() < 3 {
return Ok(RequestRead::Invalid);
}
let method = parts[0].to_string();
let path = parts[1].to_string();
let mut content_length: Option<usize> = None;
let mut accept_sse = false;
let mut keep_alive = false;
let mut origin: Option<String> = None;
loop {
let mut line = String::new();
reader.read_line(&mut line).await?;
let trimmed = line.trim_end_matches("\r\n").trim_end_matches('\n');
if trimmed.is_empty() {
break;
}
let lower = trimmed.to_lowercase();
if let Some(v) = lower.strip_prefix("content-length: ") {
content_length = v.trim().parse().ok();
}
if lower.starts_with("origin:") {
if let Some(idx) = trimmed.find(':') {
origin = Some(trimmed[idx + 1..].trim().to_string());
}
}
if lower.contains("text/event-stream") {
accept_sse = true;
}
if lower.starts_with("connection: ") && lower.contains("keep-alive") {
keep_alive = true;
}
}
// Only consume a body for a POST that can reach the MCP route. Invalid
// paths and forbidden origins retain the existing early-response behavior.
let body = if method == "POST"
&& path == "/mcp"
&& origin_allowed(origin.as_deref(), allowed_origins)
{
match content_length {
None => RequestBody::MissingLength,
// Reject the client-supplied size before allocating the body.
Some(len) if len > MAX_BODY_BYTES => RequestBody::TooLarge,
Some(len) => {
let mut body = vec![0u8; len];
reader.read_exact(&mut body).await?;
RequestBody::Read(body)
}
}
} else {
RequestBody::NotRead
};
Ok(RequestRead::Request(HttpRequest {
method,
path,
accept_sse,
keep_alive,
origin,
body,
}))
}
async fn read_request_with_timeout(
reader: &mut (impl AsyncBufRead + Unpin),
allowed_origins: Option<&str>,
timeout: Duration,
) -> Result<RequestRead> {
tokio::time::timeout(timeout, read_request(reader, allowed_origins))
.await
.map_err(|_| anyhow::anyhow!("request read timed out"))?
}
/// Origin allowlist for browser callers, read from `OBSCURA_MCP_ALLOWED_ORIGINS` /// Origin allowlist for browser callers, read from `OBSCURA_MCP_ALLOWED_ORIGINS`
/// (comma-separated). Unset/empty → permissive (unchanged `*`) so hosted /// (comma-separated). Unset/empty → permissive (unchanged `*`) so hosted
/// dashboards keep working (issue #175). /// dashboards keep working (issue #175).
@@ -87,52 +204,24 @@ async fn handle_connection(
let mut reader = BufReader::new(reader); let mut reader = BufReader::new(reader);
loop { loop {
// ── request line ───────────────────────────────────────────────────── let request = match read_request_with_timeout(
let mut request_line = String::new(); &mut reader,
if reader.read_line(&mut request_line).await? == 0 { allowed_origins,
break; REQUEST_READ_TIMEOUT,
} )
let request_line = request_line.trim().to_string(); .await?
if request_line.is_empty() { {
break; RequestRead::Closed | RequestRead::Invalid => break,
} RequestRead::Request(request) => request,
};
let parts: Vec<&str> = request_line.splitn(3, ' ').collect(); let HttpRequest {
if parts.len() < 3 { method,
break; path,
} accept_sse,
let method = parts[0]; keep_alive,
let path = parts[1]; origin,
body,
// ── headers ────────────────────────────────────────────────────────── } = request;
let mut content_length: Option<usize> = None;
let mut accept_sse = false;
let mut keep_alive = false;
let mut origin: Option<String> = None;
loop {
let mut line = String::new();
reader.read_line(&mut line).await?;
let trimmed = line.trim_end_matches("\r\n").trim_end_matches('\n');
if trimmed.is_empty() {
break;
}
let lower = trimmed.to_lowercase();
if let Some(v) = lower.strip_prefix("content-length: ") {
content_length = v.trim().parse().ok();
}
if lower.starts_with("origin:") {
if let Some(idx) = trimmed.find(':') {
origin = Some(trimmed[idx + 1..].trim().to_string());
}
}
if lower.contains("text/event-stream") {
accept_sse = true;
}
if lower.starts_with("connection: ") && lower.contains("keep-alive") {
keep_alive = true;
}
}
// ── routing ────────────────────────────────────────────────────────── // ── routing ──────────────────────────────────────────────────────────
if path != "/mcp" { if path != "/mcp" {
@@ -151,7 +240,7 @@ async fn handle_connection(
} }
let cors = cors_header(origin.as_deref(), allowed_origins); let cors = cors_header(origin.as_deref(), allowed_origins);
match method { match method.as_str() {
"OPTIONS" => { "OPTIONS" => {
// mcp-protocol-version is part of the MCP spec, Authorization / // mcp-protocol-version is part of the MCP spec, Authorization /
// X-API-Key are common for hosted deployments. Without these // X-API-Key are common for hosted deployments. Without these
@@ -169,7 +258,14 @@ async fn handle_connection(
} }
"GET" if accept_sse => { "GET" if accept_sse => {
// SSE stream: hold open and send periodic keep-alive comments // SSE stream: hold open and send periodic keep-alive comments.
// Connections are served sequentially (the browser session is
// `!Send`), so holding this infinite keep-alive loop inline would
// never return to the accept loop and would wedge every later
// request. The keep-alive touches no browser state and the write
// half is `Send + 'static`, so detach the ping loop onto its own
// task and return — the accept loop stays free while the stream
// lives on independently.
let hdr = format!( let hdr = format!(
"HTTP/1.1 200 OK\r\n\ "HTTP/1.1 200 OK\r\n\
Content-Type: text/event-stream\r\n\ Content-Type: text/event-stream\r\n\
@@ -179,36 +275,32 @@ async fn handle_connection(
\r\n" \r\n"
); );
writer.write_all(hdr.as_bytes()).await?; writer.write_all(hdr.as_bytes()).await?;
loop { tokio::spawn(async move {
tokio::time::sleep(tokio::time::Duration::from_secs(15)).await; loop {
if writer.write_all(b": ping\n\n").await.is_err() { tokio::time::sleep(tokio::time::Duration::from_secs(15)).await;
break; if writer.write_all(b": ping\n\n").await.is_err() {
break;
}
let _ = writer.flush().await;
} }
let _ = writer.flush().await; });
} return Ok(());
break;
} }
"POST" => { "POST" => {
let len = match content_length { let body = match body {
Some(n) => n, RequestBody::MissingLength => {
None => {
respond(&mut writer, 400, b"{\"error\":\"missing Content-Length\"}").await?; respond(&mut writer, 400, b"{\"error\":\"missing Content-Length\"}").await?;
break; break;
} }
RequestBody::TooLarge => {
respond(&mut writer, 413, b"{\"error\":\"payload too large\"}").await?;
break;
}
RequestBody::Read(body) => body,
RequestBody::NotRead => unreachable!("valid MCP POST body was not read"),
}; };
// Reject oversized bodies BEFORE allocating: `vec![0u8; len]`
// commits `len` bytes up front, so an attacker-chosen
// Content-Length would otherwise OOM the process (DoS).
if len > MAX_BODY_BYTES {
respond(&mut writer, 413, b"{\"error\":\"payload too large\"}").await?;
break;
}
let mut body = vec![0u8; len];
reader.read_exact(&mut body).await?;
let response = process_body(&body, state).await; let response = process_body(&body, state).await;
let bytes = serde_json::to_vec(&response)?; let bytes = serde_json::to_vec(&response)?;
respond_json(&mut writer, &bytes, &cors).await?; respond_json(&mut writer, &bytes, &cors).await?;
@@ -296,7 +388,12 @@ async fn respond(writer: &mut (impl AsyncWriteExt + Unpin), status: u16, body: &
#[cfg(test)] #[cfg(test)]
mod mcp_hardening_tests { mod mcp_hardening_tests {
use super::{origin_allowed, MAX_BODY_BYTES}; use super::{
origin_allowed, read_request_with_timeout, MAX_BODY_BYTES,
REQUEST_READ_TIMEOUT,
};
use tokio::io::{AsyncWriteExt, BufReader};
use tokio::time::Duration;
#[test] #[test]
fn no_allowlist_is_permissive() { fn no_allowlist_is_permissive() {
@@ -320,4 +417,46 @@ mod mcp_hardening_tests {
assert!(MAX_BODY_BYTES >= 1 << 20); assert!(MAX_BODY_BYTES >= 1 << 20);
assert!(MAX_BODY_BYTES <= 64 << 20); assert!(MAX_BODY_BYTES <= 64 << 20);
} }
#[test]
fn request_read_timeout_is_generous_but_bounded() {
assert!(REQUEST_READ_TIMEOUT >= Duration::from_secs(10));
assert!(REQUEST_READ_TIMEOUT <= Duration::from_secs(60));
}
#[tokio::test]
async fn stalled_request_line_hits_deadline() {
let (_client, server) = tokio::io::duplex(64);
let mut reader = BufReader::new(server);
let err = match read_request_with_timeout(&mut reader, None, Duration::from_millis(20))
.await
{
Err(err) => err,
Ok(_) => panic!("stalled request line should time out"),
};
assert!(err.to_string().contains("timed out"));
}
#[tokio::test]
async fn stalled_request_body_hits_deadline() {
let (mut client, server) = tokio::io::duplex(64);
client
.write_all(
b"POST /mcp HTTP/1.1\r\n\
Content-Length: 8\r\n\
\r\n\
{}",
)
.await
.unwrap();
let mut reader = BufReader::new(server);
let err = match read_request_with_timeout(&mut reader, None, Duration::from_millis(20))
.await
{
Err(err) => err,
Ok(_) => panic!("stalled request body should time out"),
};
assert!(err.to_string().contains("timed out"));
}
} }
+86 -8
View File
@@ -770,9 +770,9 @@ fn tool_fill(args: &Value, state: &mut BrowserState) -> Result<String, String> {
r#"(function(){{ r#"(function(){{
var el = document.querySelector({sel}); var el = document.querySelector({sel});
if (!el) return "error:element not found"; if (!el) return "error:element not found";
el.value = {val}; globalThis.__obscura_setFieldValue(el, "value", {val});
el.dispatchEvent(new Event("input", {{bubbles:true}})); el.dispatchEvent(globalThis.__obscura_markTrusted(new Event("input", {{bubbles:true}})));
el.dispatchEvent(new Event("change", {{bubbles:true}})); el.dispatchEvent(globalThis.__obscura_markTrusted(new Event("change", {{bubbles:true}})));
return "ok"; return "ok";
}})()"#, }})()"#,
sel = serde_json::to_string(&selector).unwrap(), sel = serde_json::to_string(&selector).unwrap(),
@@ -796,8 +796,8 @@ fn tool_type(args: &Value, state: &mut BrowserState) -> Result<String, String> {
r#"(function(){{ r#"(function(){{
var el = document.querySelector({sel}); var el = document.querySelector({sel});
if (!el) return "error:element not found"; if (!el) return "error:element not found";
el.value = (el.value || "") + {txt}; globalThis.__obscura_setFieldValue(el, "value", (el.value || "") + {txt});
el.dispatchEvent(new Event("input", {{bubbles:true}})); el.dispatchEvent(globalThis.__obscura_markTrusted(new Event("input", {{bubbles:true}})));
return "ok"; return "ok";
}})()"#, }})()"#,
sel = serde_json::to_string(&selector).unwrap(), sel = serde_json::to_string(&selector).unwrap(),
@@ -1329,9 +1329,9 @@ fn tool_fill_form(args: &Value, state: &mut BrowserState) -> Result<String, Stri
_ => format!(r#"(function(){{ _ => format!(r#"(function(){{
var el = document.querySelector({sel}); var el = document.querySelector({sel});
if (!el) return "error:not found"; if (!el) return "error:not found";
el.value = {val}; globalThis.__obscura_setFieldValue(el, 'value', {val});
el.dispatchEvent(new Event('input', {{bubbles:true}})); el.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {{bubbles:true}})));
el.dispatchEvent(new Event('change', {{bubbles:true}})); el.dispatchEvent(globalThis.__obscura_markTrusted(new Event('change', {{bubbles:true}})));
return "ok"; return "ok";
}})()"#, sel = serde_json::to_string(&selector).unwrap(), val = serde_json::to_string(value).unwrap()), }})()"#, sel = serde_json::to_string(&selector).unwrap(), val = serde_json::to_string(value).unwrap()),
}; };
@@ -1769,3 +1769,81 @@ fn tool_set_storage_state(args: &Value, state: &mut BrowserState) -> Result<Stri
} }
Ok(format!("Restored {applied} state entries.")) Ok(format!("Restored {applied} state entries."))
} }
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test(flavor = "current_thread")]
async fn fill_tools_notify_controlled_input_tracker() {
let mut state = BrowserState::new(None, None, false);
state
.page_mut()
.navigate("data:text/html,<div id=root><input id=field></div>")
.await
.expect("test page should navigate");
state.page_mut().evaluate(
r#"(function () {
var root = document.getElementById('root');
var input = document.getElementById('field');
var descriptor = Object.getOwnPropertyDescriptor(input.constructor.prototype, 'value');
var tracked = String(input.value);
Object.defineProperty(input, 'value', {
configurable: true,
get: function () { return descriptor.get.call(this); },
set: function (value) {
tracked = String(value);
descriptor.set.call(this, value);
}
});
window.__controlledState = '';
window.__controlledUpdates = 0;
window.__lastInputTarget = '';
window.__lastInputTrusted = false;
root.addEventListener('input', function (event) {
window.__lastInputTarget = event.target.id;
window.__lastInputTrusted = event.isTrusted;
var next = String(event.target.value);
if (next !== tracked) {
tracked = next;
window.__controlledState = next;
window.__controlledUpdates++;
}
});
})()"#,
);
tool_fill(
&json!({ "selector": "#field", "value": "filled" }),
&mut state,
)
.expect("browser_fill should succeed");
tool_type(
&json!({ "selector": "#field", "text": "-typed" }),
&mut state,
)
.expect("browser_type should succeed");
tool_fill_form(
&json!({
"fields": [{ "selector": "#field", "value": "form-filled" }]
}),
&mut state,
)
.expect("browser_fill_form should succeed");
let actual = state.page_mut().evaluate(
r#"JSON.stringify({
domValue: document.getElementById('field').value,
controlledState: window.__controlledState,
controlledUpdates: window.__controlledUpdates,
lastInputTarget: window.__lastInputTarget,
lastInputTrusted: window.__lastInputTrusted
})"#,
);
assert_eq!(
actual,
json!(r#"{"domValue":"form-filled","controlledState":"form-filled","controlledUpdates":3,"lastInputTarget":"field","lastInputTrusted":true}"#),
);
}
}
@@ -0,0 +1,97 @@
//! Regression test: an open SSE stream (`GET /mcp` with
//! `Accept: text/event-stream`) must not wedge the whole MCP HTTP server.
//!
//! The server serves connections sequentially on the single current-thread
//! runtime (the browser session is `!Send`). The SSE keep-alive is an infinite
//! ping loop, so if it is held inline it never returns and the `accept()` loop
//! never runs again — every later request hangs forever. The keep-alive carries
//! no browser state, so it must be detached and the connection handler must
//! return, leaving the accept loop free.
use std::net::TcpListener as StdListener;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpStream;
use tokio::task::LocalSet;
use tokio::time::{sleep, timeout};
fn pick_free_port() -> u16 {
let l = StdListener::bind("127.0.0.1:0").unwrap();
let p = l.local_addr().unwrap().port();
drop(l);
p
}
#[tokio::test(flavor = "current_thread")]
async fn open_sse_stream_does_not_block_other_requests() {
let port = pick_free_port();
let local = LocalSet::new();
let server = local.spawn_local(async move {
let _ = obscura_mcp::http::run("127.0.0.1".to_string(), port, None, None, false).await;
});
local
.run_until(async {
// Wait for the listener to bind.
for _ in 0..40 {
if TcpStream::connect(("127.0.0.1", port)).await.is_ok() {
break;
}
sleep(Duration::from_millis(50)).await;
}
// Connection A: open the SSE stream and read its headers so the
// server-side handler has entered its keep-alive path. Keep it open.
let mut sse = TcpStream::connect(("127.0.0.1", port))
.await
.expect("MCP server did not come up");
let get = b"GET /mcp HTTP/1.1\r\n\
Host: 127.0.0.1\r\n\
Accept: text/event-stream\r\n\
\r\n";
sse.write_all(get).await.unwrap();
sse.flush().await.unwrap();
let mut hbuf = [0u8; 512];
let hn = timeout(Duration::from_secs(2), sse.read(&mut hbuf))
.await
.expect("SSE headers read timed out")
.expect("SSE headers read failed");
let sse_head = String::from_utf8_lossy(&hbuf[..hn]).to_lowercase();
assert!(
sse_head.contains("text/event-stream"),
"expected an SSE response on the GET stream, got:\n{sse_head}"
);
// Connection B: while A's SSE stream is still open, a second request
// must still be accepted and answered promptly. Pre-fix this hangs
// because the SSE loop never yields the accept loop.
let mut stream = TcpStream::connect(("127.0.0.1", port))
.await
.expect("second connection refused");
let req = b"OPTIONS /mcp HTTP/1.1\r\n\
Host: 127.0.0.1\r\n\
Origin: https://dashboard.example.com\r\n\
Access-Control-Request-Method: POST\r\n\
\r\n";
stream.write_all(req).await.unwrap();
stream.flush().await.unwrap();
let mut buf = [0u8; 1024];
let n = timeout(Duration::from_secs(3), stream.read(&mut buf))
.await
.expect("second request timed out — the SSE stream wedged the server")
.expect("read failed");
let response = String::from_utf8_lossy(&buf[..n]).to_string();
server.abort();
drop(sse);
assert!(
response.starts_with("HTTP/1.1 204"),
"expected the second request to be served (204), got:\n{response}"
);
})
.await;
}
+9
View File
@@ -25,6 +25,15 @@ tempfile = "3"
# Pin exact versions so the build always resolves to the API wreq_client.rs targets. # Pin exact versions so the build always resolves to the API wreq_client.rs targets.
wreq-util = { version = "=3.0.0-rc.12", optional = true } wreq-util = { version = "=3.0.0-rc.12", optional = true }
[dev-dependencies]
# TLS fixtures for the configured-roots tests: rcgen mints a throwaway CA + leaf,
# tokio-rustls serves one HTTPS response with them. Pinned to the ring
# provider so the test binary has a single rustls CryptoProvider (reqwest's
# rustls also uses ring; adding aws-lc-rs would make provider auto-selection
# panic).
rcgen = "0.13"
tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12", "logging"] }
# `prefix-symbols` renames BoringSSL exports to avoid clashes with system OpenSSL, # `prefix-symbols` renames BoringSSL exports to avoid clashes with system OpenSSL,
# but the renaming is only correct on Linux and Android (per wreq's README). # but the renaming is only correct on Linux and Android (per wreq's README).
# Enabling it on other platforms produces unresolved `build_script_main_*` # Enabling it on other platforms produces unresolved `build_script_main_*`
+172 -2
View File
@@ -1,5 +1,5 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::{Arc, OnceLock};
use std::time::Duration; use std::time::Duration;
use std::net::{IpAddr, SocketAddr}; use std::net::{IpAddr, SocketAddr};
@@ -14,6 +14,52 @@ use url::Url;
use crate::cookies::CookieJar; use crate::cookies::CookieJar;
use crate::interceptor::{InterceptAction, RequestInterceptor}; use crate::interceptor::{InterceptAction, RequestInterceptor};
fn configured_root_paths() -> Vec<std::path::PathBuf> {
let mut paths = Vec::new();
if let Some(path) = std::env::var_os("SSL_CERT_FILE").filter(|path| !path.is_empty()) {
paths.push(path.into());
}
if let Some(directory) = std::env::var_os("SSL_CERT_DIR").filter(|path| !path.is_empty()) {
match std::fs::read_dir(directory) {
Ok(entries) => {
paths.extend(entries.filter_map(|entry| entry.ok().map(|entry| entry.path())));
}
Err(error) => {
tracing::warn!(%error, "failed to read SSL_CERT_DIR");
}
}
}
paths.sort();
paths
}
fn configured_root_certificates() -> &'static [reqwest::Certificate] {
static ROOTS: OnceLock<Vec<reqwest::Certificate>> = OnceLock::new();
ROOTS.get_or_init(|| {
let mut certificates = Vec::new();
for path in configured_root_paths() {
let bytes = match std::fs::read(&path) {
Ok(bytes) => bytes,
Err(error) => {
tracing::warn!(%error, path = %path.display(), "failed to read CA certificate file");
continue;
}
};
match reqwest::Certificate::from_pem_bundle(&bytes) {
Ok(mut bundle) if !bundle.is_empty() => certificates.append(&mut bundle),
_ => match reqwest::Certificate::from_der(&bytes) {
Ok(certificate) => certificates.push(certificate),
Err(error) => {
tracing::warn!(%error, path = %path.display(), "failed to parse CA certificate file");
}
},
}
}
certificates
})
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct Response { pub struct Response {
pub url: Url, pub url: Url,
@@ -456,6 +502,14 @@ impl ObscuraHttpClient {
.dns_resolver(Arc::new(SsrfGuardResolver::new(self.allow_private_network))) .dns_resolver(Arc::new(SsrfGuardResolver::new(self.allow_private_network)))
; ;
if std::env::var_os("SSL_CERT_FILE").is_some()
|| std::env::var_os("SSL_CERT_DIR").is_some()
{
for certificate in configured_root_certificates() {
builder = builder.add_root_certificate(certificate.clone());
}
}
if let Some(ref proxy) = self.proxy_url { if let Some(ref proxy) = self.proxy_url {
if let Ok(p) = reqwest::Proxy::all(proxy.as_str()) { if let Ok(p) = reqwest::Proxy::all(proxy.as_str()) {
builder = builder.proxy(p); builder = builder.proxy(p);
@@ -466,6 +520,15 @@ impl ObscuraHttpClient {
}).await }).await
} }
/// Clone the request client owned by this browser context.
///
/// Scripted fetch/XHR uses the same pool as navigation instead of a
/// process-global client. This keeps its async network state inside the
/// same ownership boundary as the V8 runtime (issue #453).
pub async fn request_client(&self) -> Client {
self.get_client().await.clone()
}
/// Read-only accessor for the proxy URL the client was configured with /// Read-only accessor for the proxy URL the client was configured with
/// (if any). Exposed so callers outside the `obscura-net` crate — notably /// (if any). Exposed so callers outside the `obscura-net` crate — notably
/// `op_fetch_url` in `obscura-js` (#139) — can route their own reqwest /// `op_fetch_url` in `obscura-js` (#139) — can route their own reqwest
@@ -755,10 +818,12 @@ pub enum ObscuraNetError {
#[cfg(test)] #[cfg(test)]
mod ssrf_tests { mod ssrf_tests {
use super::{is_forbidden_ip, validate_url, SsrfGuardResolver}; use super::{is_forbidden_ip, validate_url, ObscuraHttpClient, SsrfGuardResolver};
use crate::cookies::CookieJar;
use reqwest::dns::{Name, Resolve}; use reqwest::dns::{Name, Resolve};
use std::net::IpAddr; use std::net::IpAddr;
use std::str::FromStr; use std::str::FromStr;
use std::sync::Arc;
use url::Url; use url::Url;
fn ip(s: &str) -> IpAddr { fn ip(s: &str) -> IpAddr {
@@ -843,4 +908,109 @@ mod ssrf_tests {
), ),
} }
} }
/// Mint a throwaway CA plus a 127.0.0.1 leaf it signed, and serve one
/// canned HTTPS response with the leaf on an ephemeral port. Returns the
/// port and the CA certificate as PEM.
async fn https_fixture_with_private_ca() -> (u16, String) {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let ca_key = rcgen::KeyPair::generate().unwrap();
let mut ca_params = rcgen::CertificateParams::new(Vec::new()).unwrap();
ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained);
let ca_cert = ca_params.self_signed(&ca_key).unwrap();
let leaf_key = rcgen::KeyPair::generate().unwrap();
let leaf_params =
rcgen::CertificateParams::new(vec!["127.0.0.1".to_string()]).unwrap();
let leaf_cert = leaf_params.signed_by(&leaf_key, &ca_cert, &ca_key).unwrap();
let certs = vec![tokio_rustls::rustls::pki_types::CertificateDer::from(
leaf_cert.der().to_vec(),
)];
let key = tokio_rustls::rustls::pki_types::PrivateKeyDer::Pkcs8(
tokio_rustls::rustls::pki_types::PrivatePkcs8KeyDer::from(
leaf_key.serialize_der(),
),
);
let config = tokio_rustls::rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(certs, key)
.unwrap();
let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
tokio::spawn(async move {
while let Ok((stream, _)) = listener.accept().await {
let acceptor = acceptor.clone();
tokio::spawn(async move {
let Ok(mut tls) = acceptor.accept(stream).await else {
return; // Handshake rejection is the point of one test.
};
let mut buf = [0u8; 1024];
let _ = tls.read(&mut buf).await;
let body = "private ca ok";
let resp = format!(
"HTTP/1.1 200 OK\r\ncontent-type: text/plain\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
body.len(),
body
);
let _ = tls.write_all(resp.as_bytes()).await;
let _ = tls.shutdown().await;
});
}
});
(port, ca_cert.pem())
}
// The two configured-roots tests set/rely on SSL_CERT_FILE, which is
// cached once per process at client build. They are only correct under
// `cargo nextest` (one process per test), the same constraint the whole
// workspace already has.
#[tokio::test]
async fn configured_roots_trust_a_private_ca_via_ssl_cert_file() {
let (port, ca_pem) = https_fixture_with_private_ca().await;
let ca_file = tempfile::NamedTempFile::new().unwrap();
std::fs::write(ca_file.path(), ca_pem).unwrap();
std::env::set_var("SSL_CERT_FILE", ca_file.path());
let client =
ObscuraHttpClient::with_full_options(Arc::new(CookieJar::new()), None, true);
let url = Url::parse(&format!("https://127.0.0.1:{port}/")).unwrap();
let resp = client.fetch(&url).await.expect("private CA in SSL_CERT_FILE must be trusted");
assert_eq!(resp.status, 200);
assert_eq!(resp.text(), "private ca ok");
}
#[tokio::test]
async fn configured_roots_trust_a_private_ca_via_ssl_cert_dir() {
let (port, ca_pem) = https_fixture_with_private_ca().await;
let ca_dir = tempfile::tempdir().unwrap();
std::fs::write(ca_dir.path().join("private-ca.pem"), ca_pem).unwrap();
std::env::set_var("SSL_CERT_DIR", ca_dir.path());
let client =
ObscuraHttpClient::with_full_options(Arc::new(CookieJar::new()), None, true);
let url = Url::parse(&format!("https://127.0.0.1:{port}/")).unwrap();
let resp = client
.fetch(&url)
.await
.expect("private CA in SSL_CERT_DIR must be trusted");
assert_eq!(resp.status, 200);
assert_eq!(resp.text(), "private ca ok");
}
#[tokio::test]
async fn private_ca_is_still_rejected_without_ssl_cert_file() {
// The same fixture that the SSL_CERT_FILE test trusts must fail here. The
// listener is reachable (same setup), so an Err can only be TLS.
let (port, _ca_pem) = https_fixture_with_private_ca().await;
let client =
ObscuraHttpClient::with_full_options(Arc::new(CookieJar::new()), None, true);
let url = Url::parse(&format!("https://127.0.0.1:{port}/")).unwrap();
assert!(client.fetch(&url).await.is_err(), "unknown CA must be rejected");
}
} }
@@ -0,0 +1,89 @@
use std::io::{Read, Write};
use obscura::Browser;
fn spawn_server() -> String {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
std::thread::spawn(move || {
for incoming in listener.incoming() {
let Ok(mut stream) = incoming else {
continue;
};
std::thread::spawn(move || {
let mut request = [0u8; 2048];
let _ = stream.read(&mut request);
let request = std::str::from_utf8(&request).unwrap_or("");
let path = request.split_whitespace().nth(1).unwrap_or("/");
let (content_type, body) = if path.starts_with("/api") {
("application/json", r#"{"ok":true}"#)
} else {
(
"text/html",
"<!doctype html><html><head><title>fixture</title></head><body></body></html>",
)
};
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: {}\r\nContent-Length: {}\r\nAccess-Control-Allow-Origin: *\r\nConnection: close\r\n\r\n{}",
content_type,
body.len(),
body,
);
let _ = stream.write_all(response.as_bytes());
let _ = stream.shutdown(std::net::Shutdown::Both);
});
}
});
format!("http://{}", addr)
}
async fn run_browser(base: &str) {
let browser = Browser::new().unwrap();
let mut page = browser.new_page().await.unwrap();
page.goto(base).await.unwrap();
let script = format!(
r#"(function() {{
var done = 0;
function mark() {{
done += 1;
document.body.setAttribute('data-done', String(done));
}}
for (var i = 0; i < 8; i++) {{
fetch('{base}/api?fetch=' + i)
.then(function(r) {{ return r.json(); }})
.then(function() {{ return fetch('{base}/api?nested=' + i); }})
.then(mark)
.catch(function() {{}});
}}
for (var j = 0; j < 4; j++) {{
var xhr = new XMLHttpRequest();
xhr.open('GET', '{base}/api?xhr=' + j);
xhr.addEventListener('load', mark);
xhr.send();
}}
}})()"#,
);
page.evaluate(&script);
for _ in 0..20 {
page.settle(250).await;
if page.evaluate("document.body.getAttribute('data-done')") == serde_json::json!("12") {
break;
}
}
assert_eq!(
page.evaluate("document.body.getAttribute('data-done')"),
serde_json::json!("12")
);
}
#[tokio::test(flavor = "current_thread")]
async fn sequential_browsers_complete_concurrent_fetch_and_xhr() {
std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1");
let base = spawn_server();
for _ in 0..8 {
run_browser(&base).await;
}
}
+22
View File
@@ -24,6 +24,28 @@ cargo build --release --features stealth
Adds TLS fingerprint randomization and the tracker blocklist. See [Configure stealth and proxies](Configure-stealth-and-proxies.md). Adds TLS fingerprint randomization and the tracker blocklist. See [Configure stealth and proxies](Configure-stealth-and-proxies.md).
The stealth feature builds BoringSSL and generates Rust bindings. In addition
to the default requirements, install CMake, Clang, and the libclang/LLVM
development libraries. On Ubuntu/Debian:
```bash
sudo apt-get install build-essential cmake clang libclang-dev llvm-dev
```
On macOS, install the Xcode Command Line Tools and CMake. On Windows, install
the Visual Studio C++ Build Tools, CMake, and LLVM/Clang. Ensure the directory
containing `libclang` is available through `LIBCLANG_PATH` if bindgen cannot
locate it automatically.
On macOS 26 with the standalone Command Line Tools, Apple Clang may not find
libc++ while compiling BoringSSL. Use the active SDK for that build:
```bash
SDK_PATH="$(xcrun --show-sdk-path)"
SDKROOT="$SDK_PATH" CXXFLAGS="-isystem $SDK_PATH/usr/include/c++/v1" \
cargo build --release --features stealth
```
## OpenSSL on older systems ## OpenSSL on older systems
If the build fails on the vendored OpenSSL with an AVX-512 assembler error (common on older VPS hosts): If the build fails on the vendored OpenSSL with an AVX-512 assembler error (common on older VPS hosts):
+4 -1
View File
@@ -15,7 +15,10 @@ What `--stealth` changes:
- Loads a tracker blocklist that drops requests to known analytics and fingerprinting endpoints. - Loads a tracker blocklist that drops requests to known analytics and fingerprinting endpoints.
- Bundles webpki roots instead of relying on the system store. - Bundles webpki roots instead of relying on the system store.
Requires a build that includes the stealth feature. Release binaries on the Releases page include it. To build it yourself: Requires a build that includes the stealth feature. The Releases page provides
explicitly named `-stealth` archives alongside the lean default archives, so
users who do not need wreq/BoringSSL do not pay its binary-size or RSS cost. To
build the stealth variant yourself:
```bash ```bash
cargo build --release --features stealth cargo build --release --features stealth
+4
View File
@@ -59,6 +59,10 @@ See [Build from source](Build-from-source.md).
- `obscura`: CLI and CDP server. - `obscura`: CLI and CDP server.
- `obscura-worker`: helper for the parallel `scrape` command. Keep both in the same directory. - `obscura-worker`: helper for the parallel `scrape` command. Keep both in the same directory.
The standard archive names contain the lean default build. Choose the matching
archive with a `-stealth` suffix when you need the wreq/BoringSSL transport for
TLS impersonation.
## Smoke test ## Smoke test
```bash ```bash