diff --git a/.cargo/config.toml b/.cargo/config.toml index 75f6d91..8af59dd 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,12 +1,2 @@ [env] RUST_TEST_THREADS = "1" - -# macOS 26 (Tahoe) moved libc++ headers inside the SDK; Apple clang 17 no longer -# finds them at the old search path. This causes boring-sys's cmake step to fail -# when building with --features stealth. Setting CXXFLAGS and SDKROOT here makes -# them visible to all build scripts (including boring-sys's) so cmake can locate -# the headers. force=false lets CI or developers override via their shell env. -# Non-macOS platforms ignore SDKROOT; the -isystem path is silently skipped if -# it doesn't exist. See: https://github.com/h4ckf0r0day/obscura/issues/136 -CXXFLAGS = { value = "-isystem /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/c++/v1", force = false } -SDKROOT = { value = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk", force = false } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cf439dc..4a4af7c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,41 +50,93 @@ jobs: with: targets: ${{ matrix.target }} - - name: Build + - name: Install stealth build dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y cmake clang libclang-dev llvm-dev + + - name: Configure libclang (macOS) + if: runner.os == 'macOS' + shell: bash + run: | + # Use one coherent SDK. A hard-coded CommandLineTools include path can + # mix libc++ headers with the selected Xcode SDK and break BoringSSL. + XCODE_PATH="$(xcode-select -p)" + SDK_PATH="$(xcrun --show-sdk-path)" + echo "SDKROOT=$SDK_PATH" >> "$GITHUB_ENV" + echo "CXXFLAGS=-isystem $SDK_PATH/usr/include/c++/v1" >> "$GITHUB_ENV" + echo "LIBCLANG_PATH=$XCODE_PATH/Toolchains/XcodeDefault.xctoolchain/usr/lib" >> "$GITHUB_ENV" + + - name: Configure libclang (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + "LIBCLANG_PATH=C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Build default release binary run: cargo build --release --target ${{ matrix.target }} - - name: Build stealth - run: cargo build --release --target ${{ matrix.target }} --features stealth - continue-on-error: true + - name: Stage default release binary + shell: bash + run: | + BIN_DIR=target/${{ matrix.target }}/release + mkdir -p dist/default + if [ -f "$BIN_DIR/obscura.exe" ]; then + cp "$BIN_DIR/obscura.exe" "$BIN_DIR/obscura-worker.exe" dist/default/ + else + cp "$BIN_DIR/obscura" "$BIN_DIR/obscura-worker" dist/default/ + fi - - name: Smoke test (V8 snapshot) + # Build and package stealth separately so the default download keeps its + # lower RSS while the advertised wreq/BoringSSL variant cannot silently + # fall back to a non-stealth binary. + - name: Build stealth release binary + run: cargo build --release --target ${{ matrix.target }} --features stealth + + - name: Stage stealth release binary + shell: bash + run: | + BIN_DIR=target/${{ matrix.target }}/release + mkdir -p dist/stealth + if [ -f "$BIN_DIR/obscura.exe" ]; then + cp "$BIN_DIR/obscura.exe" "$BIN_DIR/obscura-worker.exe" dist/stealth/ + else + cp "$BIN_DIR/obscura" "$BIN_DIR/obscura-worker" dist/stealth/ + fi + + - name: Smoke test release binaries (V8 snapshot) # Run the V8 path once so a release never ships a binary that crashes at # isolate init from a mismatched snapshot (issue #290). The data: URL # keeps it offline. Every target here is native, so the runner can - # execute what it just built. + # execute both variants it just built. shell: bash run: | - BIN=target/${{ matrix.target }}/release/obscura - [ -f "$BIN.exe" ] && BIN="$BIN.exe" - "$BIN" fetch "data:text/html,ok" --eval "1+1" + for VARIANT in default stealth; do + BIN=dist/$VARIANT/obscura + [ -f "$BIN.exe" ] && BIN="$BIN.exe" + "$BIN" fetch "data:text/html,ok" --eval "1+1" + done - name: Package (Unix) if: runner.os != 'Windows' run: | - cd target/${{ matrix.target }}/release - tar czf ../../../${{ matrix.name }}.tar.gz obscura obscura-worker - cd ../../.. + tar czf ${{ matrix.name }}.tar.gz -C dist/default obscura obscura-worker + tar czf ${{ matrix.name }}-stealth.tar.gz -C dist/stealth obscura obscura-worker - name: Package (Windows) if: runner.os == 'Windows' run: | - cd target/${{ matrix.target }}/release - 7z a ../../../${{ matrix.name }}.zip obscura.exe obscura-worker.exe - cd ../../.. + cd dist/default + 7z a ../../${{ matrix.name }}.zip obscura.exe obscura-worker.exe + cd ../stealth + 7z a ../../${{ matrix.name }}-stealth.zip obscura.exe obscura-worker.exe - name: Upload uses: softprops/action-gh-release@v2 with: files: | ${{ matrix.name }}.tar.gz + ${{ matrix.name }}-stealth.tar.gz ${{ matrix.name }}.zip + ${{ matrix.name }}-stealth.zip diff --git a/Cargo.lock b/Cargo.lock index aff873d..85f8d82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -697,6 +697,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + [[package]] name = "derive_more" version = "0.99.20" @@ -1608,6 +1614,12 @@ dependencies = [ "rand 0.8.6", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.46" @@ -1764,12 +1776,14 @@ version = "0.1.0" dependencies = [ "async-trait", "encoding_rs", + "rcgen", "reqwest", "serde", "serde_json", "tempfile", "thiserror 2.0.18", "tokio", + "tokio-rustls", "tracing", "url", "wreq", @@ -1850,6 +1864,16 @@ dependencies = [ "hmac", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1961,6 +1985,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2040,9 +2070,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", "getrandom 0.3.4", @@ -2147,6 +2177,19 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2275,6 +2318,7 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -2737,6 +2781,25 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + [[package]] name = "tinystr" version = "0.8.3" @@ -3682,6 +3745,15 @@ dependencies = [ "tap", ] +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "yoke" version = "0.8.3" diff --git a/README.md b/README.md index c4d0bfd..724732a 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@

Obscura

-

Obscura

- +

+ h4ckf0r0day%2Fobscura | Trendshift +

Documentation Website Book a demo Releases

-

The open-source headless browser for AI agents and web scraping.
Lightweight, stealthy, and built in Rust. @@ -132,6 +132,26 @@ Want to sponsor? Email [hello@obscura.sh](mailto:hello@obscura.sh). 🎁 Use discount code RAPID10 to get 10% off.

+ + + + Thordata + + + + πŸš€ Obscura Γ— Thordata
+ Need more stable proxies for automation, public web scraping, SEO, or ad verification? Power your workflow with Thordata.

+ 🌍 Residential Proxies: 100M+ real residential IPs in 190+ countries
+ ♾️ Unlimited Proxies: for heavy traffic and high concurrency
+ ✨ Recently upgraded residential IP pool
+ ✨ Unlimited Concurrent Sessions
+ πŸ” Rotating & Sticky Sessions
+ πŸ“ Flexible GEO targeting
+ ⚑ Stable HTTP(S) connections

+ 🎁 Use code obscura for 10% off.

 + Get started with a 3-day free trial and test Thordata with your own workflow. + + ## Install @@ -153,6 +173,9 @@ tar xzf obscura-aarch64-linux.tar.gz # Arch Linux (AUR) yay -S obscura-browser +# NixOS +nix-env -iA nixpkgs.obscura + # macOS Apple Silicon curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-aarch64-macos.tar.gz tar xzf obscura-aarch64-macos.tar.gz @@ -169,6 +192,10 @@ No Chrome, no Node.js, no dependencies. Release archives include both `obscura` and `obscura-worker`; keep them in the same directory for the parallel `scrape` command. +Default archive names contain the lean binary. Archives ending in +`-stealth.tar.gz` or `-stealth.zip` additionally include the wreq/BoringSSL +transport used for TLS impersonation. + Linux release builds target Ubuntu 22.04 so the downloaded binary remains usable on common LTS servers with glibc 2.35+. @@ -192,6 +219,14 @@ cargo build --release --features stealth ``` Requires Rust 1.75+ ([rustup.rs](https://rustup.rs)). First build takes ~5 min (V8 compiles from source, cached after). +The stealth build also compiles BoringSSL and generates bindings, so it needs +CMake, Clang, and the libclang/LLVM development libraries. On Ubuntu/Debian: + +```bash +sudo apt-get install build-essential cmake clang libclang-dev llvm-dev +``` + +The default build does not require these additional stealth dependencies. ## Quick Start diff --git a/assets/sponsors/thordatalogo.png b/assets/sponsors/thordatalogo.png new file mode 100644 index 0000000..a06cf0f Binary files /dev/null and b/assets/sponsors/thordatalogo.png differ diff --git a/crates/obscura-cdp/tests/dynamic_script_onload_fires.rs b/crates/obscura-cdp/tests/dynamic_script_onload_fires.rs index d80a92e..bc2f4b1 100644 --- a/crates/obscura-cdp/tests/dynamic_script_onload_fires.rs +++ b/crates/obscura-cdp/tests/dynamic_script_onload_fires.rs @@ -117,3 +117,134 @@ async fn dynamic_external_scripts_execute_and_fire_load() { "dynamic scripts must execute and fire load before navigation settles" ); } + +#[tokio::test(flavor = "current_thread")] +async fn dynamic_data_scripts_execute_before_chained_load_handlers() { + let mut ctx = CdpContext::new(); + let page_id = ctx.create_page(); + let session_id = "session-1"; + ctx.sessions.insert(session_id.to_string(), page_id); + + cdp( + &mut ctx, + 1, + "Page.navigate", + json!({ + "url": "data:text/html,", + "waitUntil": "load" + }), + session_id, + ) + .await; + + cdp( + &mut ctx, + 2, + "Runtime.evaluate", + json!({ + "expression": r#"(function () { + var state = { + aExec: false, + aLoad: false, + bExec: false, + bLoad: false, + cExec: false, + cLoad: false + }; + window.__dataScriptState = state; + + var a = document.createElement('script'); + a.src = 'data:,window.__dataScriptState.aExec=true'; + a.onload = function () { + state.aLoad = true; + var b = document.createElement('script'); + b.src = 'data:text/html,' + encodeURIComponent('window.__dataScriptState.bExec=true'); + b.onload = function () { + state.bLoad = true; + var c = document.createElement('script'); + c.src = 'data:text/javascript;base64,' + + btoa('window.__dataScriptState.cExec=true').replace(/=+$/, '') + + '#ignored-fragment'; + c.onload = function () { state.cLoad = true; }; + document.head.appendChild(c); + }; + document.head.appendChild(b); + }; + document.head.appendChild(a); + return 'kicked'; + })()"#, + "returnByValue": true, + }), + session_id, + ) + .await; + + ctx.pages[0].settle(500).await; + + let result = cdp( + &mut ctx, + 3, + "Runtime.evaluate", + json!({ + "expression": "JSON.stringify(window.__dataScriptState)", + "returnByValue": true, + }), + session_id, + ) + .await; + assert_eq!( + result["result"]["value"], + r#"{"aExec":true,"aLoad":true,"bExec":true,"bLoad":true,"cExec":true,"cLoad":true}"#, + "data URL script bodies must execute before each chained load handler" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn invalid_dynamic_data_script_fires_error_not_load() { + let mut ctx = CdpContext::new(); + let page_id = ctx.create_page(); + let session_id = "session-1"; + ctx.sessions.insert(session_id.to_string(), page_id); + + cdp( + &mut ctx, + 1, + "Page.navigate", + json!({"url": "data:text/html,", "waitUntil": "load"}), + session_id, + ) + .await; + + cdp( + &mut ctx, + 2, + "Runtime.callFunctionOn", + json!({ + "functionDeclaration": r#"function () { + window.__invalidDataScript = { error: false, load: false }; + var script = document.createElement('script'); + script.src = 'data:text/javascript;base64,!'; + script.onerror = function () { window.__invalidDataScript.error = true; }; + script.onload = function () { window.__invalidDataScript.load = true; }; + document.head.appendChild(script); + }"#, + "awaitPromise": true, + }), + session_id, + ) + .await; + + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let result = cdp( + &mut ctx, + 3, + "Runtime.evaluate", + json!({ + "expression": "JSON.stringify(window.__invalidDataScript)", + "returnByValue": true, + }), + session_id, + ) + .await; + assert_eq!(result["result"]["value"], r#"{"error":true,"load":false}"#); +} diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js index 3ce7268..a63d335 100644 --- a/crates/obscura-js/js/bootstrap.js +++ b/crates/obscura-js/js/bootstrap.js @@ -18,7 +18,7 @@ '__obscura_hw', '__obscura_mem', '__documentReadyState__', '__currentUrl', // internal helpers (var-declared throughout the file) - '__processDynScriptQueue', '_markNative', '_fpRand', '_fpNoise', + '__processDynScriptQueue', '_decodeDataScriptUrl', '_markNative', '_fpRand', '_fpNoise', '_fpCache', '_getFp', '_fp', '_splitAsciiWhitespace', '_getElementsByClassName', '_docEncoding', '_docIsUtf8', '_isSpecialScheme', '_applyDocQueryEncoding', '_anchorBase', @@ -163,6 +163,51 @@ let _fpSeed = 0; // when SPAs dynamically insert multiple