diff --git a/crates/obscura-browser/Cargo.toml b/crates/obscura-browser/Cargo.toml index 0179e94..1ed64cd 100644 --- a/crates/obscura-browser/Cargo.toml +++ b/crates/obscura-browser/Cargo.toml @@ -5,7 +5,7 @@ edition.workspace = true [features] default = [] -stealth = ["obscura-net/stealth"] +stealth = ["obscura-net/stealth", "obscura-js/stealth"] [dependencies] obscura-dom = { workspace = true } diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs index 3264905..393ca9c 100644 --- a/crates/obscura-browser/src/page.rs +++ b/crates/obscura-browser/src/page.rs @@ -286,9 +286,11 @@ impl Page { if self.stealth_client.is_some() { rt.set_stealth(true); rt.set_user_agent(obscura_net::STEALTH_USER_AGENT); - // Match platform to the Linux UA so navigator.platform and - // userAgentData.platform are consistent with navigator.userAgent. - rt.set_platform("Linux x86_64", "Linux", ""); + rt.set_platform( + obscura_net::STEALTH_NAVIGATOR_PLATFORM, + obscura_net::STEALTH_UA_PLATFORM, + obscura_net::STEALTH_UA_PLATFORM_VERSION, + ); } else { if let Ok(ua) = self.http_client.user_agent.try_read() { rt.set_user_agent(&ua); @@ -317,6 +319,10 @@ impl Page { rt.set_cookie_jar(self.context.cookie_jar.clone()); rt.set_http_client(self.http_client.clone()); rt.set_blocked_urls(self.blocked_url_patterns.clone()); + #[cfg(feature = "stealth")] + if let Some(ref stealth) = self.stealth_client { + rt.set_stealth_client(stealth.clone()); + } if let Some(tx) = &self.intercept_tx { rt.set_intercept_tx(tx.clone()); diff --git a/crates/obscura-js/Cargo.toml b/crates/obscura-js/Cargo.toml index c86b92f..29625cf 100644 --- a/crates/obscura-js/Cargo.toml +++ b/crates/obscura-js/Cargo.toml @@ -3,6 +3,13 @@ name = "obscura-js" version.workspace = true edition.workspace = true +[features] +default = [] +# Builds obscura-net with its stealth (wreq) HTTP client and lets scripted +# fetch()/XHR be routed through it so JS-level requests carry the same Chrome +# TLS fingerprint and client hints as stealth navigation. +stealth = ["obscura-net/stealth"] + [build-dependencies] deno_core = "0.350" diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js index f364f54..acc2912 100644 --- a/crates/obscura-js/js/bootstrap.js +++ b/crates/obscura-js/js/bootstrap.js @@ -154,7 +154,9 @@ function _fpNoise(x, y, channel) { var _fpCache = null; function _getFp() { if (_fpCache) return _fpCache; - const isMac = (globalThis.__obscura_ua_platform || 'Windows') === 'macOS'; + const _uaPlat = globalThis.__obscura_ua_platform || 'Windows'; + const isMac = _uaPlat === 'macOS'; + const isLinux = _uaPlat === 'Linux'; const gpuPool = isMac ? [ 'ANGLE (Apple, ANGLE Metal Renderer: Apple M1, Unspecified Version)', 'ANGLE (Apple, ANGLE Metal Renderer: Apple M1 Pro, Unspecified Version)', @@ -162,6 +164,14 @@ function _getFp() { 'ANGLE (Apple, ANGLE Metal Renderer: Apple M2 Pro, Unspecified Version)', 'ANGLE (Apple, ANGLE Metal Renderer: Apple M3, Unspecified Version)', 'ANGLE (Intel Inc., ANGLE Metal Renderer: Intel(R) Iris(TM) Plus Graphics, Unspecified Version)', + ] : isLinux ? [ + 'ANGLE (Intel, Mesa Intel(R) UHD Graphics 630 (CFL GT2), OpenGL 4.6)', + 'ANGLE (Intel, Mesa Intel(R) Iris(R) Xe Graphics (TGL GT2), OpenGL 4.6)', + 'ANGLE (Intel, Mesa Intel(R) UHD Graphics 770 (RPL-S), OpenGL 4.6)', + 'ANGLE (AMD, AMD Radeon RX 580 (polaris10, LLVM 15.0.7, DRM 3.54, LLVM 15.0.7), OpenGL 4.6)', + 'ANGLE (AMD, AMD Radeon RX 6700 XT (navi22, LLVM 16.0.6, DRM 3.54, LLVM 16.0.6), OpenGL 4.6)', + 'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 OpenGL 4.6)', + 'ANGLE (NVIDIA, NVIDIA GeForce RTX 4070 OpenGL 4.6)', ] : [ 'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)', 'ANGLE (NVIDIA, NVIDIA GeForce GTX 1660 SUPER Direct3D11 vs_5_0 ps_5_0, D3D11)', @@ -180,6 +190,10 @@ function _getFp() { 'Google Inc. (Apple)','Google Inc. (Apple)','Google Inc. (Apple)', 'Google Inc. (Apple)','Google Inc. (Apple)', 'Google Inc. (Intel Inc.)', + ] : isLinux ? [ + 'Google Inc. (Intel)','Google Inc. (Intel)','Google Inc. (Intel)', + 'Google Inc. (AMD)','Google Inc. (AMD)', + 'Google Inc. (NVIDIA)','Google Inc. (NVIDIA)', ] : [ 'Google Inc. (NVIDIA)','Google Inc. (NVIDIA)','Google Inc. (NVIDIA)', 'Google Inc. (Intel)','Google Inc. (Intel)', @@ -2577,7 +2591,7 @@ function __currentUrl() { } globalThis.location = { get href() { return __currentUrl(); }, - set href(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); }, + set href(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); }, get origin() { try { return new URL(this.href).origin; } catch { return ""; } }, get protocol() { try { return new URL(this.href).protocol; } catch { return ""; } }, get host() { try { return new URL(this.href).host; } catch { return ""; } }, @@ -2587,14 +2601,14 @@ globalThis.location = { get hash() { try { return new URL(this.href).hash; } catch { return ""; } }, get port() { try { return new URL(this.href).port; } catch { return ""; } }, toString() { return this.href; }, - assign(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); }, - reload() {}, - replace(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); }, + assign(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); }, + reload() { var r = _resolveUrl(this.href); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); }, + replace(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); }, }; const _locationObj = globalThis.location; Object.defineProperty(globalThis, 'location', { get() { return _locationObj; }, - set(url) { Deno.core.ops.op_navigate(_resolveUrl(String(url)), 'GET', ''); }, + set(url) { var r = _resolveUrl(String(url)); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); }, configurable: false, enumerable: true, }); @@ -2752,8 +2766,35 @@ globalThis.NetworkInformation = NetworkInformation; globalThis.ContentIndex = class ContentIndex {}; +function _chromeMajor() { + var m = (globalThis.__obscura_ua || '').match(/Chrome\/(\d+)/); + return m ? (m[1] | 0) : 145; +} +// Chromium derives the sec-ch-ua GREASE brand, version, and brand order +// deterministically from the Chrome major version +// (components/embedder_support/user_agent_utils.cc). Replicating it keeps +// sec-ch-ua and userAgentData exact for every profile version rather than +// hardcoding one static token. +var _GREASE_CHARS = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_']; +var _GREASE_VER = ['8', '99', '24']; +var _BRAND_PERMS = [[0,1,2],[0,2,1],[1,0,2],[1,2,0],[2,0,1],[2,1,0]]; +function _uaBrands() { + var seed = _chromeMajor(); + var grease = { + brand: 'Not' + _GREASE_CHARS[seed % 11] + 'A' + _GREASE_CHARS[(seed + 1) % 11] + 'Brand', + version: _GREASE_VER[seed % 3], + }; + var ordered = [ + grease, + {brand: 'Chromium', version: String(seed)}, + {brand: 'Google Chrome', version: String(seed)}, + ]; + var p = _BRAND_PERMS[seed % 6]; + return [ordered[p[0]], ordered[p[1]], ordered[p[2]]]; +} + globalThis.navigator = { - get userAgent() { return globalThis.__obscura_ua || "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; }, + get userAgent() { return globalThis.__obscura_ua || "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; }, get appVersion() { return this.userAgent.replace('Mozilla/', ''); }, language: "en-US", languages: ["en-US","en"], get platform() { return globalThis.__obscura_platform || "Win32"; }, onLine: true, cookieEnabled: true, hardwareConcurrency: 8, @@ -2779,24 +2820,22 @@ globalThis.navigator = { ]); }, userAgentData: { - brands: [ - {brand: "Google Chrome", version: "145"}, - {brand: "Chromium", version: "145"}, - {brand: "Not;A=Brand", version: "24"}, - ], mobile: false, + get brands() { return _uaBrands(); }, get platform() { return globalThis.__obscura_ua_platform || "Windows"; }, getHighEntropyValues(hints) { + var brands = _uaBrands(); return Promise.resolve({ architecture: "x86", bitness: "64", - brands: [{brand:"Google Chrome",version:"145"},{brand:"Chromium",version:"145"},{brand:"Not;A=Brand",version:"24"}], - fullVersionList: [{brand:"Google Chrome",version:"145.0.0.0"},{brand:"Chromium",version:"145.0.0.0"},{brand:"Not;A=Brand",version:"24.0.0.0"}], + brands: brands, + fullVersionList: brands.map(function(b) { return {brand: b.brand, version: b.version + ".0.0.0"}; }), mobile: false, model: "", platform: globalThis.__obscura_ua_platform || "Windows", platformVersion: globalThis.__obscura_ua_platform_version || "15.0.0", - uaFullVersion: "145.0.0.0", + uaFullVersion: _chromeMajor() + ".0.0.0", + wow64: false, }); }, toJSON() { return {brands:this.brands,mobile:this.mobile,platform:this.platform}; }, @@ -2817,8 +2856,12 @@ globalThis.navigator = { }, clipboard: { writeText(){return Promise.resolve();}, readText(){return Promise.resolve("");} }, permissions: { query(params){ - if (params?.name === 'notifications') return Promise.resolve({state:"prompt",onchange:null}); - return Promise.resolve({state:"granted"}); + var n = params && params.name; + // Chrome defaults privacy-sensitive permissions to "prompt", not "granted"; + // returning "granted" for camera/microphone is a bot tell. + if (n === 'notifications') return Promise.resolve({state: (globalThis.Notification && Notification.permission === 'granted') ? 'granted' : 'prompt', onchange: null}); + if (n === 'geolocation' || n === 'camera' || n === 'microphone' || n === 'midi') return Promise.resolve({state: 'prompt', onchange: null}); + return Promise.resolve({state: 'granted', onchange: null}); } }, getBattery() { return Promise.resolve({ charging: _fp('batteryCharging'), chargingTime: _fp('batteryCharging') ? 0 : Infinity, dischargingTime: _fp('batteryCharging') ? Infinity : Math.floor(3600 + _fpRand(250) * 7200), level: _fp('batteryLevel'), addEventListener(){} }); }, getGamepads() { return []; }, @@ -3003,6 +3046,51 @@ function _installWasmStreamingFallback() { } _installWasmStreamingFallback(); +// Serialize a FormData into a multipart/form-data body the way a browser does +// when it is passed as fetch()/XHR body. The previous shim did String(body), +// so a FormData became the literal "[object Object]" and the multipart payload +// (with its boundary) was lost; servers replied "Invalid boundary for +// multipart/form-data" (e.g. the AWS WAF challenge /mp_verify POST). +function _formDataToMultipart(fd) { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + let bnd = '----WebKitFormBoundary'; + for (let i = 0; i < 16; i++) bnd += chars[Math.floor(Math.random() * chars.length)]; + let out = ''; + const entries = fd._d || []; + for (let i = 0; i < entries.length; i++) { + const k = entries[i][0], v = entries[i][1]; + out += '--' + bnd + '\r\n'; + if (v != null && typeof v === 'object' && v._bytes != null) { + out += 'Content-Disposition: form-data; name="' + k + '"; filename="' + (v.name || 'blob') + '"\r\n'; + out += 'Content-Type: ' + (v.type || 'application/octet-stream') + '\r\n\r\n'; + try { out += new TextDecoder().decode(v._bytes); } catch (e) {} + out += '\r\n'; + } else { + out += 'Content-Disposition: form-data; name="' + k + '"\r\n\r\n' + String(v) + '\r\n'; + } + } + out += '--' + bnd + '--\r\n'; + return { boundary: bnd, body: out }; +} + +// Coerce a fetch()/XHR body into the string op_fetch_url expects, attaching a +// Content-Type header for body types that need one (FormData, URLSearchParams). +function _serializeBody(initBody, headers) { + if (initBody == null || initBody === '') return ''; + if (initBody instanceof FormData) { + const mp = _formDataToMultipart(initBody); + headers['Content-Type'] = 'multipart/form-data; boundary=' + mp.boundary; + return mp.body; + } + if (initBody instanceof URLSearchParams) { + if (!Object.keys(headers).some(k => k.toLowerCase() === 'content-type')) { + headers['Content-Type'] = 'application/x-www-form-urlencoded;charset=UTF-8'; + } + return initBody.toString(); + } + return typeof initBody === 'string' ? initBody : String(initBody); +} + globalThis.fetch = async (input, init = {}) => { let url = typeof input === "string" ? input @@ -3016,8 +3104,9 @@ globalThis.fetch = async (input, init = {}) => { } catch(e) { /* keep as-is if URL resolution fails */ } } const method = init.method || (input instanceof Request ? input.method : "GET"); - const hdrs = JSON.stringify(init.headers instanceof Headers ? Object.fromEntries(init.headers.entries()) : init.headers || {}); - const body = init.body ? String(init.body) : ""; + let _h = init.headers instanceof Headers ? Object.fromEntries(init.headers.entries()) : (init.headers || {}); + const body = _serializeBody(init.body, _h); + const hdrs = JSON.stringify(_h); const fetchMode = init.mode || (input instanceof Request ? input.mode : "cors"); const pageOrigin = (function() { try { const u = new URL(_domParse("document_url") || "about:blank"); return u.origin; } catch(e) { return ""; } })(); const raw = await Deno.core.ops.op_fetch_url(url, method, hdrs, body, pageOrigin, fetchMode); @@ -6328,11 +6417,25 @@ globalThis.Worker = class Worker { }; globalThis.__blobStore = globalThis.__blobStore || {}; -const _origCreateObjectURL = URL.createObjectURL; URL.createObjectURL = function(blob) { - if (blob && typeof blob.text === 'function') { + if (blob) { const id = 'blob:obscura/' + Math.random().toString(36).substring(2); - blob.text().then(text => { globalThis.__blobStore[id] = text; }); + // Store synchronously so a Worker built from the blob URL in the same + // tick sees its source. Blob-URL Worker construction is synchronous in + // real browsers; the previous async blob.text().then() store raced the + // Worker constructor, so new Worker(blobURL) fell through to fetch() and + // failed (net::ERR_FAILED), which broke AWS WAF's proof-of-work worker. + // The obscura Blob materializes _bytes in its constructor; fall back to + // the async text() store only for foreign Blob shims without _bytes. + if (blob._bytes) { + let text = ''; + try { text = new TextDecoder().decode(blob._bytes); } catch (e) {} + globalThis.__blobStore[id] = text; + } else if (typeof blob.text === 'function') { + blob.text().then(text => { globalThis.__blobStore[id] = text; }); + } else { + globalThis.__blobStore[id] = ''; + } return id; } return 'blob:obscura/fallback'; @@ -6421,7 +6524,8 @@ if (!globalThis.crypto.subtle) { globalThis.crypto.subtle = { async digest(algorithm, data) { const name = (typeof algorithm === 'string' ? algorithm : algorithm?.name || '').toUpperCase().replace('_', '-'); - if (name !== 'SHA-1' && name !== 'SHA-256' && name !== 'SHA-384' && name !== 'SHA-512') { + if (name !== 'SHA-1' && name !== 'SHA-256' && name !== 'SHA-384' && name !== 'SHA-512' && + name !== 'SHA-512/224' && name !== 'SHA-512/256') { throw new DOMException('Unrecognized algorithm name', 'NotSupportedError'); } let bytes; @@ -6840,6 +6944,10 @@ if (typeof ShadowRoot !== 'undefined' && !ShadowRoot.prototype.elementFromPoint) globalThis.__obscura_init = function() { _fpSeed = Date.now() ^ (Math.random() * 0xFFFFFFFF >>> 0); _fpCache = null; + // A real navigation just completed (this runs after set_url), so drop any + // URL a location setter previewed synchronously and let document_url drive + // location.href again, including any redirect target. + globalThis.__virtualUrl = null; _installWasmStreamingFallback(); globalThis.document = new Document(+_dom("document_node_id")); @@ -6868,29 +6976,9 @@ globalThis.__obscura_init = function() { }; globalThis.Notification.permission = "default"; - if (globalThis.__obscura_stealth) { - var _stealthBrands = [ - {brand: "Not:A-Brand", version: "99"}, - {brand: "Google Chrome", version: "145"}, - {brand: "Chromium", version: "145"}, - ]; - globalThis.navigator.userAgentData.brands = _stealthBrands; - globalThis.navigator.userAgentData.getHighEntropyValues = function(hints) { - return Promise.resolve({ - architecture: "x86", bitness: "64", - brands: _stealthBrands, - fullVersionList: [ - {brand:"Not:A-Brand",version:"99.0.0.0"}, - {brand:"Google Chrome",version:"145.0.0.0"}, - {brand:"Chromium",version:"145.0.0.0"}, - ], - mobile: false, model: "", - platform: "Windows", - platformVersion: "10.0.0", - uaFullVersion: "145.0.0.0", - }); - }; - } + // userAgentData brands and getHighEntropyValues now derive the Chrome + // version from navigator.userAgent and read the platform from the page + // globals, so every stealth surface agrees without a per-mode override. // Hide internals (_*, obscura, Obscura). The set of keys is static at // snapshot-build time, so we precompute it ONCE below (after this diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs index 6ae576b..c33958f 100644 --- a/crates/obscura-js/src/ops.rs +++ b/crates/obscura-js/src/ops.rs @@ -9,6 +9,8 @@ use deno_core::Extension; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use obscura_dom::{DomTree, NodeData, NodeId}; use obscura_net::{CookieJar, ObscuraHttpClient}; +#[cfg(feature = "stealth")] +use obscura_net::StealthHttpClient; use tokio::sync::Mutex; pub type InterceptCallback = Arc Option<(u16, String, String)> + Send + Sync>>>>; @@ -55,6 +57,11 @@ pub struct ObscuraState { pub blocked_urls: Vec, pub cookie_jar: Option>, pub http_client: Option>, + /// When set (stealth mode), scripted fetch()/XHR is routed through the wreq + /// client so the request carries the Chrome TLS fingerprint and client + /// hints instead of the rustls ClientHello op_fetch_url would otherwise send. + #[cfg(feature = "stealth")] + pub stealth_client: Option>, pub pending_navigation: Option<(String, String, String)>, pub intercept_tx: Option>, pub intercept_counter: u64, @@ -82,6 +89,8 @@ impl ObscuraState { blocked_urls: Vec::new(), cookie_jar: None, http_client: None, + #[cfg(feature = "stealth")] + stealth_client: None, pending_navigation: None, intercept_tx: None, intercept_counter: 0, @@ -707,6 +716,34 @@ async fn op_fetch_url( let custom_headers: std::collections::HashMap = serde_json::from_str(&headers_json).unwrap_or_default(); + // Stealth mode: route the scripted request through the wreq client so its + // TLS fingerprint and Chrome client hints match the main navigation. The + // rustls ClientHello plus missing client hints that op_fetch_url's reqwest + // path sends otherwise read as a non-browser script to bot managers (the + // AWS WAF challenge verify call, Akamai sensors, etc.). + #[cfg(feature = "stealth")] + { + let stealth = { + let st = state.borrow(); + let gs = st.borrow::().clone(); + let client = gs.borrow().stealth_client.clone(); + client + }; + if let Some(stealth) = stealth { + return stealth_fetch_all( + stealth, + url.clone(), + req_method.as_str().to_string(), + custom_headers.clone(), + body.clone(), + page_origin.clone(), + is_cross_origin, + mode.clone(), + ) + .await; + } + } + let needs_preflight = is_cross_origin && mode == "cors" && (req_method != reqwest::Method::GET @@ -946,6 +983,120 @@ async fn op_fetch_url( .to_string()) } +/// Stealth-mode scripted fetch()/XHR: mirrors op_fetch_url's redirect, SSRF, +/// and CORS semantics but sends every hop through the wreq stealth client so +/// the request carries the Chrome TLS fingerprint and client hints. Cookie +/// handling lives inside StealthHttpClient::send_single, which shares the +/// context jar. Response bodies are not mirrored into the CDP +/// Network.getResponseBody buffer here; that is a follow-up for stealth fetches. +#[cfg(feature = "stealth")] +async fn stealth_fetch_all( + stealth: Arc, + url: String, + method: String, + custom_headers: HashMap, + body: String, + page_origin: String, + is_cross_origin: bool, + mode: String, +) -> Result { + let mut current_url = url.clone(); + let mut current_method = method; + let mut current_body = body; + let mut redirects_followed: usize = 0; + + let (status, resp_headers, resp_bytes): (u16, HashMap, Vec) = loop { + let parsed_current = match url::Url::parse(¤t_url) { + Ok(u) => u, + Err(_) => { + return Ok(serde_json::json!({ + "status": 0, "body": "", "url": current_url, "headers": {}, + }) + .to_string()); + } + }; + + let mut req_headers: HashMap = HashMap::new(); + if is_cross_origin { + req_headers.insert("origin".to_string(), page_origin.clone()); + } + for (k, v) in &custom_headers { + req_headers.insert(k.to_lowercase(), v.clone()); + } + + let r = stealth + .send_single(¤t_method, &parsed_current, &req_headers, ¤t_body) + .await + .map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?; + + if !(300..400).contains(&r.status) { + break (r.status, r.headers, r.body); + } + let Some(location) = r.headers.get("location").cloned() else { + break (r.status, r.headers, r.body); + }; + let next_url = match parsed_current.join(&location) { + Ok(u) => u, + Err(_) => break (r.status, r.headers, r.body), + }; + // Re-validate every redirect target against the SSRF policy, matching + // op_fetch_url (GHSA-8v6v-g4rh-jmcm). + if let Err(reason) = validate_fetch_url(&next_url) { + return Ok(serde_json::json!({ + "status": 0, "body": "", "url": next_url.to_string(), "headers": {}, + "blocked": true, + "error": format!("Redirect to forbidden URL blocked: {}", reason), + }) + .to_string()); + } + redirects_followed += 1; + if redirects_followed > FETCH_REDIRECT_LIMIT { + return Ok(serde_json::json!({ + "status": 0, "body": "", "url": next_url.to_string(), "headers": {}, + "blocked": true, + "error": format!("Too many redirects (>{})", FETCH_REDIRECT_LIMIT), + }) + .to_string()); + } + // Browser semantics: 301/302/303 downgrade to GET with no body. + if r.status == 301 || r.status == 302 || r.status == 303 { + current_method = "GET".to_string(); + current_body.clear(); + } + current_url = next_url.to_string(); + }; + + if is_cross_origin && mode == "cors" { + let allowed = resp_headers + .get("access-control-allow-origin") + .map(|s| s.as_str()) + .unwrap_or(""); + if allowed != "*" && allowed != page_origin { + return Ok(serde_json::json!({ + "status": 0, "body": "", "url": url, "headers": {}, + "corsBlocked": true, + "corsError": format!( + "CORS error: Origin '{}' not in Access-Control-Allow-Origin '{}'", + page_origin, allowed + ), + }) + .to_string()); + } + } + + let resp_body = String::from_utf8_lossy(&resp_bytes).to_string(); + let resp_body_base64 = BASE64.encode(&resp_bytes); + + Ok(serde_json::json!({ + "status": status, + "body": resp_body, + "bodyBase64": resp_body_base64, + "url": url, + "headers": resp_headers, + }) + .to_string()) +} + fn glob_match(pattern: &str, url: &str) -> bool { if pattern == "*" { return true; @@ -1107,8 +1258,9 @@ fn op_binding_called(state: &OpState, #[string] name: &str, #[string] payload: & } /// Real WebCrypto `crypto.subtle.digest`. `algorithm` is the SubtleCrypto -/// algorithm name (`SHA-1` / `SHA-256` / `SHA-384` / `SHA-512`). The JS shim -/// validates the name before calling this op; any other value is unreachable. +/// algorithm name (`SHA-1` / `SHA-256` / `SHA-384` / `SHA-512`, plus the +/// FIPS 180-4 truncated variants `SHA-512/224` and `SHA-512/256`). The JS +/// shim validates the name; any other value is unreachable. /// Returns the raw digest bytes so the JS shim can hand them back as an ArrayBuffer. #[op2] #[buffer] @@ -1120,6 +1272,8 @@ fn op_subtle_digest(#[string] algorithm: &str, #[buffer] data: &[u8]) -> Vec "SHA-256" => sha2::Sha256::digest(data).to_vec(), "SHA-384" => sha2::Sha384::digest(data).to_vec(), "SHA-512" => sha2::Sha512::digest(data).to_vec(), + "SHA-512/224" => sha2::Sha512_224::digest(data).to_vec(), + "SHA-512/256" => sha2::Sha512_256::digest(data).to_vec(), _ => vec![], } } diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs index 3bdd722..ca3304d 100644 --- a/crates/obscura-js/src/runtime.rs +++ b/crates/obscura-js/src/runtime.rs @@ -153,6 +153,13 @@ impl ObscuraJsRuntime { self.state.borrow_mut().http_client = Some(client); } + /// Install the stealth (wreq) HTTP client so scripted fetch()/XHR is routed + /// through it in stealth mode (see op_fetch_url / stealth_fetch_all). + #[cfg(feature = "stealth")] + pub fn set_stealth_client(&self, client: std::sync::Arc) { + self.state.borrow_mut().stealth_client = Some(client); + } + pub fn set_dom(&self, dom: DomTree) { self.state.borrow_mut().dom = Some(dom); } diff --git a/crates/obscura-net/src/client.rs b/crates/obscura-net/src/client.rs index 5f7a561..83d0e5f 100644 --- a/crates/obscura-net/src/client.rs +++ b/crates/obscura-net/src/client.rs @@ -273,6 +273,46 @@ pub struct ObscuraHttpClient { pub allow_private_network: bool, } +/// Derive the sec-ch-ua and sec-ch-ua-platform client-hint header values from a +/// User-Agent string, using Chromium's per-major-version GREASE algorithm so +/// the non-stealth HTTP path agrees with navigator.userAgentData instead of +/// shipping a fixed Linux/Chrome-145 hint that contradicts a Windows profile. +fn chrome_client_hints(ua: &str) -> (String, String) { + let major: usize = ua + .split("Chrome/") + .nth(1) + .and_then(|s| s.split('.').next()) + .and_then(|s| s.parse::().ok()) + .unwrap_or(145); + const GREASE_CHARS: [char; 11] = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_']; + const GREASE_VER: [&str; 3] = ["8", "99", "24"]; + const PERMS: [[usize; 3]; 6] = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]]; + let grease_brand = format!( + "Not{}A{}Brand", + GREASE_CHARS[major % 11], + GREASE_CHARS[(major + 1) % 11] + ); + let brands = [ + (grease_brand, GREASE_VER[major % 3].to_string()), + ("Chromium".to_string(), major.to_string()), + ("Google Chrome".to_string(), major.to_string()), + ]; + let p = PERMS[major % 6]; + let sec_ch_ua = p + .iter() + .map(|&i| format!("\"{}\";v=\"{}\"", brands[i].0, brands[i].1)) + .collect::>() + .join(", "); + let platform = if ua.contains("Windows NT") { + "\"Windows\"" + } else if ua.contains("Macintosh") { + "\"macOS\"" + } else { + "\"Linux\"" + }; + (sec_ch_ua, platform.to_string()) +} + impl ObscuraHttpClient { pub fn new() -> Self { Self::with_cookie_jar(Arc::new(CookieJar::new())) @@ -407,50 +447,38 @@ impl ObscuraHttpClient { } let ua = self.user_agent.read().await.clone(); + let (sec_ch_ua, sec_ch_ua_platform) = chrome_client_hints(&ua); let mut headers = HeaderMap::new(); + // Chrome's top-level navigation header order. (reqwest appends + // accept-encoding/host after these, so accept-encoding lands after + // accept-language rather than before it; the rest matches Chrome.) + headers.insert( + HeaderName::from_static("sec-ch-ua"), + HeaderValue::from_str(&sec_ch_ua) + .unwrap_or_else(|_| HeaderValue::from_static("\"Not:A-Brand\";v=\"99\", \"Google Chrome\";v=\"145\", \"Chromium\";v=\"145\"")), + ); + headers.insert(HeaderName::from_static("sec-ch-ua-mobile"), HeaderValue::from_static("?0")); + headers.insert( + HeaderName::from_static("sec-ch-ua-platform"), + HeaderValue::from_str(&sec_ch_ua_platform) + .unwrap_or_else(|_| HeaderValue::from_static("\"Windows\"")), + ); + headers.insert(HeaderName::from_static("upgrade-insecure-requests"), HeaderValue::from_static("1")); headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap_or_else(|_| { - HeaderValue::from_static("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36") + HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36") })); headers.insert( reqwest::header::ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"), ); + headers.insert(HeaderName::from_static("sec-fetch-site"), HeaderValue::from_static("none")); + headers.insert(HeaderName::from_static("sec-fetch-mode"), HeaderValue::from_static("navigate")); + headers.insert(HeaderName::from_static("sec-fetch-user"), HeaderValue::from_static("?1")); + headers.insert(HeaderName::from_static("sec-fetch-dest"), HeaderValue::from_static("document")); headers.insert( reqwest::header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"), ); - headers.insert( - HeaderName::from_static("sec-ch-ua"), - HeaderValue::from_static("\"Chromium\";v=\"145\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"145\""), - ); - headers.insert( - HeaderName::from_static("sec-ch-ua-mobile"), - HeaderValue::from_static("?0"), - ); - headers.insert( - HeaderName::from_static("sec-ch-ua-platform"), - HeaderValue::from_static("\"Linux\""), - ); - headers.insert( - HeaderName::from_static("sec-fetch-dest"), - HeaderValue::from_static("document"), - ); - headers.insert( - HeaderName::from_static("sec-fetch-mode"), - HeaderValue::from_static("navigate"), - ); - headers.insert( - HeaderName::from_static("sec-fetch-site"), - HeaderValue::from_static("none"), - ); - headers.insert( - HeaderName::from_static("sec-fetch-user"), - HeaderValue::from_static("?1"), - ); - headers.insert( - HeaderName::from_static("upgrade-insecure-requests"), - HeaderValue::from_static("1"), - ); let cookie_header = self.cookie_jar.get_cookie_header(¤t_url); tracing::debug!( diff --git a/crates/obscura-net/src/lib.rs b/crates/obscura-net/src/lib.rs index e373575..f9840ad 100644 --- a/crates/obscura-net/src/lib.rs +++ b/crates/obscura-net/src/lib.rs @@ -19,4 +19,7 @@ pub use encoding::{ pub use robots::RobotsCache; pub use blocklist::is_blocked as is_tracker_blocked; #[cfg(feature = "stealth")] -pub use wreq_client::{StealthHttpClient, STEALTH_USER_AGENT}; +pub use wreq_client::{ + StealthHttpClient, STEALTH_NAVIGATOR_PLATFORM, STEALTH_UA_PLATFORM, + STEALTH_UA_PLATFORM_VERSION, STEALTH_USER_AGENT, +}; diff --git a/crates/obscura-net/src/wreq_client.rs b/crates/obscura-net/src/wreq_client.rs index e0405d4..bf56a9a 100644 --- a/crates/obscura-net/src/wreq_client.rs +++ b/crates/obscura-net/src/wreq_client.rs @@ -19,7 +19,18 @@ use crate::client::{Response, ObscuraNetError}; #[cfg(feature = "stealth")] pub const STEALTH_USER_AGENT: &str = - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; + +// The wreq emulation (Profile::Chrome145, Platform::Windows) sends this exact +// UA and sec-ch-ua-platform "Windows" on the wire. navigator has to report the +// same identity, otherwise the TLS/HTTP layer and the JS layer disagree and a +// site cross-checks the mismatch as a bot signal. +#[cfg(feature = "stealth")] +pub const STEALTH_NAVIGATOR_PLATFORM: &str = "Win32"; +#[cfg(feature = "stealth")] +pub const STEALTH_UA_PLATFORM: &str = "Windows"; +#[cfg(feature = "stealth")] +pub const STEALTH_UA_PLATFORM_VERSION: &str = "15.0.0"; #[cfg(feature = "stealth")] pub struct StealthHttpClient { @@ -38,7 +49,7 @@ impl StealthHttpClient { pub fn with_proxy(cookie_jar: Arc, proxy_url: Option<&str>) -> Self { let emulation_opts = wreq_util::Emulation::builder() .profile(wreq_util::Profile::Chrome145) - .platform(wreq_util::Platform::Linux) + .platform(wreq_util::Platform::Windows) .build(); let mut builder = wreq::Client::builder() @@ -143,6 +154,85 @@ impl StealthHttpClient { Err(ObscuraNetError::TooManyRedirects(url.to_string())) } + /// One request with no redirect following, for scripted fetch()/XHR. Reads + /// the cookie jar for the Cookie header and stores Set-Cookie back into it, + /// so the caller only owns redirect hops and SSRF re-validation. Used in + /// stealth mode so JS-level requests carry the same Chrome TLS fingerprint + /// and client hints as the main navigation instead of the rustls ClientHello + /// that op_fetch_url would otherwise send (which bot managers read as a + /// non-browser script and reject, e.g. the AWS WAF challenge verify call). + pub async fn send_single( + &self, + method: &str, + url: &Url, + headers: &HashMap, + body: &str, + ) -> Result { + if let Some(host) = url.host_str() { + if crate::blocklist::is_blocked(host) { + tracing::debug!("Blocked tracker: {}", url); + return Ok(Response { + status: 0, + url: url.clone(), + headers: HashMap::new(), + body: Vec::new(), + redirected_from: Vec::new(), + }); + } + } + + let req_method = method + .parse::() + .map_err(|e| ObscuraNetError::Network(format!("invalid method '{}': {}", method, e)))?; + let mut req = self.client.request(req_method, url.as_str()); + + let cookie_header = self.cookie_jar.get_cookie_header(url); + if !cookie_header.is_empty() { + req = req.header("cookie", &cookie_header); + } + for (k, v) in self.extra_headers.read().await.iter() { + req = req.header(k.as_str(), v.as_str()); + } + for (k, v) in headers.iter() { + req = req.header(k.as_str(), v.as_str()); + } + if !body.is_empty() { + req = req.body(body.to_string()); + } + + self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let resp = req.send().await.map_err(|e| { + self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + ObscuraNetError::Network(format!("{}: {}", url, e)) + })?; + self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + + let status = resp.status(); + for val in resp.headers().get_all("set-cookie") { + if let Ok(s) = val.to_str() { + self.cookie_jar.set_cookie(s, url); + } + } + let response_headers: HashMap = resp + .headers() + .iter() + .map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string())) + .collect(); + let resp_body = resp + .bytes() + .await + .map_err(|e| ObscuraNetError::Network(format!("Failed to read body: {}", e)))? + .to_vec(); + + Ok(Response { + url: url.clone(), + status: status.as_u16(), + headers: response_headers, + body: resp_body, + redirected_from: Vec::new(), + }) + } + pub async fn set_extra_headers(&self, headers: HashMap) { *self.extra_headers.write().await = headers; }