diff --git a/crates/obscura-browser/Cargo.toml b/crates/obscura-browser/Cargo.toml index 0179e94..1ed64cd 100644 --- a/crates/obscura-browser/Cargo.toml +++ b/crates/obscura-browser/Cargo.toml @@ -5,7 +5,7 @@ edition.workspace = true [features] default = [] -stealth = ["obscura-net/stealth"] +stealth = ["obscura-net/stealth", "obscura-js/stealth"] [dependencies] obscura-dom = { workspace = true } diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs index 44fb44f..95ad7ae 100644 --- a/crates/obscura-browser/src/page.rs +++ b/crates/obscura-browser/src/page.rs @@ -319,6 +319,10 @@ impl Page { rt.set_cookie_jar(self.context.cookie_jar.clone()); rt.set_http_client(self.http_client.clone()); + #[cfg(feature = "stealth")] + if let Some(ref stealth) = self.stealth_client { + rt.set_stealth_client(stealth.clone()); + } if let Some(tx) = &self.intercept_tx { rt.set_intercept_tx(tx.clone()); diff --git a/crates/obscura-js/Cargo.toml b/crates/obscura-js/Cargo.toml index c86b92f..29625cf 100644 --- a/crates/obscura-js/Cargo.toml +++ b/crates/obscura-js/Cargo.toml @@ -3,6 +3,13 @@ name = "obscura-js" version.workspace = true edition.workspace = true +[features] +default = [] +# Builds obscura-net with its stealth (wreq) HTTP client and lets scripted +# fetch()/XHR be routed through it so JS-level requests carry the same Chrome +# TLS fingerprint and client hints as stealth navigation. +stealth = ["obscura-net/stealth"] + [build-dependencies] deno_core = "0.350" diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs index 99cccd3..752f47e 100644 --- a/crates/obscura-js/src/ops.rs +++ b/crates/obscura-js/src/ops.rs @@ -9,6 +9,8 @@ use deno_core::Extension; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use obscura_dom::{DomTree, NodeData, NodeId}; use obscura_net::{CookieJar, ObscuraHttpClient}; +#[cfg(feature = "stealth")] +use obscura_net::StealthHttpClient; use tokio::sync::Mutex; pub type InterceptCallback = Arc Option<(u16, String, String)> + Send + Sync>>>>; @@ -55,6 +57,11 @@ pub struct ObscuraState { pub blocked_urls: Vec, pub cookie_jar: Option>, pub http_client: Option>, + /// When set (stealth mode), scripted fetch()/XHR is routed through the wreq + /// client so the request carries the Chrome TLS fingerprint and client + /// hints instead of the rustls ClientHello op_fetch_url would otherwise send. + #[cfg(feature = "stealth")] + pub stealth_client: Option>, pub pending_navigation: Option<(String, String, String)>, pub intercept_tx: Option>, pub intercept_counter: u64, @@ -82,6 +89,8 @@ impl ObscuraState { blocked_urls: Vec::new(), cookie_jar: None, http_client: None, + #[cfg(feature = "stealth")] + stealth_client: None, pending_navigation: None, intercept_tx: None, intercept_counter: 0, @@ -707,6 +716,34 @@ async fn op_fetch_url( let custom_headers: std::collections::HashMap = serde_json::from_str(&headers_json).unwrap_or_default(); + // Stealth mode: route the scripted request through the wreq client so its + // TLS fingerprint and Chrome client hints match the main navigation. The + // rustls ClientHello plus missing client hints that op_fetch_url's reqwest + // path sends otherwise read as a non-browser script to bot managers (the + // AWS WAF challenge verify call, Akamai sensors, etc.). + #[cfg(feature = "stealth")] + { + let stealth = { + let st = state.borrow(); + let gs = st.borrow::().clone(); + let client = gs.borrow().stealth_client.clone(); + client + }; + if let Some(stealth) = stealth { + return stealth_fetch_all( + stealth, + url.clone(), + req_method.as_str().to_string(), + custom_headers.clone(), + body.clone(), + page_origin.clone(), + is_cross_origin, + mode.clone(), + ) + .await; + } + } + let needs_preflight = is_cross_origin && mode == "cors" && (req_method != reqwest::Method::GET @@ -946,6 +983,120 @@ async fn op_fetch_url( .to_string()) } +/// Stealth-mode scripted fetch()/XHR: mirrors op_fetch_url's redirect, SSRF, +/// and CORS semantics but sends every hop through the wreq stealth client so +/// the request carries the Chrome TLS fingerprint and client hints. Cookie +/// handling lives inside StealthHttpClient::send_single, which shares the +/// context jar. Response bodies are not mirrored into the CDP +/// Network.getResponseBody buffer here; that is a follow-up for stealth fetches. +#[cfg(feature = "stealth")] +async fn stealth_fetch_all( + stealth: Arc, + url: String, + method: String, + custom_headers: HashMap, + body: String, + page_origin: String, + is_cross_origin: bool, + mode: String, +) -> Result { + let mut current_url = url.clone(); + let mut current_method = method; + let mut current_body = body; + let mut redirects_followed: usize = 0; + + let (status, resp_headers, resp_bytes): (u16, HashMap, Vec) = loop { + let parsed_current = match url::Url::parse(¤t_url) { + Ok(u) => u, + Err(_) => { + return Ok(serde_json::json!({ + "status": 0, "body": "", "url": current_url, "headers": {}, + }) + .to_string()); + } + }; + + let mut req_headers: HashMap = HashMap::new(); + if is_cross_origin { + req_headers.insert("origin".to_string(), page_origin.clone()); + } + for (k, v) in &custom_headers { + req_headers.insert(k.to_lowercase(), v.clone()); + } + + let r = stealth + .send_single(¤t_method, &parsed_current, &req_headers, ¤t_body) + .await + .map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?; + + if !(300..400).contains(&r.status) { + break (r.status, r.headers, r.body); + } + let Some(location) = r.headers.get("location").cloned() else { + break (r.status, r.headers, r.body); + }; + let next_url = match parsed_current.join(&location) { + Ok(u) => u, + Err(_) => break (r.status, r.headers, r.body), + }; + // Re-validate every redirect target against the SSRF policy, matching + // op_fetch_url (GHSA-8v6v-g4rh-jmcm). + if let Err(reason) = validate_fetch_url(&next_url) { + return Ok(serde_json::json!({ + "status": 0, "body": "", "url": next_url.to_string(), "headers": {}, + "blocked": true, + "error": format!("Redirect to forbidden URL blocked: {}", reason), + }) + .to_string()); + } + redirects_followed += 1; + if redirects_followed > FETCH_REDIRECT_LIMIT { + return Ok(serde_json::json!({ + "status": 0, "body": "", "url": next_url.to_string(), "headers": {}, + "blocked": true, + "error": format!("Too many redirects (>{})", FETCH_REDIRECT_LIMIT), + }) + .to_string()); + } + // Browser semantics: 301/302/303 downgrade to GET with no body. + if r.status == 301 || r.status == 302 || r.status == 303 { + current_method = "GET".to_string(); + current_body.clear(); + } + current_url = next_url.to_string(); + }; + + if is_cross_origin && mode == "cors" { + let allowed = resp_headers + .get("access-control-allow-origin") + .map(|s| s.as_str()) + .unwrap_or(""); + if allowed != "*" && allowed != page_origin { + return Ok(serde_json::json!({ + "status": 0, "body": "", "url": url, "headers": {}, + "corsBlocked": true, + "corsError": format!( + "CORS error: Origin '{}' not in Access-Control-Allow-Origin '{}'", + page_origin, allowed + ), + }) + .to_string()); + } + } + + let resp_body = String::from_utf8_lossy(&resp_bytes).to_string(); + let resp_body_base64 = BASE64.encode(&resp_bytes); + + Ok(serde_json::json!({ + "status": status, + "body": resp_body, + "bodyBase64": resp_body_base64, + "url": url, + "headers": resp_headers, + }) + .to_string()) +} + fn glob_match(pattern: &str, url: &str) -> bool { if pattern == "*" { return true; diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs index dcd7c96..f01c0bb 100644 --- a/crates/obscura-js/src/runtime.rs +++ b/crates/obscura-js/src/runtime.rs @@ -153,6 +153,13 @@ impl ObscuraJsRuntime { self.state.borrow_mut().http_client = Some(client); } + /// Install the stealth (wreq) HTTP client so scripted fetch()/XHR is routed + /// through it in stealth mode (see op_fetch_url / stealth_fetch_all). + #[cfg(feature = "stealth")] + pub fn set_stealth_client(&self, client: std::sync::Arc) { + self.state.borrow_mut().stealth_client = Some(client); + } + pub fn set_dom(&self, dom: DomTree) { self.state.borrow_mut().dom = Some(dom); } diff --git a/crates/obscura-net/src/wreq_client.rs b/crates/obscura-net/src/wreq_client.rs index 1361640..bf56a9a 100644 --- a/crates/obscura-net/src/wreq_client.rs +++ b/crates/obscura-net/src/wreq_client.rs @@ -154,6 +154,85 @@ impl StealthHttpClient { Err(ObscuraNetError::TooManyRedirects(url.to_string())) } + /// One request with no redirect following, for scripted fetch()/XHR. Reads + /// the cookie jar for the Cookie header and stores Set-Cookie back into it, + /// so the caller only owns redirect hops and SSRF re-validation. Used in + /// stealth mode so JS-level requests carry the same Chrome TLS fingerprint + /// and client hints as the main navigation instead of the rustls ClientHello + /// that op_fetch_url would otherwise send (which bot managers read as a + /// non-browser script and reject, e.g. the AWS WAF challenge verify call). + pub async fn send_single( + &self, + method: &str, + url: &Url, + headers: &HashMap, + body: &str, + ) -> Result { + if let Some(host) = url.host_str() { + if crate::blocklist::is_blocked(host) { + tracing::debug!("Blocked tracker: {}", url); + return Ok(Response { + status: 0, + url: url.clone(), + headers: HashMap::new(), + body: Vec::new(), + redirected_from: Vec::new(), + }); + } + } + + let req_method = method + .parse::() + .map_err(|e| ObscuraNetError::Network(format!("invalid method '{}': {}", method, e)))?; + let mut req = self.client.request(req_method, url.as_str()); + + let cookie_header = self.cookie_jar.get_cookie_header(url); + if !cookie_header.is_empty() { + req = req.header("cookie", &cookie_header); + } + for (k, v) in self.extra_headers.read().await.iter() { + req = req.header(k.as_str(), v.as_str()); + } + for (k, v) in headers.iter() { + req = req.header(k.as_str(), v.as_str()); + } + if !body.is_empty() { + req = req.body(body.to_string()); + } + + self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let resp = req.send().await.map_err(|e| { + self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + ObscuraNetError::Network(format!("{}: {}", url, e)) + })?; + self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + + let status = resp.status(); + for val in resp.headers().get_all("set-cookie") { + if let Ok(s) = val.to_str() { + self.cookie_jar.set_cookie(s, url); + } + } + let response_headers: HashMap = resp + .headers() + .iter() + .map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string())) + .collect(); + let resp_body = resp + .bytes() + .await + .map_err(|e| ObscuraNetError::Network(format!("Failed to read body: {}", e)))? + .to_vec(); + + Ok(Response { + url: url.clone(), + status: status.as_u16(), + headers: response_headers, + body: resp_body, + redirected_from: Vec::new(), + }) + } + pub async fn set_extra_headers(&self, headers: HashMap) { *self.extra_headers.write().await = headers; }