diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js index 9b4611c..50376b4 100644 --- a/crates/obscura-js/js/bootstrap.js +++ b/crates/obscura-js/js/bootstrap.js @@ -252,7 +252,7 @@ async function __processDynScriptQueue() { if (task.url.startsWith('data:')) { body = _decodeDataScriptUrl(task.url); } else { - const raw = await Deno.core.ops.op_fetch_url(task.url, "GET", "{}", "", task.pageOrigin, "no-cors"); + const raw = await Deno.core.ops.op_fetch_url(task.url, "GET", "{}", "", task.pageOrigin, "no-cors", "same-origin"); const parsed = JSON.parse(raw); // The HTML script-fetch algorithm treats an unsuccessful HTTP // response as a network error. Evaluating its response body is both @@ -392,7 +392,7 @@ async function _fetchLinkedCss(url, pageOrigin, depth = 0, seen = new Set()) { if (depth > 4 || seen.has(url)) return ""; seen.add(url); const raw = await Deno.core.ops.op_fetch_url( - url, "GET", "{}", "", pageOrigin, "no-cors" + url, "GET", "{}", "", pageOrigin, "no-cors", "same-origin" ); const parsed = JSON.parse(raw); if (parsed.blocked || parsed.status >= 400 || parsed.status === 0) { @@ -4911,6 +4911,7 @@ function _serializeBody(initBody, headers) { } globalThis.fetch = async (input, init = {}) => { + init = init || {}; let url = typeof input === "string" ? input : (input instanceof Request @@ -4927,8 +4928,14 @@ globalThis.fetch = async (input, init = {}) => { const body = _serializeBody(init.body, _h); const hdrs = JSON.stringify(_h); const fetchMode = init.mode || (input instanceof Request ? input.mode : "cors"); + const fetchCredentials = init.credentials !== undefined + ? String(init.credentials) + : (input instanceof Request ? input.credentials : "same-origin"); + if (fetchCredentials !== "omit" && fetchCredentials !== "same-origin" && fetchCredentials !== "include") { + throw new TypeError("Failed to execute 'fetch': '" + fetchCredentials + "' is not a valid RequestCredentials value"); + } const pageOrigin = (function() { try { const u = new URL(_domParse("document_url") || "about:blank"); return u.origin; } catch(e) { return ""; } })(); - const raw = await Deno.core.ops.op_fetch_url(url, method, hdrs, body, pageOrigin, fetchMode); + const raw = await Deno.core.ops.op_fetch_url(url, method, hdrs, body, pageOrigin, fetchMode, fetchCredentials); const parsed = JSON.parse(raw); if (parsed.blocked) { const err = new TypeError('net::ERR_FAILED'); @@ -5097,6 +5104,7 @@ globalThis.XMLHttpRequest = class XMLHttpRequest extends XMLHttpRequestEventTarg headers: this._headers, body: body || undefined, mode: 'cors', + credentials: this.withCredentials ? 'include' : 'same-origin', }).then(async (resp) => { if (xhr._aborted) return; @@ -5318,21 +5326,39 @@ _markNative(globalThis.cancelIdleCallback); if (typeof Request === 'undefined') { globalThis.Request = class Request { constructor(input, init = {}) { + const inputRequest = input instanceof Request ? input : null; if (typeof input === 'string') { this.url = input; } - else if (input instanceof Request) { this.url = input.url; init = { ...input, ...init }; } + else if (inputRequest) { this.url = inputRequest.url; init = { ...inputRequest, ...init }; } else if (typeof URL === 'function' && input instanceof URL) { this.url = input.href; } else { this.url = input?.url || input?.href || String(input); } this.method = (init.method || 'GET').toUpperCase(); this.headers = new Headers(init.headers); this.body = init.body || null; this.mode = init.mode || 'cors'; - this.credentials = init.credentials || 'same-origin'; + this.credentials = init.credentials !== undefined + ? String(init.credentials) + : (inputRequest ? inputRequest.credentials : 'same-origin'); + if (this.credentials !== 'omit' && this.credentials !== 'same-origin' && this.credentials !== 'include') { + throw new TypeError("Failed to construct 'Request': '" + this.credentials + "' is not a valid RequestCredentials value"); + } this.redirect = init.redirect || 'follow'; this.referrer = init.referrer || ''; this.signal = init.signal || { aborted: false, addEventListener(){}, removeEventListener(){} }; this.cache = init.cache || 'default'; } - clone() { return new Request(this.url, { method: this.method, headers: this.headers, body: this.body }); } + clone() { + return new Request(this.url, { + method: this.method, + headers: this.headers, + body: this.body, + mode: this.mode, + credentials: this.credentials, + redirect: this.redirect, + referrer: this.referrer, + signal: this.signal, + cache: this.cache, + }); + } async text() { return this.body ? String(this.body) : ''; } async json() { return JSON.parse(await this.text()); } async arrayBuffer() { return new TextEncoder().encode(await this.text()).buffer; } diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs index ee607c5..524cf98 100644 --- a/crates/obscura-js/src/ops.rs +++ b/crates/obscura-js/src/ops.rs @@ -772,6 +772,52 @@ fn fetch_timeout() -> std::time::Duration { /// Matches reqwest's default policy of 10. const FETCH_REDIRECT_LIMIT: usize = 10; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FetchCredentials { + Omit, + SameOrigin, + Include, +} + +impl FetchCredentials { + fn parse(value: &str) -> Self { + match value { + "omit" => Self::Omit, + "include" => Self::Include, + _ => Self::SameOrigin, + } + } + + fn allows(self, page_origin: &str, request_url: &str) -> bool { + match self { + Self::Omit => false, + Self::Include => true, + Self::SameOrigin => request_origin(request_url) + .map(|origin| origin == page_origin) + .unwrap_or(false), + } + } +} + +fn request_origin(request_url: &str) -> Option { + url::Url::parse(request_url) + .ok() + .map(|url| url.origin().ascii_serialization()) +} + +fn cors_response_allows( + credentials: FetchCredentials, + page_origin: &str, + allowed_origin: &str, + allow_credentials: &str, +) -> bool { + if credentials == FetchCredentials::Include { + allowed_origin == page_origin && allow_credentials == "true" + } else { + allowed_origin == "*" || allowed_origin == page_origin + } +} + #[op2(async)] #[string] async fn op_fetch_url( @@ -782,6 +828,7 @@ async fn op_fetch_url( #[string] body: String, #[string] origin: String, #[string] mode: String, + #[string] credentials: String, ) -> Result { tracing::debug!("op_fetch_url called: {} {} (intercept check pending)", method, url); @@ -926,18 +973,15 @@ async fn op_fetch_url( .map_err(deno_error::JsErrorBox::generic)?, }; - let request_origin = url::Url::parse(&url) - .ok() - .map(|u| { - let host = u.host_str().unwrap_or(""); - match u.port() { - Some(p) => format!("{}://{}:{}", u.scheme(), host, p), - None => format!("{}://{}", u.scheme(), host), - } - }) - .unwrap_or_default(); - let page_origin = if origin.is_empty() { request_origin.clone() } else { origin.clone() }; - let is_cross_origin = !page_origin.is_empty() && request_origin != page_origin; + let initial_request_origin = request_origin(&url).unwrap_or_default(); + let page_origin = if origin.is_empty() { + initial_request_origin.clone() + } else { + origin.clone() + }; + let is_cross_origin = + !page_origin.is_empty() && initial_request_origin != page_origin; + let credentials = FetchCredentials::parse(&credentials); let req_method: reqwest::Method = method.parse().unwrap_or(reqwest::Method::GET); @@ -962,35 +1006,6 @@ async fn op_fetch_url( } } - // Stealth mode: route the scripted request through the wreq client so its - // TLS fingerprint and Chrome client hints match the main navigation. The - // rustls ClientHello plus missing client hints that op_fetch_url's reqwest - // path sends otherwise read as a non-browser script to bot managers (the - // AWS WAF challenge verify call, Akamai sensors, etc.). - #[cfg(feature = "stealth")] - { - let stealth = { - let st = state.borrow(); - let gs = st.borrow::().clone(); - let client = gs.borrow().stealth_client.clone(); - client - }; - if let Some(stealth) = stealth { - return stealth_fetch_all( - stealth, - url.clone(), - req_method.as_str().to_string(), - custom_headers.clone(), - body.clone(), - page_origin.clone(), - is_cross_origin, - mode.clone(), - callbacks.clone(), - ) - .await; - } - } - let needs_preflight = is_cross_origin && mode == "cors" && (req_method != reqwest::Method::GET @@ -1022,7 +1037,17 @@ async fn op_fetch_url( .and_then(|v| v.to_str().ok()) .unwrap_or(""); - if allowed_origin != "*" && allowed_origin != page_origin { + let allow_credentials = preflight + .headers() + .get("access-control-allow-credentials") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + if !cors_response_allows( + credentials, + &page_origin, + allowed_origin, + allow_credentials, + ) { return Err(deno_error::JsErrorBox::generic(format!( "CORS preflight: Origin '{}' not allowed by Access-Control-Allow-Origin '{}'", page_origin, allowed_origin @@ -1030,6 +1055,33 @@ async fn op_fetch_url( } } + // Stealth mode: route scripted requests through wreq after the CORS + // preflight. stealth_fetch_all applies the credentials decision to each + // redirect hop without losing the Chrome TLS/client-hint transport. + #[cfg(feature = "stealth")] + { + let stealth = { + let st = state.borrow(); + let gs = st.borrow::().clone(); + let client = gs.borrow().stealth_client.clone(); + client + }; + if let Some(stealth) = stealth { + return stealth_fetch_all( + stealth, + url.clone(), + req_method.as_str().to_string(), + custom_headers.clone(), + body.clone(), + page_origin.clone(), + mode.clone(), + credentials, + callbacks.clone(), + ) + .await; + } + } + // Follow redirects manually so the SSRF policy applies to every hop. // reqwest's auto-follow would bypass validate_fetch_url on the redirect // target and let an attacker-allowed origin 302 to http://127.0.0.1 @@ -1043,11 +1095,15 @@ async fn op_fetch_url( .request(current_method.clone(), ¤t_url) .timeout(fetch_timeout()); - if is_cross_origin { + let current_is_cross_origin = request_origin(¤t_url) + .map(|request_origin| request_origin != page_origin) + .unwrap_or(false); + if current_is_cross_origin { req = req.header("Origin", &page_origin); } - if !is_cross_origin { + let credentials_allowed = credentials.allows(&page_origin, ¤t_url); + if credentials_allowed { if let Some(ref jar) = cookie_jar { if let Ok(parsed_url) = url::Url::parse(¤t_url) { let cookie_header = jar.get_cookie_header(&parsed_url); @@ -1091,11 +1147,13 @@ async fn op_fetch_url( counter.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); } - if let Some(ref jar) = cookie_jar { - if let Ok(parsed_url) = url::Url::parse(¤t_url) { - for val in resp.headers().get_all(reqwest::header::SET_COOKIE) { - if let Ok(s) = val.to_str() { - jar.set_cookie(s, &parsed_url); + if credentials_allowed { + if let Some(ref jar) = cookie_jar { + if let Ok(parsed_url) = url::Url::parse(¤t_url) { + for val in resp.headers().get_all(reqwest::header::SET_COOKIE) { + if let Ok(s) = val.to_str() { + jar.set_cookie(s, &parsed_url); + } } } } @@ -1169,20 +1227,34 @@ async fn op_fetch_url( .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) .collect(); - if is_cross_origin && mode == "cors" { + let final_is_cross_origin = request_origin(¤t_url) + .map(|request_origin| request_origin != page_origin) + .unwrap_or(false); + if final_is_cross_origin && mode == "cors" { let allowed = resp_headers .get("access-control-allow-origin") .map(|s| s.as_str()) .unwrap_or(""); - if allowed != "*" && allowed != page_origin { + let allow_credentials = resp_headers + .get("access-control-allow-credentials") + .map(|s| s.as_str()) + .unwrap_or(""); + if !cors_response_allows(credentials, &page_origin, allowed, allow_credentials) { return Ok(serde_json::json!({ "status": 0, "body": "", "url": url, "headers": {}, "corsBlocked": true, - "corsError": format!("CORS error: Origin '{}' not in Access-Control-Allow-Origin '{}'", page_origin, allowed), + "corsError": if credentials == FetchCredentials::Include { + format!( + "CORS error: credentialed request requires Access-Control-Allow-Origin '{}' and Access-Control-Allow-Credentials 'true'", + page_origin + ) + } else { + format!("CORS error: Origin '{}' not in Access-Control-Allow-Origin '{}'", page_origin, allowed) + }, }) .to_string()); } @@ -1294,8 +1366,8 @@ async fn stealth_fetch_all( custom_headers: HashMap, body: String, page_origin: String, - is_cross_origin: bool, mode: String, + credentials: FetchCredentials, callbacks: Option>, ) -> Result { let mut current_url = url.clone(); @@ -1315,15 +1387,25 @@ async fn stealth_fetch_all( }; let mut req_headers: HashMap = HashMap::new(); - if is_cross_origin { + let current_is_cross_origin = + parsed_current.origin().ascii_serialization() != page_origin; + if current_is_cross_origin { req_headers.insert("origin".to_string(), page_origin.clone()); } for (k, v) in &custom_headers { req_headers.insert(k.to_lowercase(), v.clone()); } + let credentials_allowed = credentials.allows(&page_origin, ¤t_url); let r = stealth - .send_single(¤t_method, &parsed_current, &req_headers, ¤t_body) + .send_single( + ¤t_method, + &parsed_current, + &req_headers, + ¤t_body, + credentials_allowed, + credentials_allowed, + ) .await .map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?; @@ -1364,19 +1446,33 @@ async fn stealth_fetch_all( current_url = next_url.to_string(); }; - if is_cross_origin && mode == "cors" { + let final_is_cross_origin = request_origin(¤t_url) + .map(|request_origin| request_origin != page_origin) + .unwrap_or(false); + if final_is_cross_origin && mode == "cors" { let allowed = resp_headers .get("access-control-allow-origin") .map(|s| s.as_str()) .unwrap_or(""); - if allowed != "*" && allowed != page_origin { + let allow_credentials = resp_headers + .get("access-control-allow-credentials") + .map(|s| s.as_str()) + .unwrap_or(""); + if !cors_response_allows(credentials, &page_origin, allowed, allow_credentials) { return Ok(serde_json::json!({ "status": 0, "body": "", "url": url, "headers": {}, "corsBlocked": true, - "corsError": format!( - "CORS error: Origin '{}' not in Access-Control-Allow-Origin '{}'", - page_origin, allowed - ), + "corsError": if credentials == FetchCredentials::Include { + format!( + "CORS error: credentialed request requires Access-Control-Allow-Origin '{}' and Access-Control-Allow-Credentials 'true'", + page_origin + ) + } else { + format!( + "CORS error: Origin '{}' not in Access-Control-Allow-Origin '{}'", + page_origin, allowed + ) + }, }) .to_string()); } @@ -1436,7 +1532,7 @@ fn glob_match(pattern: &str, url: &str) -> bool { #[cfg(test)] mod tests { - use super::glob_match; + use super::{cors_response_allows, glob_match, FetchCredentials}; #[test] fn glob_match_handles_cdp_blocked_url_patterns() { @@ -1461,6 +1557,54 @@ mod tests { "https://fonts.gstatic.com/s/inter/v18/font.woff", )); } + + #[test] + fn fetch_credentials_gate_cookie_send_and_storage_per_request_origin() { + let page_origin = "https://www.example.com"; + let same_origin_url = "https://www.example.com/api"; + let explicit_default_port = "https://www.example.com:443/api"; + let cross_origin_url = "https://api.example.com/data"; + + assert!(!FetchCredentials::Omit.allows(page_origin, same_origin_url)); + assert!(!FetchCredentials::Omit.allows(page_origin, cross_origin_url)); + + assert!(FetchCredentials::SameOrigin.allows(page_origin, same_origin_url)); + assert!(FetchCredentials::SameOrigin.allows(page_origin, explicit_default_port)); + assert!(!FetchCredentials::SameOrigin.allows(page_origin, cross_origin_url)); + + assert!(FetchCredentials::Include.allows(page_origin, same_origin_url)); + assert!(FetchCredentials::Include.allows(page_origin, cross_origin_url)); + } + + #[test] + fn credentialed_cors_requires_exact_origin_and_allow_credentials() { + let page_origin = "https://www.example.com"; + + assert!(cors_response_allows( + FetchCredentials::SameOrigin, + page_origin, + "*", + "", + )); + assert!(!cors_response_allows( + FetchCredentials::Include, + page_origin, + "*", + "true", + )); + assert!(!cors_response_allows( + FetchCredentials::Include, + page_origin, + page_origin, + "", + )); + assert!(cors_response_allows( + FetchCredentials::Include, + page_origin, + page_origin, + "true", + )); + } } fn validate_fetch_url(url: &url::Url) -> Result<(), String> { diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs index a0a3e62..5c6fd13 100644 --- a/crates/obscura-js/src/runtime.rs +++ b/crates/obscura-js/src/runtime.rs @@ -5603,6 +5603,81 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread")] + async fn fetch_and_xhr_forward_browser_credentials_modes() { + let mut rt = setup_runtime(""); + let result = rt + .call_function_on_for_cdp( + r#"async () => { + const originalFetchOp = Deno.core.ops.op_fetch_url; + const calls = []; + try { + Deno.core.ops.op_fetch_url = + (url, method, headers, body, origin, mode, credentials) => { + calls.push({ url, credentials }); + return JSON.stringify({ + status: 200, + headers: {}, + body: "ok", + url, + }); + }; + + await fetch("/default"); + await fetch("/omit", { credentials: "omit" }); + const request = new Request("/included", { credentials: "include" }); + await fetch(request); + await fetch(request.clone()); + await fetch(request, { credentials: "same-origin" }); + + const sendXhr = (path, withCredentials) => new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.open("GET", path); + xhr.withCredentials = withCredentials; + xhr.onload = resolve; + xhr.onerror = reject; + xhr.send(); + }); + await sendXhr("/xhr-default", false); + await sendXhr("/xhr-credentialed", true); + + let invalidFetchRejected = false; + try { + await fetch("/bad", { credentials: "invalid" }); + } catch (error) { + invalidFetchRejected = error instanceof TypeError; + } + + return { calls, invalidFetchRejected }; + } finally { + Deno.core.ops.op_fetch_url = originalFetchOp; + } + }"#, + None, + &[], + true, + true, + ) + .await + .unwrap(); + + assert_eq!( + result.value.unwrap(), + serde_json::json!({ + "calls": [ + { "url": "http://example.com/default", "credentials": "same-origin" }, + { "url": "http://example.com/omit", "credentials": "omit" }, + { "url": "http://example.com/included", "credentials": "include" }, + { "url": "http://example.com/included", "credentials": "include" }, + { "url": "http://example.com/included", "credentials": "same-origin" }, + { "url": "http://example.com/xhr-default", "credentials": "same-origin" }, + { "url": "http://example.com/xhr-credentialed", "credentials": "include" }, + ], + "invalidFetchRejected": true, + }) + ); + } + #[tokio::test(flavor = "current_thread")] async fn dynamic_linked_stylesheet_enters_the_live_dom_with_imports_rebased() { let mut rt = setup_runtime( diff --git a/crates/obscura-net/src/wreq_client.rs b/crates/obscura-net/src/wreq_client.rs index 259d66a..ccb275e 100644 --- a/crates/obscura-net/src/wreq_client.rs +++ b/crates/obscura-net/src/wreq_client.rs @@ -188,19 +188,17 @@ impl StealthHttpClient { Err(ObscuraNetError::TooManyRedirects(url.to_string())) } - /// One request with no redirect following, for scripted fetch()/XHR. Reads - /// the cookie jar for the Cookie header and stores Set-Cookie back into it, - /// so the caller only owns redirect hops and SSRF re-validation. Used in - /// stealth mode so JS-level requests carry the same Chrome TLS fingerprint - /// and client hints as the main navigation instead of the rustls ClientHello - /// that op_fetch_url would otherwise send (which bot managers read as a - /// non-browser script and reject, e.g. the AWS WAF challenge verify call). + /// One request with no redirect following, for scripted fetch()/XHR. The + /// caller supplies the Fetch credentials decision for this redirect hop, + /// while this method preserves the Chrome transport fingerprint. pub async fn send_single( &self, method: &str, url: &Url, headers: &HashMap, body: &str, + send_cookies: bool, + store_cookies: bool, ) -> Result { if let Some(host) = url.host_str() { if crate::blocklist::is_blocked(host) { @@ -220,9 +218,11 @@ impl StealthHttpClient { .map_err(|e| ObscuraNetError::Network(format!("invalid method '{}': {}", method, e)))?; let mut req = self.client.request(req_method, url.as_str()); - let cookie_header = self.cookie_jar.get_cookie_header(url); - if !cookie_header.is_empty() { - req = req.header("cookie", &cookie_header); + if send_cookies { + let cookie_header = self.cookie_jar.get_cookie_header(url); + if !cookie_header.is_empty() { + req = req.header("cookie", &cookie_header); + } } for (k, v) in self.extra_headers.read().await.iter() { req = req.header(k.as_str(), v.as_str()); @@ -242,9 +242,11 @@ impl StealthHttpClient { self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); let status = resp.status(); - for val in resp.headers().get_all("set-cookie") { - if let Ok(s) = val.to_str() { - self.cookie_jar.set_cookie(s, url); + if store_cookies { + for val in resp.headers().get_all("set-cookie") { + if let Ok(s) = val.to_str() { + self.cookie_jar.set_cookie(s, url); + } } } let response_headers: HashMap = resp