stealth: route scripted fetch()/XHR through the wreq client
op_fetch_url backed JS fetch() and XHR with a plain reqwest/rustls client, so in stealth mode scripted requests carried a non-browser TLS fingerprint and none of the Chrome client hints the main navigation sends. Bot managers that score subresource requests (the AWS WAF challenge verify call, Akamai sensors) read that as a script and reject it. Add StealthHttpClient::send_single and a stealth_fetch_all path that op_fetch_url uses when a stealth client is installed, preserving the redirect, SSRF, and CORS handling so JS-level requests share the navigation's Chrome TLS fingerprint and sec-ch-ua client hints. The reqwest path is unchanged for non-stealth. Adds a stealth feature to obscura-js gated on obscura-net/stealth.
This commit is contained in:
1 parent
4309935abd
commit
2e0981ef3f
6 files changed
+249
-1
No files matched your search
@@ -5,7 +5,7 @@ edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
stealth = ["obscura-net/stealth"]
|
||||
stealth = ["obscura-net/stealth", "obscura-js/stealth"]
|
||||
|
||||
[dependencies]
|
||||
obscura-dom = { workspace = true }
|
||||
|
||||
@@ -319,6 +319,10 @@ impl Page {
|
||||
|
||||
rt.set_cookie_jar(self.context.cookie_jar.clone());
|
||||
rt.set_http_client(self.http_client.clone());
|
||||
#[cfg(feature = "stealth")]
|
||||
if let Some(ref stealth) = self.stealth_client {
|
||||
rt.set_stealth_client(stealth.clone());
|
||||
}
|
||||
|
||||
if let Some(tx) = &self.intercept_tx {
|
||||
rt.set_intercept_tx(tx.clone());
|
||||
|
||||
@@ -3,6 +3,13 @@ name = "obscura-js"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Builds obscura-net with its stealth (wreq) HTTP client and lets scripted
|
||||
# fetch()/XHR be routed through it so JS-level requests carry the same Chrome
|
||||
# TLS fingerprint and client hints as stealth navigation.
|
||||
stealth = ["obscura-net/stealth"]
|
||||
|
||||
[build-dependencies]
|
||||
deno_core = "0.350"
|
||||
|
||||
|
||||
@@ -9,6 +9,8 @@ use deno_core::Extension;
|
||||
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
|
||||
use obscura_dom::{DomTree, NodeData, NodeId};
|
||||
use obscura_net::{CookieJar, ObscuraHttpClient};
|
||||
#[cfg(feature = "stealth")]
|
||||
use obscura_net::StealthHttpClient;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
pub type InterceptCallback = Arc<Mutex<Option<Box<dyn Fn(String, String, String) -> Option<(u16, String, String)> + Send + Sync>>>>;
|
||||
@@ -55,6 +57,11 @@ pub struct ObscuraState {
|
||||
pub blocked_urls: Vec<String>,
|
||||
pub cookie_jar: Option<Arc<CookieJar>>,
|
||||
pub http_client: Option<Arc<ObscuraHttpClient>>,
|
||||
/// When set (stealth mode), scripted fetch()/XHR is routed through the wreq
|
||||
/// client so the request carries the Chrome TLS fingerprint and client
|
||||
/// hints instead of the rustls ClientHello op_fetch_url would otherwise send.
|
||||
#[cfg(feature = "stealth")]
|
||||
pub stealth_client: Option<Arc<StealthHttpClient>>,
|
||||
pub pending_navigation: Option<(String, String, String)>,
|
||||
pub intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<InterceptedRequest>>,
|
||||
pub intercept_counter: u64,
|
||||
@@ -82,6 +89,8 @@ impl ObscuraState {
|
||||
blocked_urls: Vec::new(),
|
||||
cookie_jar: None,
|
||||
http_client: None,
|
||||
#[cfg(feature = "stealth")]
|
||||
stealth_client: None,
|
||||
pending_navigation: None,
|
||||
intercept_tx: None,
|
||||
intercept_counter: 0,
|
||||
@@ -707,6 +716,34 @@ async fn op_fetch_url(
|
||||
let custom_headers: std::collections::HashMap<String, String> =
|
||||
serde_json::from_str(&headers_json).unwrap_or_default();
|
||||
|
||||
// Stealth mode: route the scripted request through the wreq client so its
|
||||
// TLS fingerprint and Chrome client hints match the main navigation. The
|
||||
// rustls ClientHello plus missing client hints that op_fetch_url's reqwest
|
||||
// path sends otherwise read as a non-browser script to bot managers (the
|
||||
// AWS WAF challenge verify call, Akamai sensors, etc.).
|
||||
#[cfg(feature = "stealth")]
|
||||
{
|
||||
let stealth = {
|
||||
let st = state.borrow();
|
||||
let gs = st.borrow::<SharedState>().clone();
|
||||
let client = gs.borrow().stealth_client.clone();
|
||||
client
|
||||
};
|
||||
if let Some(stealth) = stealth {
|
||||
return stealth_fetch_all(
|
||||
stealth,
|
||||
url.clone(),
|
||||
req_method.as_str().to_string(),
|
||||
custom_headers.clone(),
|
||||
body.clone(),
|
||||
page_origin.clone(),
|
||||
is_cross_origin,
|
||||
mode.clone(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
let needs_preflight = is_cross_origin
|
||||
&& mode == "cors"
|
||||
&& (req_method != reqwest::Method::GET
|
||||
@@ -946,6 +983,120 @@ async fn op_fetch_url(
|
||||
.to_string())
|
||||
}
|
||||
|
||||
/// Stealth-mode scripted fetch()/XHR: mirrors op_fetch_url's redirect, SSRF,
|
||||
/// and CORS semantics but sends every hop through the wreq stealth client so
|
||||
/// the request carries the Chrome TLS fingerprint and client hints. Cookie
|
||||
/// handling lives inside StealthHttpClient::send_single, which shares the
|
||||
/// context jar. Response bodies are not mirrored into the CDP
|
||||
/// Network.getResponseBody buffer here; that is a follow-up for stealth fetches.
|
||||
#[cfg(feature = "stealth")]
|
||||
async fn stealth_fetch_all(
|
||||
stealth: Arc<StealthHttpClient>,
|
||||
url: String,
|
||||
method: String,
|
||||
custom_headers: HashMap<String, String>,
|
||||
body: String,
|
||||
page_origin: String,
|
||||
is_cross_origin: bool,
|
||||
mode: String,
|
||||
) -> Result<String, deno_error::JsErrorBox> {
|
||||
let mut current_url = url.clone();
|
||||
let mut current_method = method;
|
||||
let mut current_body = body;
|
||||
let mut redirects_followed: usize = 0;
|
||||
|
||||
let (status, resp_headers, resp_bytes): (u16, HashMap<String, String>, Vec<u8>) = loop {
|
||||
let parsed_current = match url::Url::parse(¤t_url) {
|
||||
Ok(u) => u,
|
||||
Err(_) => {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0, "body": "", "url": current_url, "headers": {},
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
};
|
||||
|
||||
let mut req_headers: HashMap<String, String> = HashMap::new();
|
||||
if is_cross_origin {
|
||||
req_headers.insert("origin".to_string(), page_origin.clone());
|
||||
}
|
||||
for (k, v) in &custom_headers {
|
||||
req_headers.insert(k.to_lowercase(), v.clone());
|
||||
}
|
||||
|
||||
let r = stealth
|
||||
.send_single(¤t_method, &parsed_current, &req_headers, ¤t_body)
|
||||
.await
|
||||
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?;
|
||||
|
||||
if !(300..400).contains(&r.status) {
|
||||
break (r.status, r.headers, r.body);
|
||||
}
|
||||
let Some(location) = r.headers.get("location").cloned() else {
|
||||
break (r.status, r.headers, r.body);
|
||||
};
|
||||
let next_url = match parsed_current.join(&location) {
|
||||
Ok(u) => u,
|
||||
Err(_) => break (r.status, r.headers, r.body),
|
||||
};
|
||||
// Re-validate every redirect target against the SSRF policy, matching
|
||||
// op_fetch_url (GHSA-8v6v-g4rh-jmcm).
|
||||
if let Err(reason) = validate_fetch_url(&next_url) {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0, "body": "", "url": next_url.to_string(), "headers": {},
|
||||
"blocked": true,
|
||||
"error": format!("Redirect to forbidden URL blocked: {}", reason),
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
redirects_followed += 1;
|
||||
if redirects_followed > FETCH_REDIRECT_LIMIT {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0, "body": "", "url": next_url.to_string(), "headers": {},
|
||||
"blocked": true,
|
||||
"error": format!("Too many redirects (>{})", FETCH_REDIRECT_LIMIT),
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
// Browser semantics: 301/302/303 downgrade to GET with no body.
|
||||
if r.status == 301 || r.status == 302 || r.status == 303 {
|
||||
current_method = "GET".to_string();
|
||||
current_body.clear();
|
||||
}
|
||||
current_url = next_url.to_string();
|
||||
};
|
||||
|
||||
if is_cross_origin && mode == "cors" {
|
||||
let allowed = resp_headers
|
||||
.get("access-control-allow-origin")
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or("");
|
||||
if allowed != "*" && allowed != page_origin {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0, "body": "", "url": url, "headers": {},
|
||||
"corsBlocked": true,
|
||||
"corsError": format!(
|
||||
"CORS error: Origin '{}' not in Access-Control-Allow-Origin '{}'",
|
||||
page_origin, allowed
|
||||
),
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
|
||||
let resp_body_base64 = BASE64.encode(&resp_bytes);
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"status": status,
|
||||
"body": resp_body,
|
||||
"bodyBase64": resp_body_base64,
|
||||
"url": url,
|
||||
"headers": resp_headers,
|
||||
})
|
||||
.to_string())
|
||||
}
|
||||
|
||||
fn glob_match(pattern: &str, url: &str) -> bool {
|
||||
if pattern == "*" {
|
||||
return true;
|
||||
|
||||
@@ -153,6 +153,13 @@ impl ObscuraJsRuntime {
|
||||
self.state.borrow_mut().http_client = Some(client);
|
||||
}
|
||||
|
||||
/// Install the stealth (wreq) HTTP client so scripted fetch()/XHR is routed
|
||||
/// through it in stealth mode (see op_fetch_url / stealth_fetch_all).
|
||||
#[cfg(feature = "stealth")]
|
||||
pub fn set_stealth_client(&self, client: std::sync::Arc<obscura_net::StealthHttpClient>) {
|
||||
self.state.borrow_mut().stealth_client = Some(client);
|
||||
}
|
||||
|
||||
pub fn set_dom(&self, dom: DomTree) {
|
||||
self.state.borrow_mut().dom = Some(dom);
|
||||
}
|
||||
|
||||
@@ -154,6 +154,85 @@ impl StealthHttpClient {
|
||||
Err(ObscuraNetError::TooManyRedirects(url.to_string()))
|
||||
}
|
||||
|
||||
/// One request with no redirect following, for scripted fetch()/XHR. Reads
|
||||
/// the cookie jar for the Cookie header and stores Set-Cookie back into it,
|
||||
/// so the caller only owns redirect hops and SSRF re-validation. Used in
|
||||
/// stealth mode so JS-level requests carry the same Chrome TLS fingerprint
|
||||
/// and client hints as the main navigation instead of the rustls ClientHello
|
||||
/// that op_fetch_url would otherwise send (which bot managers read as a
|
||||
/// non-browser script and reject, e.g. the AWS WAF challenge verify call).
|
||||
pub async fn send_single(
|
||||
&self,
|
||||
method: &str,
|
||||
url: &Url,
|
||||
headers: &HashMap<String, String>,
|
||||
body: &str,
|
||||
) -> Result<Response, ObscuraNetError> {
|
||||
if let Some(host) = url.host_str() {
|
||||
if crate::blocklist::is_blocked(host) {
|
||||
tracing::debug!("Blocked tracker: {}", url);
|
||||
return Ok(Response {
|
||||
status: 0,
|
||||
url: url.clone(),
|
||||
headers: HashMap::new(),
|
||||
body: Vec::new(),
|
||||
redirected_from: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let req_method = method
|
||||
.parse::<wreq::Method>()
|
||||
.map_err(|e| ObscuraNetError::Network(format!("invalid method '{}': {}", method, e)))?;
|
||||
let mut req = self.client.request(req_method, url.as_str());
|
||||
|
||||
let cookie_header = self.cookie_jar.get_cookie_header(url);
|
||||
if !cookie_header.is_empty() {
|
||||
req = req.header("cookie", &cookie_header);
|
||||
}
|
||||
for (k, v) in self.extra_headers.read().await.iter() {
|
||||
req = req.header(k.as_str(), v.as_str());
|
||||
}
|
||||
for (k, v) in headers.iter() {
|
||||
req = req.header(k.as_str(), v.as_str());
|
||||
}
|
||||
if !body.is_empty() {
|
||||
req = req.body(body.to_string());
|
||||
}
|
||||
|
||||
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let resp = req.send().await.map_err(|e| {
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
ObscuraNetError::Network(format!("{}: {}", url, e))
|
||||
})?;
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let status = resp.status();
|
||||
for val in resp.headers().get_all("set-cookie") {
|
||||
if let Ok(s) = val.to_str() {
|
||||
self.cookie_jar.set_cookie(s, url);
|
||||
}
|
||||
}
|
||||
let response_headers: HashMap<String, String> = resp
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
let resp_body = resp
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| ObscuraNetError::Network(format!("Failed to read body: {}", e)))?
|
||||
.to_vec();
|
||||
|
||||
Ok(Response {
|
||||
url: url.clone(),
|
||||
status: status.as_u16(),
|
||||
headers: response_headers,
|
||||
body: resp_body,
|
||||
redirected_from: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
|
||||
*self.extra_headers.write().await = headers;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user