Merge stealth-overhaul into main

Resolved conflicts:
- page.rs: keep both set_blocked_urls (#315) and stealth client wiring
- ops.rs: keep SHA-512/224 + SHA-512/256 from stealth branch; keep
  the unreachable catch-all (vec![]) from #314 instead of silent
  SHA-256 fallback; updated comment accordingly
- bootstrap.js: extend the digest algorithm allowlist to include
  SHA-512/224 and SHA-512/256 to match the Rust additions
This commit is contained in:
SGavrl committed 2026-06-23 18:05:55 +00:00
commit 8582fcceca
9 files changed
+471 -88

No files matched your search

+1 -1
View File
@@ -5,7 +5,7 @@ edition.workspace = true
[features]
default = []
stealth = ["obscura-net/stealth"]
stealth = ["obscura-net/stealth", "obscura-js/stealth"]
[dependencies]
obscura-dom = { workspace = true }
+9 -3
View File
@@ -286,9 +286,11 @@ impl Page {
if self.stealth_client.is_some() {
rt.set_stealth(true);
rt.set_user_agent(obscura_net::STEALTH_USER_AGENT);
// Match platform to the Linux UA so navigator.platform and
// userAgentData.platform are consistent with navigator.userAgent.
rt.set_platform("Linux x86_64", "Linux", "");
rt.set_platform(
obscura_net::STEALTH_NAVIGATOR_PLATFORM,
obscura_net::STEALTH_UA_PLATFORM,
obscura_net::STEALTH_UA_PLATFORM_VERSION,
);
} else {
if let Ok(ua) = self.http_client.user_agent.try_read() {
rt.set_user_agent(&ua);
@@ -317,6 +319,10 @@ impl Page {
rt.set_cookie_jar(self.context.cookie_jar.clone());
rt.set_http_client(self.http_client.clone());
rt.set_blocked_urls(self.blocked_url_patterns.clone());
#[cfg(feature = "stealth")]
if let Some(ref stealth) = self.stealth_client {
rt.set_stealth_client(stealth.clone());
}
if let Some(tx) = &self.intercept_tx {
rt.set_intercept_tx(tx.clone());
+7
View File
@@ -3,6 +3,13 @@ name = "obscura-js"
version.workspace = true
edition.workspace = true
[features]
default = []
# Builds obscura-net with its stealth (wreq) HTTP client and lets scripted
# fetch()/XHR be routed through it so JS-level requests carry the same Chrome
# TLS fingerprint and client hints as stealth navigation.
stealth = ["obscura-net/stealth"]
[build-dependencies]
deno_core = "0.350"
+134 -46
View File
@@ -154,7 +154,9 @@ function _fpNoise(x, y, channel) {
var _fpCache = null;
function _getFp() {
if (_fpCache) return _fpCache;
const isMac = (globalThis.__obscura_ua_platform || 'Windows') === 'macOS';
const _uaPlat = globalThis.__obscura_ua_platform || 'Windows';
const isMac = _uaPlat === 'macOS';
const isLinux = _uaPlat === 'Linux';
const gpuPool = isMac ? [
'ANGLE (Apple, ANGLE Metal Renderer: Apple M1, Unspecified Version)',
'ANGLE (Apple, ANGLE Metal Renderer: Apple M1 Pro, Unspecified Version)',
@@ -162,6 +164,14 @@ function _getFp() {
'ANGLE (Apple, ANGLE Metal Renderer: Apple M2 Pro, Unspecified Version)',
'ANGLE (Apple, ANGLE Metal Renderer: Apple M3, Unspecified Version)',
'ANGLE (Intel Inc., ANGLE Metal Renderer: Intel(R) Iris(TM) Plus Graphics, Unspecified Version)',
] : isLinux ? [
'ANGLE (Intel, Mesa Intel(R) UHD Graphics 630 (CFL GT2), OpenGL 4.6)',
'ANGLE (Intel, Mesa Intel(R) Iris(R) Xe Graphics (TGL GT2), OpenGL 4.6)',
'ANGLE (Intel, Mesa Intel(R) UHD Graphics 770 (RPL-S), OpenGL 4.6)',
'ANGLE (AMD, AMD Radeon RX 580 (polaris10, LLVM 15.0.7, DRM 3.54, LLVM 15.0.7), OpenGL 4.6)',
'ANGLE (AMD, AMD Radeon RX 6700 XT (navi22, LLVM 16.0.6, DRM 3.54, LLVM 16.0.6), OpenGL 4.6)',
'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 OpenGL 4.6)',
'ANGLE (NVIDIA, NVIDIA GeForce RTX 4070 OpenGL 4.6)',
] : [
'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)',
'ANGLE (NVIDIA, NVIDIA GeForce GTX 1660 SUPER Direct3D11 vs_5_0 ps_5_0, D3D11)',
@@ -180,6 +190,10 @@ function _getFp() {
'Google Inc. (Apple)','Google Inc. (Apple)','Google Inc. (Apple)',
'Google Inc. (Apple)','Google Inc. (Apple)',
'Google Inc. (Intel Inc.)',
] : isLinux ? [
'Google Inc. (Intel)','Google Inc. (Intel)','Google Inc. (Intel)',
'Google Inc. (AMD)','Google Inc. (AMD)',
'Google Inc. (NVIDIA)','Google Inc. (NVIDIA)',
] : [
'Google Inc. (NVIDIA)','Google Inc. (NVIDIA)','Google Inc. (NVIDIA)',
'Google Inc. (Intel)','Google Inc. (Intel)',
@@ -2577,7 +2591,7 @@ function __currentUrl() {
}
globalThis.location = {
get href() { return __currentUrl(); },
set href(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); },
set href(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
get origin() { try { return new URL(this.href).origin; } catch { return ""; } },
get protocol() { try { return new URL(this.href).protocol; } catch { return ""; } },
get host() { try { return new URL(this.href).host; } catch { return ""; } },
@@ -2587,14 +2601,14 @@ globalThis.location = {
get hash() { try { return new URL(this.href).hash; } catch { return ""; } },
get port() { try { return new URL(this.href).port; } catch { return ""; } },
toString() { return this.href; },
assign(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); },
reload() {},
replace(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); },
assign(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
reload() { var r = _resolveUrl(this.href); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
replace(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
};
const _locationObj = globalThis.location;
Object.defineProperty(globalThis, 'location', {
get() { return _locationObj; },
set(url) { Deno.core.ops.op_navigate(_resolveUrl(String(url)), 'GET', ''); },
set(url) { var r = _resolveUrl(String(url)); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
configurable: false,
enumerable: true,
});
@@ -2752,8 +2766,35 @@ globalThis.NetworkInformation = NetworkInformation;
globalThis.ContentIndex = class ContentIndex {};
function _chromeMajor() {
var m = (globalThis.__obscura_ua || '').match(/Chrome\/(\d+)/);
return m ? (m[1] | 0) : 145;
}
// Chromium derives the sec-ch-ua GREASE brand, version, and brand order
// deterministically from the Chrome major version
// (components/embedder_support/user_agent_utils.cc). Replicating it keeps
// sec-ch-ua and userAgentData exact for every profile version rather than
// hardcoding one static token.
var _GREASE_CHARS = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_'];
var _GREASE_VER = ['8', '99', '24'];
var _BRAND_PERMS = [[0,1,2],[0,2,1],[1,0,2],[1,2,0],[2,0,1],[2,1,0]];
function _uaBrands() {
var seed = _chromeMajor();
var grease = {
brand: 'Not' + _GREASE_CHARS[seed % 11] + 'A' + _GREASE_CHARS[(seed + 1) % 11] + 'Brand',
version: _GREASE_VER[seed % 3],
};
var ordered = [
grease,
{brand: 'Chromium', version: String(seed)},
{brand: 'Google Chrome', version: String(seed)},
];
var p = _BRAND_PERMS[seed % 6];
return [ordered[p[0]], ordered[p[1]], ordered[p[2]]];
}
globalThis.navigator = {
get userAgent() { return globalThis.__obscura_ua || "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; },
get userAgent() { return globalThis.__obscura_ua || "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; },
get appVersion() { return this.userAgent.replace('Mozilla/', ''); },
language: "en-US", languages: ["en-US","en"], get platform() { return globalThis.__obscura_platform || "Win32"; },
onLine: true, cookieEnabled: true, hardwareConcurrency: 8,
@@ -2779,24 +2820,22 @@ globalThis.navigator = {
]);
},
userAgentData: {
brands: [
{brand: "Google Chrome", version: "145"},
{brand: "Chromium", version: "145"},
{brand: "Not;A=Brand", version: "24"},
],
mobile: false,
get brands() { return _uaBrands(); },
get platform() { return globalThis.__obscura_ua_platform || "Windows"; },
getHighEntropyValues(hints) {
var brands = _uaBrands();
return Promise.resolve({
architecture: "x86",
bitness: "64",
brands: [{brand:"Google Chrome",version:"145"},{brand:"Chromium",version:"145"},{brand:"Not;A=Brand",version:"24"}],
fullVersionList: [{brand:"Google Chrome",version:"145.0.0.0"},{brand:"Chromium",version:"145.0.0.0"},{brand:"Not;A=Brand",version:"24.0.0.0"}],
brands: brands,
fullVersionList: brands.map(function(b) { return {brand: b.brand, version: b.version + ".0.0.0"}; }),
mobile: false,
model: "",
platform: globalThis.__obscura_ua_platform || "Windows",
platformVersion: globalThis.__obscura_ua_platform_version || "15.0.0",
uaFullVersion: "145.0.0.0",
uaFullVersion: _chromeMajor() + ".0.0.0",
wow64: false,
});
},
toJSON() { return {brands:this.brands,mobile:this.mobile,platform:this.platform}; },
@@ -2817,8 +2856,12 @@ globalThis.navigator = {
},
clipboard: { writeText(){return Promise.resolve();}, readText(){return Promise.resolve("");} },
permissions: { query(params){
if (params?.name === 'notifications') return Promise.resolve({state:"prompt",onchange:null});
return Promise.resolve({state:"granted"});
var n = params && params.name;
// Chrome defaults privacy-sensitive permissions to "prompt", not "granted";
// returning "granted" for camera/microphone is a bot tell.
if (n === 'notifications') return Promise.resolve({state: (globalThis.Notification && Notification.permission === 'granted') ? 'granted' : 'prompt', onchange: null});
if (n === 'geolocation' || n === 'camera' || n === 'microphone' || n === 'midi') return Promise.resolve({state: 'prompt', onchange: null});
return Promise.resolve({state: 'granted', onchange: null});
} },
getBattery() { return Promise.resolve({ charging: _fp('batteryCharging'), chargingTime: _fp('batteryCharging') ? 0 : Infinity, dischargingTime: _fp('batteryCharging') ? Infinity : Math.floor(3600 + _fpRand(250) * 7200), level: _fp('batteryLevel'), addEventListener(){} }); },
getGamepads() { return []; },
@@ -3003,6 +3046,51 @@ function _installWasmStreamingFallback() {
}
_installWasmStreamingFallback();
// Serialize a FormData into a multipart/form-data body the way a browser does
// when it is passed as fetch()/XHR body. The previous shim did String(body),
// so a FormData became the literal "[object Object]" and the multipart payload
// (with its boundary) was lost; servers replied "Invalid boundary for
// multipart/form-data" (e.g. the AWS WAF challenge /mp_verify POST).
function _formDataToMultipart(fd) {
const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
let bnd = '----WebKitFormBoundary';
for (let i = 0; i < 16; i++) bnd += chars[Math.floor(Math.random() * chars.length)];
let out = '';
const entries = fd._d || [];
for (let i = 0; i < entries.length; i++) {
const k = entries[i][0], v = entries[i][1];
out += '--' + bnd + '\r\n';
if (v != null && typeof v === 'object' && v._bytes != null) {
out += 'Content-Disposition: form-data; name="' + k + '"; filename="' + (v.name || 'blob') + '"\r\n';
out += 'Content-Type: ' + (v.type || 'application/octet-stream') + '\r\n\r\n';
try { out += new TextDecoder().decode(v._bytes); } catch (e) {}
out += '\r\n';
} else {
out += 'Content-Disposition: form-data; name="' + k + '"\r\n\r\n' + String(v) + '\r\n';
}
}
out += '--' + bnd + '--\r\n';
return { boundary: bnd, body: out };
}
// Coerce a fetch()/XHR body into the string op_fetch_url expects, attaching a
// Content-Type header for body types that need one (FormData, URLSearchParams).
function _serializeBody(initBody, headers) {
if (initBody == null || initBody === '') return '';
if (initBody instanceof FormData) {
const mp = _formDataToMultipart(initBody);
headers['Content-Type'] = 'multipart/form-data; boundary=' + mp.boundary;
return mp.body;
}
if (initBody instanceof URLSearchParams) {
if (!Object.keys(headers).some(k => k.toLowerCase() === 'content-type')) {
headers['Content-Type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
}
return initBody.toString();
}
return typeof initBody === 'string' ? initBody : String(initBody);
}
globalThis.fetch = async (input, init = {}) => {
let url = typeof input === "string"
? input
@@ -3016,8 +3104,9 @@ globalThis.fetch = async (input, init = {}) => {
} catch(e) { /* keep as-is if URL resolution fails */ }
}
const method = init.method || (input instanceof Request ? input.method : "GET");
const hdrs = JSON.stringify(init.headers instanceof Headers ? Object.fromEntries(init.headers.entries()) : init.headers || {});
const body = init.body ? String(init.body) : "";
let _h = init.headers instanceof Headers ? Object.fromEntries(init.headers.entries()) : (init.headers || {});
const body = _serializeBody(init.body, _h);
const hdrs = JSON.stringify(_h);
const fetchMode = init.mode || (input instanceof Request ? input.mode : "cors");
const pageOrigin = (function() { try { const u = new URL(_domParse("document_url") || "about:blank"); return u.origin; } catch(e) { return ""; } })();
const raw = await Deno.core.ops.op_fetch_url(url, method, hdrs, body, pageOrigin, fetchMode);
@@ -6328,11 +6417,25 @@ globalThis.Worker = class Worker {
};
globalThis.__blobStore = globalThis.__blobStore || {};
const _origCreateObjectURL = URL.createObjectURL;
URL.createObjectURL = function(blob) {
if (blob && typeof blob.text === 'function') {
if (blob) {
const id = 'blob:obscura/' + Math.random().toString(36).substring(2);
blob.text().then(text => { globalThis.__blobStore[id] = text; });
// Store synchronously so a Worker built from the blob URL in the same
// tick sees its source. Blob-URL Worker construction is synchronous in
// real browsers; the previous async blob.text().then() store raced the
// Worker constructor, so new Worker(blobURL) fell through to fetch() and
// failed (net::ERR_FAILED), which broke AWS WAF's proof-of-work worker.
// The obscura Blob materializes _bytes in its constructor; fall back to
// the async text() store only for foreign Blob shims without _bytes.
if (blob._bytes) {
let text = '';
try { text = new TextDecoder().decode(blob._bytes); } catch (e) {}
globalThis.__blobStore[id] = text;
} else if (typeof blob.text === 'function') {
blob.text().then(text => { globalThis.__blobStore[id] = text; });
} else {
globalThis.__blobStore[id] = '';
}
return id;
}
return 'blob:obscura/fallback';
@@ -6421,7 +6524,8 @@ if (!globalThis.crypto.subtle) {
globalThis.crypto.subtle = {
async digest(algorithm, data) {
const name = (typeof algorithm === 'string' ? algorithm : algorithm?.name || '').toUpperCase().replace('_', '-');
if (name !== 'SHA-1' && name !== 'SHA-256' && name !== 'SHA-384' && name !== 'SHA-512') {
if (name !== 'SHA-1' && name !== 'SHA-256' && name !== 'SHA-384' && name !== 'SHA-512' &&
name !== 'SHA-512/224' && name !== 'SHA-512/256') {
throw new DOMException('Unrecognized algorithm name', 'NotSupportedError');
}
let bytes;
@@ -6840,6 +6944,10 @@ if (typeof ShadowRoot !== 'undefined' && !ShadowRoot.prototype.elementFromPoint)
globalThis.__obscura_init = function() {
_fpSeed = Date.now() ^ (Math.random() * 0xFFFFFFFF >>> 0);
_fpCache = null;
// A real navigation just completed (this runs after set_url), so drop any
// URL a location setter previewed synchronously and let document_url drive
// location.href again, including any redirect target.
globalThis.__virtualUrl = null;
_installWasmStreamingFallback();
globalThis.document = new Document(+_dom("document_node_id"));
@@ -6868,29 +6976,9 @@ globalThis.__obscura_init = function() {
};
globalThis.Notification.permission = "default";
if (globalThis.__obscura_stealth) {
var _stealthBrands = [
{brand: "Not:A-Brand", version: "99"},
{brand: "Google Chrome", version: "145"},
{brand: "Chromium", version: "145"},
];
globalThis.navigator.userAgentData.brands = _stealthBrands;
globalThis.navigator.userAgentData.getHighEntropyValues = function(hints) {
return Promise.resolve({
architecture: "x86", bitness: "64",
brands: _stealthBrands,
fullVersionList: [
{brand:"Not:A-Brand",version:"99.0.0.0"},
{brand:"Google Chrome",version:"145.0.0.0"},
{brand:"Chromium",version:"145.0.0.0"},
],
mobile: false, model: "",
platform: "Windows",
platformVersion: "10.0.0",
uaFullVersion: "145.0.0.0",
});
};
}
// userAgentData brands and getHighEntropyValues now derive the Chrome
// version from navigator.userAgent and read the platform from the page
// globals, so every stealth surface agrees without a per-mode override.
// Hide internals (_*, obscura, Obscura). The set of keys is static at
// snapshot-build time, so we precompute it ONCE below (after this
+156 -2
View File
@@ -9,6 +9,8 @@ use deno_core::Extension;
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use obscura_dom::{DomTree, NodeData, NodeId};
use obscura_net::{CookieJar, ObscuraHttpClient};
#[cfg(feature = "stealth")]
use obscura_net::StealthHttpClient;
use tokio::sync::Mutex;
pub type InterceptCallback = Arc<Mutex<Option<Box<dyn Fn(String, String, String) -> Option<(u16, String, String)> + Send + Sync>>>>;
@@ -55,6 +57,11 @@ pub struct ObscuraState {
pub blocked_urls: Vec<String>,
pub cookie_jar: Option<Arc<CookieJar>>,
pub http_client: Option<Arc<ObscuraHttpClient>>,
/// When set (stealth mode), scripted fetch()/XHR is routed through the wreq
/// client so the request carries the Chrome TLS fingerprint and client
/// hints instead of the rustls ClientHello op_fetch_url would otherwise send.
#[cfg(feature = "stealth")]
pub stealth_client: Option<Arc<StealthHttpClient>>,
pub pending_navigation: Option<(String, String, String)>,
pub intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<InterceptedRequest>>,
pub intercept_counter: u64,
@@ -82,6 +89,8 @@ impl ObscuraState {
blocked_urls: Vec::new(),
cookie_jar: None,
http_client: None,
#[cfg(feature = "stealth")]
stealth_client: None,
pending_navigation: None,
intercept_tx: None,
intercept_counter: 0,
@@ -707,6 +716,34 @@ async fn op_fetch_url(
let custom_headers: std::collections::HashMap<String, String> =
serde_json::from_str(&headers_json).unwrap_or_default();
// Stealth mode: route the scripted request through the wreq client so its
// TLS fingerprint and Chrome client hints match the main navigation. The
// rustls ClientHello plus missing client hints that op_fetch_url's reqwest
// path sends otherwise read as a non-browser script to bot managers (the
// AWS WAF challenge verify call, Akamai sensors, etc.).
#[cfg(feature = "stealth")]
{
let stealth = {
let st = state.borrow();
let gs = st.borrow::<SharedState>().clone();
let client = gs.borrow().stealth_client.clone();
client
};
if let Some(stealth) = stealth {
return stealth_fetch_all(
stealth,
url.clone(),
req_method.as_str().to_string(),
custom_headers.clone(),
body.clone(),
page_origin.clone(),
is_cross_origin,
mode.clone(),
)
.await;
}
}
let needs_preflight = is_cross_origin
&& mode == "cors"
&& (req_method != reqwest::Method::GET
@@ -946,6 +983,120 @@ async fn op_fetch_url(
.to_string())
}
/// Stealth-mode scripted fetch()/XHR: mirrors op_fetch_url's redirect, SSRF,
/// and CORS semantics but sends every hop through the wreq stealth client so
/// the request carries the Chrome TLS fingerprint and client hints. Cookie
/// handling lives inside StealthHttpClient::send_single, which shares the
/// context jar. Response bodies are not mirrored into the CDP
/// Network.getResponseBody buffer here; that is a follow-up for stealth fetches.
#[cfg(feature = "stealth")]
async fn stealth_fetch_all(
stealth: Arc<StealthHttpClient>,
url: String,
method: String,
custom_headers: HashMap<String, String>,
body: String,
page_origin: String,
is_cross_origin: bool,
mode: String,
) -> Result<String, deno_error::JsErrorBox> {
let mut current_url = url.clone();
let mut current_method = method;
let mut current_body = body;
let mut redirects_followed: usize = 0;
let (status, resp_headers, resp_bytes): (u16, HashMap<String, String>, Vec<u8>) = loop {
let parsed_current = match url::Url::parse(&current_url) {
Ok(u) => u,
Err(_) => {
return Ok(serde_json::json!({
"status": 0, "body": "", "url": current_url, "headers": {},
})
.to_string());
}
};
let mut req_headers: HashMap<String, String> = HashMap::new();
if is_cross_origin {
req_headers.insert("origin".to_string(), page_origin.clone());
}
for (k, v) in &custom_headers {
req_headers.insert(k.to_lowercase(), v.clone());
}
let r = stealth
.send_single(&current_method, &parsed_current, &req_headers, &current_body)
.await
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?;
if !(300..400).contains(&r.status) {
break (r.status, r.headers, r.body);
}
let Some(location) = r.headers.get("location").cloned() else {
break (r.status, r.headers, r.body);
};
let next_url = match parsed_current.join(&location) {
Ok(u) => u,
Err(_) => break (r.status, r.headers, r.body),
};
// Re-validate every redirect target against the SSRF policy, matching
// op_fetch_url (GHSA-8v6v-g4rh-jmcm).
if let Err(reason) = validate_fetch_url(&next_url) {
return Ok(serde_json::json!({
"status": 0, "body": "", "url": next_url.to_string(), "headers": {},
"blocked": true,
"error": format!("Redirect to forbidden URL blocked: {}", reason),
})
.to_string());
}
redirects_followed += 1;
if redirects_followed > FETCH_REDIRECT_LIMIT {
return Ok(serde_json::json!({
"status": 0, "body": "", "url": next_url.to_string(), "headers": {},
"blocked": true,
"error": format!("Too many redirects (>{})", FETCH_REDIRECT_LIMIT),
})
.to_string());
}
// Browser semantics: 301/302/303 downgrade to GET with no body.
if r.status == 301 || r.status == 302 || r.status == 303 {
current_method = "GET".to_string();
current_body.clear();
}
current_url = next_url.to_string();
};
if is_cross_origin && mode == "cors" {
let allowed = resp_headers
.get("access-control-allow-origin")
.map(|s| s.as_str())
.unwrap_or("");
if allowed != "*" && allowed != page_origin {
return Ok(serde_json::json!({
"status": 0, "body": "", "url": url, "headers": {},
"corsBlocked": true,
"corsError": format!(
"CORS error: Origin '{}' not in Access-Control-Allow-Origin '{}'",
page_origin, allowed
),
})
.to_string());
}
}
let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
let resp_body_base64 = BASE64.encode(&resp_bytes);
Ok(serde_json::json!({
"status": status,
"body": resp_body,
"bodyBase64": resp_body_base64,
"url": url,
"headers": resp_headers,
})
.to_string())
}
fn glob_match(pattern: &str, url: &str) -> bool {
if pattern == "*" {
return true;
@@ -1107,8 +1258,9 @@ fn op_binding_called(state: &OpState, #[string] name: &str, #[string] payload: &
}
/// Real WebCrypto `crypto.subtle.digest`. `algorithm` is the SubtleCrypto
/// algorithm name (`SHA-1` / `SHA-256` / `SHA-384` / `SHA-512`). The JS shim
/// validates the name before calling this op; any other value is unreachable.
/// algorithm name (`SHA-1` / `SHA-256` / `SHA-384` / `SHA-512`, plus the
/// FIPS 180-4 truncated variants `SHA-512/224` and `SHA-512/256`). The JS
/// shim validates the name; any other value is unreachable.
/// Returns the raw digest bytes so the JS shim can hand them back as an ArrayBuffer.
#[op2]
#[buffer]
@@ -1120,6 +1272,8 @@ fn op_subtle_digest(#[string] algorithm: &str, #[buffer] data: &[u8]) -> Vec<u8>
"SHA-256" => sha2::Sha256::digest(data).to_vec(),
"SHA-384" => sha2::Sha384::digest(data).to_vec(),
"SHA-512" => sha2::Sha512::digest(data).to_vec(),
"SHA-512/224" => sha2::Sha512_224::digest(data).to_vec(),
"SHA-512/256" => sha2::Sha512_256::digest(data).to_vec(),
_ => vec![],
}
}
+7
View File
@@ -153,6 +153,13 @@ impl ObscuraJsRuntime {
self.state.borrow_mut().http_client = Some(client);
}
/// Install the stealth (wreq) HTTP client so scripted fetch()/XHR is routed
/// through it in stealth mode (see op_fetch_url / stealth_fetch_all).
#[cfg(feature = "stealth")]
pub fn set_stealth_client(&self, client: std::sync::Arc<obscura_net::StealthHttpClient>) {
self.state.borrow_mut().stealth_client = Some(client);
}
pub fn set_dom(&self, dom: DomTree) {
self.state.borrow_mut().dom = Some(dom);
}
+61 -33
View File
@@ -273,6 +273,46 @@ pub struct ObscuraHttpClient {
pub allow_private_network: bool,
}
/// Derive the sec-ch-ua and sec-ch-ua-platform client-hint header values from a
/// User-Agent string, using Chromium's per-major-version GREASE algorithm so
/// the non-stealth HTTP path agrees with navigator.userAgentData instead of
/// shipping a fixed Linux/Chrome-145 hint that contradicts a Windows profile.
fn chrome_client_hints(ua: &str) -> (String, String) {
let major: usize = ua
.split("Chrome/")
.nth(1)
.and_then(|s| s.split('.').next())
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or(145);
const GREASE_CHARS: [char; 11] = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_'];
const GREASE_VER: [&str; 3] = ["8", "99", "24"];
const PERMS: [[usize; 3]; 6] = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]];
let grease_brand = format!(
"Not{}A{}Brand",
GREASE_CHARS[major % 11],
GREASE_CHARS[(major + 1) % 11]
);
let brands = [
(grease_brand, GREASE_VER[major % 3].to_string()),
("Chromium".to_string(), major.to_string()),
("Google Chrome".to_string(), major.to_string()),
];
let p = PERMS[major % 6];
let sec_ch_ua = p
.iter()
.map(|&i| format!("\"{}\";v=\"{}\"", brands[i].0, brands[i].1))
.collect::<Vec<_>>()
.join(", ");
let platform = if ua.contains("Windows NT") {
"\"Windows\""
} else if ua.contains("Macintosh") {
"\"macOS\""
} else {
"\"Linux\""
};
(sec_ch_ua, platform.to_string())
}
impl ObscuraHttpClient {
pub fn new() -> Self {
Self::with_cookie_jar(Arc::new(CookieJar::new()))
@@ -407,50 +447,38 @@ impl ObscuraHttpClient {
}
let ua = self.user_agent.read().await.clone();
let (sec_ch_ua, sec_ch_ua_platform) = chrome_client_hints(&ua);
let mut headers = HeaderMap::new();
// Chrome's top-level navigation header order. (reqwest appends
// accept-encoding/host after these, so accept-encoding lands after
// accept-language rather than before it; the rest matches Chrome.)
headers.insert(
HeaderName::from_static("sec-ch-ua"),
HeaderValue::from_str(&sec_ch_ua)
.unwrap_or_else(|_| HeaderValue::from_static("\"Not:A-Brand\";v=\"99\", \"Google Chrome\";v=\"145\", \"Chromium\";v=\"145\"")),
);
headers.insert(HeaderName::from_static("sec-ch-ua-mobile"), HeaderValue::from_static("?0"));
headers.insert(
HeaderName::from_static("sec-ch-ua-platform"),
HeaderValue::from_str(&sec_ch_ua_platform)
.unwrap_or_else(|_| HeaderValue::from_static("\"Windows\"")),
);
headers.insert(HeaderName::from_static("upgrade-insecure-requests"), HeaderValue::from_static("1"));
headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap_or_else(|_| {
HeaderValue::from_static("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36")
HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36")
}));
headers.insert(
reqwest::header::ACCEPT,
HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"),
);
headers.insert(HeaderName::from_static("sec-fetch-site"), HeaderValue::from_static("none"));
headers.insert(HeaderName::from_static("sec-fetch-mode"), HeaderValue::from_static("navigate"));
headers.insert(HeaderName::from_static("sec-fetch-user"), HeaderValue::from_static("?1"));
headers.insert(HeaderName::from_static("sec-fetch-dest"), HeaderValue::from_static("document"));
headers.insert(
reqwest::header::ACCEPT_LANGUAGE,
HeaderValue::from_static("en-US,en;q=0.9"),
);
headers.insert(
HeaderName::from_static("sec-ch-ua"),
HeaderValue::from_static("\"Chromium\";v=\"145\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"145\""),
);
headers.insert(
HeaderName::from_static("sec-ch-ua-mobile"),
HeaderValue::from_static("?0"),
);
headers.insert(
HeaderName::from_static("sec-ch-ua-platform"),
HeaderValue::from_static("\"Linux\""),
);
headers.insert(
HeaderName::from_static("sec-fetch-dest"),
HeaderValue::from_static("document"),
);
headers.insert(
HeaderName::from_static("sec-fetch-mode"),
HeaderValue::from_static("navigate"),
);
headers.insert(
HeaderName::from_static("sec-fetch-site"),
HeaderValue::from_static("none"),
);
headers.insert(
HeaderName::from_static("sec-fetch-user"),
HeaderValue::from_static("?1"),
);
headers.insert(
HeaderName::from_static("upgrade-insecure-requests"),
HeaderValue::from_static("1"),
);
let cookie_header = self.cookie_jar.get_cookie_header(&current_url);
tracing::debug!(
+4 -1
View File
@@ -19,4 +19,7 @@ pub use encoding::{
pub use robots::RobotsCache;
pub use blocklist::is_blocked as is_tracker_blocked;
#[cfg(feature = "stealth")]
pub use wreq_client::{StealthHttpClient, STEALTH_USER_AGENT};
pub use wreq_client::{
StealthHttpClient, STEALTH_NAVIGATOR_PLATFORM, STEALTH_UA_PLATFORM,
STEALTH_UA_PLATFORM_VERSION, STEALTH_USER_AGENT,
};
+92 -2
View File
@@ -19,7 +19,18 @@ use crate::client::{Response, ObscuraNetError};
#[cfg(feature = "stealth")]
pub const STEALTH_USER_AGENT: &str =
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
// The wreq emulation (Profile::Chrome145, Platform::Windows) sends this exact
// UA and sec-ch-ua-platform "Windows" on the wire. navigator has to report the
// same identity, otherwise the TLS/HTTP layer and the JS layer disagree and a
// site cross-checks the mismatch as a bot signal.
#[cfg(feature = "stealth")]
pub const STEALTH_NAVIGATOR_PLATFORM: &str = "Win32";
#[cfg(feature = "stealth")]
pub const STEALTH_UA_PLATFORM: &str = "Windows";
#[cfg(feature = "stealth")]
pub const STEALTH_UA_PLATFORM_VERSION: &str = "15.0.0";
#[cfg(feature = "stealth")]
pub struct StealthHttpClient {
@@ -38,7 +49,7 @@ impl StealthHttpClient {
pub fn with_proxy(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
let emulation_opts = wreq_util::Emulation::builder()
.profile(wreq_util::Profile::Chrome145)
.platform(wreq_util::Platform::Linux)
.platform(wreq_util::Platform::Windows)
.build();
let mut builder = wreq::Client::builder()
@@ -143,6 +154,85 @@ impl StealthHttpClient {
Err(ObscuraNetError::TooManyRedirects(url.to_string()))
}
/// One request with no redirect following, for scripted fetch()/XHR. Reads
/// the cookie jar for the Cookie header and stores Set-Cookie back into it,
/// so the caller only owns redirect hops and SSRF re-validation. Used in
/// stealth mode so JS-level requests carry the same Chrome TLS fingerprint
/// and client hints as the main navigation instead of the rustls ClientHello
/// that op_fetch_url would otherwise send (which bot managers read as a
/// non-browser script and reject, e.g. the AWS WAF challenge verify call).
pub async fn send_single(
&self,
method: &str,
url: &Url,
headers: &HashMap<String, String>,
body: &str,
) -> Result<Response, ObscuraNetError> {
if let Some(host) = url.host_str() {
if crate::blocklist::is_blocked(host) {
tracing::debug!("Blocked tracker: {}", url);
return Ok(Response {
status: 0,
url: url.clone(),
headers: HashMap::new(),
body: Vec::new(),
redirected_from: Vec::new(),
});
}
}
let req_method = method
.parse::<wreq::Method>()
.map_err(|e| ObscuraNetError::Network(format!("invalid method '{}': {}", method, e)))?;
let mut req = self.client.request(req_method, url.as_str());
let cookie_header = self.cookie_jar.get_cookie_header(url);
if !cookie_header.is_empty() {
req = req.header("cookie", &cookie_header);
}
for (k, v) in self.extra_headers.read().await.iter() {
req = req.header(k.as_str(), v.as_str());
}
for (k, v) in headers.iter() {
req = req.header(k.as_str(), v.as_str());
}
if !body.is_empty() {
req = req.body(body.to_string());
}
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let resp = req.send().await.map_err(|e| {
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
ObscuraNetError::Network(format!("{}: {}", url, e))
})?;
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
let status = resp.status();
for val in resp.headers().get_all("set-cookie") {
if let Ok(s) = val.to_str() {
self.cookie_jar.set_cookie(s, url);
}
}
let response_headers: HashMap<String, String> = resp
.headers()
.iter()
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
.collect();
let resp_body = resp
.bytes()
.await
.map_err(|e| ObscuraNetError::Network(format!("Failed to read body: {}", e)))?
.to_vec();
Ok(Response {
url: url.clone(),
status: status.as_u16(),
headers: response_headers,
body: resp_body,
redirected_from: Vec::new(),
})
}
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
*self.extra_headers.write().await = headers;
}