stealth: fingerprint consistency, scripted-request TLS, FormData

**Fingerprint consistency (stealth)** The stealth TLS, navigator, and WebGL surfaces previously disagreed. The wreq client reported Windows Chrome but navigator reported Linux, and WebGL returned Direct3D11 renderers for a Linux UA — a combination no real browser ever produces. Everything now presents one consistent Windows Chrome 145 identity: the HTTP User-Agent header, sec-ch-ua-platform, navigator.userAgent, navigator.platform, userAgentData, and WebGL renderer all agree. userAgentData brands are derived from the Chrome major version via Chromium's GREASE algorithm instead of a single hardcoded token.

**sec-ch-ua client hints on the default (non-stealth) path** The plain navigation client hardcoded `sec-ch-ua-platform: "Linux"` and Chrome 149 brands regardless of which profile was selected. Both headers are now derived from the actual User-Agent using the same GREASE algorithm, so non-stealth requests are also internally consistent.

**Route scripted fetch()/XHR through the stealth TLS client** In stealth mode, JS `fetch()` and `XMLHttpRequest` were still going through the plain reqwest/rustls stack, so subresource requests (sensors, WAF challenge verify calls) arrived with a non-browser TLS fingerprint. They now share the navigation's Chrome TLS fingerprint and sec-ch-ua client hints. The non-stealth path is unchanged.

**FormData serialization in fetch() — the booking.com fix** Passing a `FormData` as a `fetch()` or `XHR.send()` body sent the literal string `"[object Object]"` instead of a multipart payload. booking.com's AWS WAF challenge POSTs its proof-of-work solution to `/mp_verify` as FormData, so the verify always failed and no `aws-waf-token` was issued. `FormData` is now serialized to a proper `multipart/form-data` body with a generated boundary; `URLSearchParams` gets `application/x-www-form-urlencoded`. XHR routes through `fetch()` so it is covered too.

**navigator.permissions.query defaults** `permissions.query` returned `"granted"` for every permission. Real Chrome defaults camera, microphone, geolocation, and midi to `"prompt"`. Fixed.

**SHA-512/224 and SHA-512/256 in crypto.subtle.digest** Added FIPS 180-4 truncated SHA-512 variants to the Rust op and the JS allowlist. WAF challenge scripts that use these algorithms now get real output instead of a NotSupportedError.

**location updates synchronously on navigation, Blob-URL Worker race fixed**
This commit is contained in:
Gabriel authored and GitHub committed 2026-06-23 18:14:44 +00:00
commit d2206fea84
9 files changed
+470 -87

No files matched your search

+1 -1
View File
@@ -5,7 +5,7 @@ edition.workspace = true
[features] [features]
default = [] default = []
stealth = ["obscura-net/stealth"] stealth = ["obscura-net/stealth", "obscura-js/stealth"]
[dependencies] [dependencies]
obscura-dom = { workspace = true } obscura-dom = { workspace = true }
+9 -3
View File
@@ -286,9 +286,11 @@ impl Page {
if self.stealth_client.is_some() { if self.stealth_client.is_some() {
rt.set_stealth(true); rt.set_stealth(true);
rt.set_user_agent(obscura_net::STEALTH_USER_AGENT); rt.set_user_agent(obscura_net::STEALTH_USER_AGENT);
// Match platform to the Linux UA so navigator.platform and rt.set_platform(
// userAgentData.platform are consistent with navigator.userAgent. obscura_net::STEALTH_NAVIGATOR_PLATFORM,
rt.set_platform("Linux x86_64", "Linux", ""); obscura_net::STEALTH_UA_PLATFORM,
obscura_net::STEALTH_UA_PLATFORM_VERSION,
);
} else { } else {
if let Ok(ua) = self.http_client.user_agent.try_read() { if let Ok(ua) = self.http_client.user_agent.try_read() {
rt.set_user_agent(&ua); rt.set_user_agent(&ua);
@@ -317,6 +319,10 @@ impl Page {
rt.set_cookie_jar(self.context.cookie_jar.clone()); rt.set_cookie_jar(self.context.cookie_jar.clone());
rt.set_http_client(self.http_client.clone()); rt.set_http_client(self.http_client.clone());
rt.set_blocked_urls(self.blocked_url_patterns.clone()); rt.set_blocked_urls(self.blocked_url_patterns.clone());
#[cfg(feature = "stealth")]
if let Some(ref stealth) = self.stealth_client {
rt.set_stealth_client(stealth.clone());
}
if let Some(tx) = &self.intercept_tx { if let Some(tx) = &self.intercept_tx {
rt.set_intercept_tx(tx.clone()); rt.set_intercept_tx(tx.clone());
+7
View File
@@ -3,6 +3,13 @@ name = "obscura-js"
version.workspace = true version.workspace = true
edition.workspace = true edition.workspace = true
[features]
default = []
# Builds obscura-net with its stealth (wreq) HTTP client and lets scripted
# fetch()/XHR be routed through it so JS-level requests carry the same Chrome
# TLS fingerprint and client hints as stealth navigation.
stealth = ["obscura-net/stealth"]
[build-dependencies] [build-dependencies]
deno_core = "0.350" deno_core = "0.350"
+133 -45
View File
@@ -154,7 +154,9 @@ function _fpNoise(x, y, channel) {
var _fpCache = null; var _fpCache = null;
function _getFp() { function _getFp() {
if (_fpCache) return _fpCache; if (_fpCache) return _fpCache;
const isMac = (globalThis.__obscura_ua_platform || 'Windows') === 'macOS'; const _uaPlat = globalThis.__obscura_ua_platform || 'Windows';
const isMac = _uaPlat === 'macOS';
const isLinux = _uaPlat === 'Linux';
const gpuPool = isMac ? [ const gpuPool = isMac ? [
'ANGLE (Apple, ANGLE Metal Renderer: Apple M1, Unspecified Version)', 'ANGLE (Apple, ANGLE Metal Renderer: Apple M1, Unspecified Version)',
'ANGLE (Apple, ANGLE Metal Renderer: Apple M1 Pro, Unspecified Version)', 'ANGLE (Apple, ANGLE Metal Renderer: Apple M1 Pro, Unspecified Version)',
@@ -162,6 +164,14 @@ function _getFp() {
'ANGLE (Apple, ANGLE Metal Renderer: Apple M2 Pro, Unspecified Version)', 'ANGLE (Apple, ANGLE Metal Renderer: Apple M2 Pro, Unspecified Version)',
'ANGLE (Apple, ANGLE Metal Renderer: Apple M3, Unspecified Version)', 'ANGLE (Apple, ANGLE Metal Renderer: Apple M3, Unspecified Version)',
'ANGLE (Intel Inc., ANGLE Metal Renderer: Intel(R) Iris(TM) Plus Graphics, Unspecified Version)', 'ANGLE (Intel Inc., ANGLE Metal Renderer: Intel(R) Iris(TM) Plus Graphics, Unspecified Version)',
] : isLinux ? [
'ANGLE (Intel, Mesa Intel(R) UHD Graphics 630 (CFL GT2), OpenGL 4.6)',
'ANGLE (Intel, Mesa Intel(R) Iris(R) Xe Graphics (TGL GT2), OpenGL 4.6)',
'ANGLE (Intel, Mesa Intel(R) UHD Graphics 770 (RPL-S), OpenGL 4.6)',
'ANGLE (AMD, AMD Radeon RX 580 (polaris10, LLVM 15.0.7, DRM 3.54, LLVM 15.0.7), OpenGL 4.6)',
'ANGLE (AMD, AMD Radeon RX 6700 XT (navi22, LLVM 16.0.6, DRM 3.54, LLVM 16.0.6), OpenGL 4.6)',
'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 OpenGL 4.6)',
'ANGLE (NVIDIA, NVIDIA GeForce RTX 4070 OpenGL 4.6)',
] : [ ] : [
'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)', 'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)',
'ANGLE (NVIDIA, NVIDIA GeForce GTX 1660 SUPER Direct3D11 vs_5_0 ps_5_0, D3D11)', 'ANGLE (NVIDIA, NVIDIA GeForce GTX 1660 SUPER Direct3D11 vs_5_0 ps_5_0, D3D11)',
@@ -180,6 +190,10 @@ function _getFp() {
'Google Inc. (Apple)','Google Inc. (Apple)','Google Inc. (Apple)', 'Google Inc. (Apple)','Google Inc. (Apple)','Google Inc. (Apple)',
'Google Inc. (Apple)','Google Inc. (Apple)', 'Google Inc. (Apple)','Google Inc. (Apple)',
'Google Inc. (Intel Inc.)', 'Google Inc. (Intel Inc.)',
] : isLinux ? [
'Google Inc. (Intel)','Google Inc. (Intel)','Google Inc. (Intel)',
'Google Inc. (AMD)','Google Inc. (AMD)',
'Google Inc. (NVIDIA)','Google Inc. (NVIDIA)',
] : [ ] : [
'Google Inc. (NVIDIA)','Google Inc. (NVIDIA)','Google Inc. (NVIDIA)', 'Google Inc. (NVIDIA)','Google Inc. (NVIDIA)','Google Inc. (NVIDIA)',
'Google Inc. (Intel)','Google Inc. (Intel)', 'Google Inc. (Intel)','Google Inc. (Intel)',
@@ -2577,7 +2591,7 @@ function __currentUrl() {
} }
globalThis.location = { globalThis.location = {
get href() { return __currentUrl(); }, get href() { return __currentUrl(); },
set href(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); }, set href(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
get origin() { try { return new URL(this.href).origin; } catch { return ""; } }, get origin() { try { return new URL(this.href).origin; } catch { return ""; } },
get protocol() { try { return new URL(this.href).protocol; } catch { return ""; } }, get protocol() { try { return new URL(this.href).protocol; } catch { return ""; } },
get host() { try { return new URL(this.href).host; } catch { return ""; } }, get host() { try { return new URL(this.href).host; } catch { return ""; } },
@@ -2587,14 +2601,14 @@ globalThis.location = {
get hash() { try { return new URL(this.href).hash; } catch { return ""; } }, get hash() { try { return new URL(this.href).hash; } catch { return ""; } },
get port() { try { return new URL(this.href).port; } catch { return ""; } }, get port() { try { return new URL(this.href).port; } catch { return ""; } },
toString() { return this.href; }, toString() { return this.href; },
assign(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); }, assign(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
reload() {}, reload() { var r = _resolveUrl(this.href); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
replace(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); }, replace(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
}; };
const _locationObj = globalThis.location; const _locationObj = globalThis.location;
Object.defineProperty(globalThis, 'location', { Object.defineProperty(globalThis, 'location', {
get() { return _locationObj; }, get() { return _locationObj; },
set(url) { Deno.core.ops.op_navigate(_resolveUrl(String(url)), 'GET', ''); }, set(url) { var r = _resolveUrl(String(url)); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
configurable: false, configurable: false,
enumerable: true, enumerable: true,
}); });
@@ -2752,8 +2766,35 @@ globalThis.NetworkInformation = NetworkInformation;
globalThis.ContentIndex = class ContentIndex {}; globalThis.ContentIndex = class ContentIndex {};
function _chromeMajor() {
var m = (globalThis.__obscura_ua || '').match(/Chrome\/(\d+)/);
return m ? (m[1] | 0) : 145;
}
// Chromium derives the sec-ch-ua GREASE brand, version, and brand order
// deterministically from the Chrome major version
// (components/embedder_support/user_agent_utils.cc). Replicating it keeps
// sec-ch-ua and userAgentData exact for every profile version rather than
// hardcoding one static token.
var _GREASE_CHARS = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_'];
var _GREASE_VER = ['8', '99', '24'];
var _BRAND_PERMS = [[0,1,2],[0,2,1],[1,0,2],[1,2,0],[2,0,1],[2,1,0]];
function _uaBrands() {
var seed = _chromeMajor();
var grease = {
brand: 'Not' + _GREASE_CHARS[seed % 11] + 'A' + _GREASE_CHARS[(seed + 1) % 11] + 'Brand',
version: _GREASE_VER[seed % 3],
};
var ordered = [
grease,
{brand: 'Chromium', version: String(seed)},
{brand: 'Google Chrome', version: String(seed)},
];
var p = _BRAND_PERMS[seed % 6];
return [ordered[p[0]], ordered[p[1]], ordered[p[2]]];
}
globalThis.navigator = { globalThis.navigator = {
get userAgent() { return globalThis.__obscura_ua || "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; }, get userAgent() { return globalThis.__obscura_ua || "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; },
get appVersion() { return this.userAgent.replace('Mozilla/', ''); }, get appVersion() { return this.userAgent.replace('Mozilla/', ''); },
language: "en-US", languages: ["en-US","en"], get platform() { return globalThis.__obscura_platform || "Win32"; }, language: "en-US", languages: ["en-US","en"], get platform() { return globalThis.__obscura_platform || "Win32"; },
onLine: true, cookieEnabled: true, hardwareConcurrency: 8, onLine: true, cookieEnabled: true, hardwareConcurrency: 8,
@@ -2779,24 +2820,22 @@ globalThis.navigator = {
]); ]);
}, },
userAgentData: { userAgentData: {
brands: [
{brand: "Google Chrome", version: "145"},
{brand: "Chromium", version: "145"},
{brand: "Not;A=Brand", version: "24"},
],
mobile: false, mobile: false,
get brands() { return _uaBrands(); },
get platform() { return globalThis.__obscura_ua_platform || "Windows"; }, get platform() { return globalThis.__obscura_ua_platform || "Windows"; },
getHighEntropyValues(hints) { getHighEntropyValues(hints) {
var brands = _uaBrands();
return Promise.resolve({ return Promise.resolve({
architecture: "x86", architecture: "x86",
bitness: "64", bitness: "64",
brands: [{brand:"Google Chrome",version:"145"},{brand:"Chromium",version:"145"},{brand:"Not;A=Brand",version:"24"}], brands: brands,
fullVersionList: [{brand:"Google Chrome",version:"145.0.0.0"},{brand:"Chromium",version:"145.0.0.0"},{brand:"Not;A=Brand",version:"24.0.0.0"}], fullVersionList: brands.map(function(b) { return {brand: b.brand, version: b.version + ".0.0.0"}; }),
mobile: false, mobile: false,
model: "", model: "",
platform: globalThis.__obscura_ua_platform || "Windows", platform: globalThis.__obscura_ua_platform || "Windows",
platformVersion: globalThis.__obscura_ua_platform_version || "15.0.0", platformVersion: globalThis.__obscura_ua_platform_version || "15.0.0",
uaFullVersion: "145.0.0.0", uaFullVersion: _chromeMajor() + ".0.0.0",
wow64: false,
}); });
}, },
toJSON() { return {brands:this.brands,mobile:this.mobile,platform:this.platform}; }, toJSON() { return {brands:this.brands,mobile:this.mobile,platform:this.platform}; },
@@ -2817,8 +2856,12 @@ globalThis.navigator = {
}, },
clipboard: { writeText(){return Promise.resolve();}, readText(){return Promise.resolve("");} }, clipboard: { writeText(){return Promise.resolve();}, readText(){return Promise.resolve("");} },
permissions: { query(params){ permissions: { query(params){
if (params?.name === 'notifications') return Promise.resolve({state:"prompt",onchange:null}); var n = params && params.name;
return Promise.resolve({state:"granted"}); // Chrome defaults privacy-sensitive permissions to "prompt", not "granted";
// returning "granted" for camera/microphone is a bot tell.
if (n === 'notifications') return Promise.resolve({state: (globalThis.Notification && Notification.permission === 'granted') ? 'granted' : 'prompt', onchange: null});
if (n === 'geolocation' || n === 'camera' || n === 'microphone' || n === 'midi') return Promise.resolve({state: 'prompt', onchange: null});
return Promise.resolve({state: 'granted', onchange: null});
} }, } },
getBattery() { return Promise.resolve({ charging: _fp('batteryCharging'), chargingTime: _fp('batteryCharging') ? 0 : Infinity, dischargingTime: _fp('batteryCharging') ? Infinity : Math.floor(3600 + _fpRand(250) * 7200), level: _fp('batteryLevel'), addEventListener(){} }); }, getBattery() { return Promise.resolve({ charging: _fp('batteryCharging'), chargingTime: _fp('batteryCharging') ? 0 : Infinity, dischargingTime: _fp('batteryCharging') ? Infinity : Math.floor(3600 + _fpRand(250) * 7200), level: _fp('batteryLevel'), addEventListener(){} }); },
getGamepads() { return []; }, getGamepads() { return []; },
@@ -3003,6 +3046,51 @@ function _installWasmStreamingFallback() {
} }
_installWasmStreamingFallback(); _installWasmStreamingFallback();
// Serialize a FormData into a multipart/form-data body the way a browser does
// when it is passed as fetch()/XHR body. The previous shim did String(body),
// so a FormData became the literal "[object Object]" and the multipart payload
// (with its boundary) was lost; servers replied "Invalid boundary for
// multipart/form-data" (e.g. the AWS WAF challenge /mp_verify POST).
function _formDataToMultipart(fd) {
const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
let bnd = '----WebKitFormBoundary';
for (let i = 0; i < 16; i++) bnd += chars[Math.floor(Math.random() * chars.length)];
let out = '';
const entries = fd._d || [];
for (let i = 0; i < entries.length; i++) {
const k = entries[i][0], v = entries[i][1];
out += '--' + bnd + '\r\n';
if (v != null && typeof v === 'object' && v._bytes != null) {
out += 'Content-Disposition: form-data; name="' + k + '"; filename="' + (v.name || 'blob') + '"\r\n';
out += 'Content-Type: ' + (v.type || 'application/octet-stream') + '\r\n\r\n';
try { out += new TextDecoder().decode(v._bytes); } catch (e) {}
out += '\r\n';
} else {
out += 'Content-Disposition: form-data; name="' + k + '"\r\n\r\n' + String(v) + '\r\n';
}
}
out += '--' + bnd + '--\r\n';
return { boundary: bnd, body: out };
}
// Coerce a fetch()/XHR body into the string op_fetch_url expects, attaching a
// Content-Type header for body types that need one (FormData, URLSearchParams).
function _serializeBody(initBody, headers) {
if (initBody == null || initBody === '') return '';
if (initBody instanceof FormData) {
const mp = _formDataToMultipart(initBody);
headers['Content-Type'] = 'multipart/form-data; boundary=' + mp.boundary;
return mp.body;
}
if (initBody instanceof URLSearchParams) {
if (!Object.keys(headers).some(k => k.toLowerCase() === 'content-type')) {
headers['Content-Type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
}
return initBody.toString();
}
return typeof initBody === 'string' ? initBody : String(initBody);
}
globalThis.fetch = async (input, init = {}) => { globalThis.fetch = async (input, init = {}) => {
let url = typeof input === "string" let url = typeof input === "string"
? input ? input
@@ -3016,8 +3104,9 @@ globalThis.fetch = async (input, init = {}) => {
} catch(e) { /* keep as-is if URL resolution fails */ } } catch(e) { /* keep as-is if URL resolution fails */ }
} }
const method = init.method || (input instanceof Request ? input.method : "GET"); const method = init.method || (input instanceof Request ? input.method : "GET");
const hdrs = JSON.stringify(init.headers instanceof Headers ? Object.fromEntries(init.headers.entries()) : init.headers || {}); let _h = init.headers instanceof Headers ? Object.fromEntries(init.headers.entries()) : (init.headers || {});
const body = init.body ? String(init.body) : ""; const body = _serializeBody(init.body, _h);
const hdrs = JSON.stringify(_h);
const fetchMode = init.mode || (input instanceof Request ? input.mode : "cors"); const fetchMode = init.mode || (input instanceof Request ? input.mode : "cors");
const pageOrigin = (function() { try { const u = new URL(_domParse("document_url") || "about:blank"); return u.origin; } catch(e) { return ""; } })(); const pageOrigin = (function() { try { const u = new URL(_domParse("document_url") || "about:blank"); return u.origin; } catch(e) { return ""; } })();
const raw = await Deno.core.ops.op_fetch_url(url, method, hdrs, body, pageOrigin, fetchMode); const raw = await Deno.core.ops.op_fetch_url(url, method, hdrs, body, pageOrigin, fetchMode);
@@ -6328,11 +6417,25 @@ globalThis.Worker = class Worker {
}; };
globalThis.__blobStore = globalThis.__blobStore || {}; globalThis.__blobStore = globalThis.__blobStore || {};
const _origCreateObjectURL = URL.createObjectURL;
URL.createObjectURL = function(blob) { URL.createObjectURL = function(blob) {
if (blob && typeof blob.text === 'function') { if (blob) {
const id = 'blob:obscura/' + Math.random().toString(36).substring(2); const id = 'blob:obscura/' + Math.random().toString(36).substring(2);
// Store synchronously so a Worker built from the blob URL in the same
// tick sees its source. Blob-URL Worker construction is synchronous in
// real browsers; the previous async blob.text().then() store raced the
// Worker constructor, so new Worker(blobURL) fell through to fetch() and
// failed (net::ERR_FAILED), which broke AWS WAF's proof-of-work worker.
// The obscura Blob materializes _bytes in its constructor; fall back to
// the async text() store only for foreign Blob shims without _bytes.
if (blob._bytes) {
let text = '';
try { text = new TextDecoder().decode(blob._bytes); } catch (e) {}
globalThis.__blobStore[id] = text;
} else if (typeof blob.text === 'function') {
blob.text().then(text => { globalThis.__blobStore[id] = text; }); blob.text().then(text => { globalThis.__blobStore[id] = text; });
} else {
globalThis.__blobStore[id] = '';
}
return id; return id;
} }
return 'blob:obscura/fallback'; return 'blob:obscura/fallback';
@@ -6421,7 +6524,8 @@ if (!globalThis.crypto.subtle) {
globalThis.crypto.subtle = { globalThis.crypto.subtle = {
async digest(algorithm, data) { async digest(algorithm, data) {
const name = (typeof algorithm === 'string' ? algorithm : algorithm?.name || '').toUpperCase().replace('_', '-'); const name = (typeof algorithm === 'string' ? algorithm : algorithm?.name || '').toUpperCase().replace('_', '-');
if (name !== 'SHA-1' && name !== 'SHA-256' && name !== 'SHA-384' && name !== 'SHA-512') { if (name !== 'SHA-1' && name !== 'SHA-256' && name !== 'SHA-384' && name !== 'SHA-512' &&
name !== 'SHA-512/224' && name !== 'SHA-512/256') {
throw new DOMException('Unrecognized algorithm name', 'NotSupportedError'); throw new DOMException('Unrecognized algorithm name', 'NotSupportedError');
} }
let bytes; let bytes;
@@ -6840,6 +6944,10 @@ if (typeof ShadowRoot !== 'undefined' && !ShadowRoot.prototype.elementFromPoint)
globalThis.__obscura_init = function() { globalThis.__obscura_init = function() {
_fpSeed = Date.now() ^ (Math.random() * 0xFFFFFFFF >>> 0); _fpSeed = Date.now() ^ (Math.random() * 0xFFFFFFFF >>> 0);
_fpCache = null; _fpCache = null;
// A real navigation just completed (this runs after set_url), so drop any
// URL a location setter previewed synchronously and let document_url drive
// location.href again, including any redirect target.
globalThis.__virtualUrl = null;
_installWasmStreamingFallback(); _installWasmStreamingFallback();
globalThis.document = new Document(+_dom("document_node_id")); globalThis.document = new Document(+_dom("document_node_id"));
@@ -6868,29 +6976,9 @@ globalThis.__obscura_init = function() {
}; };
globalThis.Notification.permission = "default"; globalThis.Notification.permission = "default";
if (globalThis.__obscura_stealth) { // userAgentData brands and getHighEntropyValues now derive the Chrome
var _stealthBrands = [ // version from navigator.userAgent and read the platform from the page
{brand: "Not:A-Brand", version: "99"}, // globals, so every stealth surface agrees without a per-mode override.
{brand: "Google Chrome", version: "145"},
{brand: "Chromium", version: "145"},
];
globalThis.navigator.userAgentData.brands = _stealthBrands;
globalThis.navigator.userAgentData.getHighEntropyValues = function(hints) {
return Promise.resolve({
architecture: "x86", bitness: "64",
brands: _stealthBrands,
fullVersionList: [
{brand:"Not:A-Brand",version:"99.0.0.0"},
{brand:"Google Chrome",version:"145.0.0.0"},
{brand:"Chromium",version:"145.0.0.0"},
],
mobile: false, model: "",
platform: "Windows",
platformVersion: "10.0.0",
uaFullVersion: "145.0.0.0",
});
};
}
// Hide internals (_*, obscura, Obscura). The set of keys is static at // Hide internals (_*, obscura, Obscura). The set of keys is static at
// snapshot-build time, so we precompute it ONCE below (after this // snapshot-build time, so we precompute it ONCE below (after this
+156 -2
View File
@@ -9,6 +9,8 @@ use deno_core::Extension;
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use obscura_dom::{DomTree, NodeData, NodeId}; use obscura_dom::{DomTree, NodeData, NodeId};
use obscura_net::{CookieJar, ObscuraHttpClient}; use obscura_net::{CookieJar, ObscuraHttpClient};
#[cfg(feature = "stealth")]
use obscura_net::StealthHttpClient;
use tokio::sync::Mutex; use tokio::sync::Mutex;
pub type InterceptCallback = Arc<Mutex<Option<Box<dyn Fn(String, String, String) -> Option<(u16, String, String)> + Send + Sync>>>>; pub type InterceptCallback = Arc<Mutex<Option<Box<dyn Fn(String, String, String) -> Option<(u16, String, String)> + Send + Sync>>>>;
@@ -55,6 +57,11 @@ pub struct ObscuraState {
pub blocked_urls: Vec<String>, pub blocked_urls: Vec<String>,
pub cookie_jar: Option<Arc<CookieJar>>, pub cookie_jar: Option<Arc<CookieJar>>,
pub http_client: Option<Arc<ObscuraHttpClient>>, pub http_client: Option<Arc<ObscuraHttpClient>>,
/// When set (stealth mode), scripted fetch()/XHR is routed through the wreq
/// client so the request carries the Chrome TLS fingerprint and client
/// hints instead of the rustls ClientHello op_fetch_url would otherwise send.
#[cfg(feature = "stealth")]
pub stealth_client: Option<Arc<StealthHttpClient>>,
pub pending_navigation: Option<(String, String, String)>, pub pending_navigation: Option<(String, String, String)>,
pub intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<InterceptedRequest>>, pub intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<InterceptedRequest>>,
pub intercept_counter: u64, pub intercept_counter: u64,
@@ -82,6 +89,8 @@ impl ObscuraState {
blocked_urls: Vec::new(), blocked_urls: Vec::new(),
cookie_jar: None, cookie_jar: None,
http_client: None, http_client: None,
#[cfg(feature = "stealth")]
stealth_client: None,
pending_navigation: None, pending_navigation: None,
intercept_tx: None, intercept_tx: None,
intercept_counter: 0, intercept_counter: 0,
@@ -707,6 +716,34 @@ async fn op_fetch_url(
let custom_headers: std::collections::HashMap<String, String> = let custom_headers: std::collections::HashMap<String, String> =
serde_json::from_str(&headers_json).unwrap_or_default(); serde_json::from_str(&headers_json).unwrap_or_default();
// Stealth mode: route the scripted request through the wreq client so its
// TLS fingerprint and Chrome client hints match the main navigation. The
// rustls ClientHello plus missing client hints that op_fetch_url's reqwest
// path sends otherwise read as a non-browser script to bot managers (the
// AWS WAF challenge verify call, Akamai sensors, etc.).
#[cfg(feature = "stealth")]
{
let stealth = {
let st = state.borrow();
let gs = st.borrow::<SharedState>().clone();
let client = gs.borrow().stealth_client.clone();
client
};
if let Some(stealth) = stealth {
return stealth_fetch_all(
stealth,
url.clone(),
req_method.as_str().to_string(),
custom_headers.clone(),
body.clone(),
page_origin.clone(),
is_cross_origin,
mode.clone(),
)
.await;
}
}
let needs_preflight = is_cross_origin let needs_preflight = is_cross_origin
&& mode == "cors" && mode == "cors"
&& (req_method != reqwest::Method::GET && (req_method != reqwest::Method::GET
@@ -946,6 +983,120 @@ async fn op_fetch_url(
.to_string()) .to_string())
} }
/// Stealth-mode scripted fetch()/XHR: mirrors op_fetch_url's redirect, SSRF,
/// and CORS semantics but sends every hop through the wreq stealth client so
/// the request carries the Chrome TLS fingerprint and client hints. Cookie
/// handling lives inside StealthHttpClient::send_single, which shares the
/// context jar. Response bodies are not mirrored into the CDP
/// Network.getResponseBody buffer here; that is a follow-up for stealth fetches.
#[cfg(feature = "stealth")]
async fn stealth_fetch_all(
stealth: Arc<StealthHttpClient>,
url: String,
method: String,
custom_headers: HashMap<String, String>,
body: String,
page_origin: String,
is_cross_origin: bool,
mode: String,
) -> Result<String, deno_error::JsErrorBox> {
let mut current_url = url.clone();
let mut current_method = method;
let mut current_body = body;
let mut redirects_followed: usize = 0;
let (status, resp_headers, resp_bytes): (u16, HashMap<String, String>, Vec<u8>) = loop {
let parsed_current = match url::Url::parse(&current_url) {
Ok(u) => u,
Err(_) => {
return Ok(serde_json::json!({
"status": 0, "body": "", "url": current_url, "headers": {},
})
.to_string());
}
};
let mut req_headers: HashMap<String, String> = HashMap::new();
if is_cross_origin {
req_headers.insert("origin".to_string(), page_origin.clone());
}
for (k, v) in &custom_headers {
req_headers.insert(k.to_lowercase(), v.clone());
}
let r = stealth
.send_single(&current_method, &parsed_current, &req_headers, &current_body)
.await
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?;
if !(300..400).contains(&r.status) {
break (r.status, r.headers, r.body);
}
let Some(location) = r.headers.get("location").cloned() else {
break (r.status, r.headers, r.body);
};
let next_url = match parsed_current.join(&location) {
Ok(u) => u,
Err(_) => break (r.status, r.headers, r.body),
};
// Re-validate every redirect target against the SSRF policy, matching
// op_fetch_url (GHSA-8v6v-g4rh-jmcm).
if let Err(reason) = validate_fetch_url(&next_url) {
return Ok(serde_json::json!({
"status": 0, "body": "", "url": next_url.to_string(), "headers": {},
"blocked": true,
"error": format!("Redirect to forbidden URL blocked: {}", reason),
})
.to_string());
}
redirects_followed += 1;
if redirects_followed > FETCH_REDIRECT_LIMIT {
return Ok(serde_json::json!({
"status": 0, "body": "", "url": next_url.to_string(), "headers": {},
"blocked": true,
"error": format!("Too many redirects (>{})", FETCH_REDIRECT_LIMIT),
})
.to_string());
}
// Browser semantics: 301/302/303 downgrade to GET with no body.
if r.status == 301 || r.status == 302 || r.status == 303 {
current_method = "GET".to_string();
current_body.clear();
}
current_url = next_url.to_string();
};
if is_cross_origin && mode == "cors" {
let allowed = resp_headers
.get("access-control-allow-origin")
.map(|s| s.as_str())
.unwrap_or("");
if allowed != "*" && allowed != page_origin {
return Ok(serde_json::json!({
"status": 0, "body": "", "url": url, "headers": {},
"corsBlocked": true,
"corsError": format!(
"CORS error: Origin '{}' not in Access-Control-Allow-Origin '{}'",
page_origin, allowed
),
})
.to_string());
}
}
let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
let resp_body_base64 = BASE64.encode(&resp_bytes);
Ok(serde_json::json!({
"status": status,
"body": resp_body,
"bodyBase64": resp_body_base64,
"url": url,
"headers": resp_headers,
})
.to_string())
}
fn glob_match(pattern: &str, url: &str) -> bool { fn glob_match(pattern: &str, url: &str) -> bool {
if pattern == "*" { if pattern == "*" {
return true; return true;
@@ -1107,8 +1258,9 @@ fn op_binding_called(state: &OpState, #[string] name: &str, #[string] payload: &
} }
/// Real WebCrypto `crypto.subtle.digest`. `algorithm` is the SubtleCrypto /// Real WebCrypto `crypto.subtle.digest`. `algorithm` is the SubtleCrypto
/// algorithm name (`SHA-1` / `SHA-256` / `SHA-384` / `SHA-512`). The JS shim /// algorithm name (`SHA-1` / `SHA-256` / `SHA-384` / `SHA-512`, plus the
/// validates the name before calling this op; any other value is unreachable. /// FIPS 180-4 truncated variants `SHA-512/224` and `SHA-512/256`). The JS
/// shim validates the name; any other value is unreachable.
/// Returns the raw digest bytes so the JS shim can hand them back as an ArrayBuffer. /// Returns the raw digest bytes so the JS shim can hand them back as an ArrayBuffer.
#[op2] #[op2]
#[buffer] #[buffer]
@@ -1120,6 +1272,8 @@ fn op_subtle_digest(#[string] algorithm: &str, #[buffer] data: &[u8]) -> Vec<u8>
"SHA-256" => sha2::Sha256::digest(data).to_vec(), "SHA-256" => sha2::Sha256::digest(data).to_vec(),
"SHA-384" => sha2::Sha384::digest(data).to_vec(), "SHA-384" => sha2::Sha384::digest(data).to_vec(),
"SHA-512" => sha2::Sha512::digest(data).to_vec(), "SHA-512" => sha2::Sha512::digest(data).to_vec(),
"SHA-512/224" => sha2::Sha512_224::digest(data).to_vec(),
"SHA-512/256" => sha2::Sha512_256::digest(data).to_vec(),
_ => vec![], _ => vec![],
} }
} }
+7
View File
@@ -153,6 +153,13 @@ impl ObscuraJsRuntime {
self.state.borrow_mut().http_client = Some(client); self.state.borrow_mut().http_client = Some(client);
} }
/// Install the stealth (wreq) HTTP client so scripted fetch()/XHR is routed
/// through it in stealth mode (see op_fetch_url / stealth_fetch_all).
#[cfg(feature = "stealth")]
pub fn set_stealth_client(&self, client: std::sync::Arc<obscura_net::StealthHttpClient>) {
self.state.borrow_mut().stealth_client = Some(client);
}
pub fn set_dom(&self, dom: DomTree) { pub fn set_dom(&self, dom: DomTree) {
self.state.borrow_mut().dom = Some(dom); self.state.borrow_mut().dom = Some(dom);
} }
+61 -33
View File
@@ -273,6 +273,46 @@ pub struct ObscuraHttpClient {
pub allow_private_network: bool, pub allow_private_network: bool,
} }
/// Derive the sec-ch-ua and sec-ch-ua-platform client-hint header values from a
/// User-Agent string, using Chromium's per-major-version GREASE algorithm so
/// the non-stealth HTTP path agrees with navigator.userAgentData instead of
/// shipping a fixed Linux/Chrome-145 hint that contradicts a Windows profile.
fn chrome_client_hints(ua: &str) -> (String, String) {
let major: usize = ua
.split("Chrome/")
.nth(1)
.and_then(|s| s.split('.').next())
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or(145);
const GREASE_CHARS: [char; 11] = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_'];
const GREASE_VER: [&str; 3] = ["8", "99", "24"];
const PERMS: [[usize; 3]; 6] = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]];
let grease_brand = format!(
"Not{}A{}Brand",
GREASE_CHARS[major % 11],
GREASE_CHARS[(major + 1) % 11]
);
let brands = [
(grease_brand, GREASE_VER[major % 3].to_string()),
("Chromium".to_string(), major.to_string()),
("Google Chrome".to_string(), major.to_string()),
];
let p = PERMS[major % 6];
let sec_ch_ua = p
.iter()
.map(|&i| format!("\"{}\";v=\"{}\"", brands[i].0, brands[i].1))
.collect::<Vec<_>>()
.join(", ");
let platform = if ua.contains("Windows NT") {
"\"Windows\""
} else if ua.contains("Macintosh") {
"\"macOS\""
} else {
"\"Linux\""
};
(sec_ch_ua, platform.to_string())
}
impl ObscuraHttpClient { impl ObscuraHttpClient {
pub fn new() -> Self { pub fn new() -> Self {
Self::with_cookie_jar(Arc::new(CookieJar::new())) Self::with_cookie_jar(Arc::new(CookieJar::new()))
@@ -407,50 +447,38 @@ impl ObscuraHttpClient {
} }
let ua = self.user_agent.read().await.clone(); let ua = self.user_agent.read().await.clone();
let (sec_ch_ua, sec_ch_ua_platform) = chrome_client_hints(&ua);
let mut headers = HeaderMap::new(); let mut headers = HeaderMap::new();
// Chrome's top-level navigation header order. (reqwest appends
// accept-encoding/host after these, so accept-encoding lands after
// accept-language rather than before it; the rest matches Chrome.)
headers.insert(
HeaderName::from_static("sec-ch-ua"),
HeaderValue::from_str(&sec_ch_ua)
.unwrap_or_else(|_| HeaderValue::from_static("\"Not:A-Brand\";v=\"99\", \"Google Chrome\";v=\"145\", \"Chromium\";v=\"145\"")),
);
headers.insert(HeaderName::from_static("sec-ch-ua-mobile"), HeaderValue::from_static("?0"));
headers.insert(
HeaderName::from_static("sec-ch-ua-platform"),
HeaderValue::from_str(&sec_ch_ua_platform)
.unwrap_or_else(|_| HeaderValue::from_static("\"Windows\"")),
);
headers.insert(HeaderName::from_static("upgrade-insecure-requests"), HeaderValue::from_static("1"));
headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap_or_else(|_| { headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap_or_else(|_| {
HeaderValue::from_static("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36") HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36")
})); }));
headers.insert( headers.insert(
reqwest::header::ACCEPT, reqwest::header::ACCEPT,
HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"), HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"),
); );
headers.insert(HeaderName::from_static("sec-fetch-site"), HeaderValue::from_static("none"));
headers.insert(HeaderName::from_static("sec-fetch-mode"), HeaderValue::from_static("navigate"));
headers.insert(HeaderName::from_static("sec-fetch-user"), HeaderValue::from_static("?1"));
headers.insert(HeaderName::from_static("sec-fetch-dest"), HeaderValue::from_static("document"));
headers.insert( headers.insert(
reqwest::header::ACCEPT_LANGUAGE, reqwest::header::ACCEPT_LANGUAGE,
HeaderValue::from_static("en-US,en;q=0.9"), HeaderValue::from_static("en-US,en;q=0.9"),
); );
headers.insert(
HeaderName::from_static("sec-ch-ua"),
HeaderValue::from_static("\"Chromium\";v=\"145\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"145\""),
);
headers.insert(
HeaderName::from_static("sec-ch-ua-mobile"),
HeaderValue::from_static("?0"),
);
headers.insert(
HeaderName::from_static("sec-ch-ua-platform"),
HeaderValue::from_static("\"Linux\""),
);
headers.insert(
HeaderName::from_static("sec-fetch-dest"),
HeaderValue::from_static("document"),
);
headers.insert(
HeaderName::from_static("sec-fetch-mode"),
HeaderValue::from_static("navigate"),
);
headers.insert(
HeaderName::from_static("sec-fetch-site"),
HeaderValue::from_static("none"),
);
headers.insert(
HeaderName::from_static("sec-fetch-user"),
HeaderValue::from_static("?1"),
);
headers.insert(
HeaderName::from_static("upgrade-insecure-requests"),
HeaderValue::from_static("1"),
);
let cookie_header = self.cookie_jar.get_cookie_header(&current_url); let cookie_header = self.cookie_jar.get_cookie_header(&current_url);
tracing::debug!( tracing::debug!(
+4 -1
View File
@@ -19,4 +19,7 @@ pub use encoding::{
pub use robots::RobotsCache; pub use robots::RobotsCache;
pub use blocklist::is_blocked as is_tracker_blocked; pub use blocklist::is_blocked as is_tracker_blocked;
#[cfg(feature = "stealth")] #[cfg(feature = "stealth")]
pub use wreq_client::{StealthHttpClient, STEALTH_USER_AGENT}; pub use wreq_client::{
StealthHttpClient, STEALTH_NAVIGATOR_PLATFORM, STEALTH_UA_PLATFORM,
STEALTH_UA_PLATFORM_VERSION, STEALTH_USER_AGENT,
};
+92 -2
View File
@@ -19,7 +19,18 @@ use crate::client::{Response, ObscuraNetError};
#[cfg(feature = "stealth")] #[cfg(feature = "stealth")]
pub const STEALTH_USER_AGENT: &str = pub const STEALTH_USER_AGENT: &str =
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
// The wreq emulation (Profile::Chrome145, Platform::Windows) sends this exact
// UA and sec-ch-ua-platform "Windows" on the wire. navigator has to report the
// same identity, otherwise the TLS/HTTP layer and the JS layer disagree and a
// site cross-checks the mismatch as a bot signal.
#[cfg(feature = "stealth")]
pub const STEALTH_NAVIGATOR_PLATFORM: &str = "Win32";
#[cfg(feature = "stealth")]
pub const STEALTH_UA_PLATFORM: &str = "Windows";
#[cfg(feature = "stealth")]
pub const STEALTH_UA_PLATFORM_VERSION: &str = "15.0.0";
#[cfg(feature = "stealth")] #[cfg(feature = "stealth")]
pub struct StealthHttpClient { pub struct StealthHttpClient {
@@ -38,7 +49,7 @@ impl StealthHttpClient {
pub fn with_proxy(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self { pub fn with_proxy(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
let emulation_opts = wreq_util::Emulation::builder() let emulation_opts = wreq_util::Emulation::builder()
.profile(wreq_util::Profile::Chrome145) .profile(wreq_util::Profile::Chrome145)
.platform(wreq_util::Platform::Linux) .platform(wreq_util::Platform::Windows)
.build(); .build();
let mut builder = wreq::Client::builder() let mut builder = wreq::Client::builder()
@@ -143,6 +154,85 @@ impl StealthHttpClient {
Err(ObscuraNetError::TooManyRedirects(url.to_string())) Err(ObscuraNetError::TooManyRedirects(url.to_string()))
} }
/// One request with no redirect following, for scripted fetch()/XHR. Reads
/// the cookie jar for the Cookie header and stores Set-Cookie back into it,
/// so the caller only owns redirect hops and SSRF re-validation. Used in
/// stealth mode so JS-level requests carry the same Chrome TLS fingerprint
/// and client hints as the main navigation instead of the rustls ClientHello
/// that op_fetch_url would otherwise send (which bot managers read as a
/// non-browser script and reject, e.g. the AWS WAF challenge verify call).
pub async fn send_single(
&self,
method: &str,
url: &Url,
headers: &HashMap<String, String>,
body: &str,
) -> Result<Response, ObscuraNetError> {
if let Some(host) = url.host_str() {
if crate::blocklist::is_blocked(host) {
tracing::debug!("Blocked tracker: {}", url);
return Ok(Response {
status: 0,
url: url.clone(),
headers: HashMap::new(),
body: Vec::new(),
redirected_from: Vec::new(),
});
}
}
let req_method = method
.parse::<wreq::Method>()
.map_err(|e| ObscuraNetError::Network(format!("invalid method '{}': {}", method, e)))?;
let mut req = self.client.request(req_method, url.as_str());
let cookie_header = self.cookie_jar.get_cookie_header(url);
if !cookie_header.is_empty() {
req = req.header("cookie", &cookie_header);
}
for (k, v) in self.extra_headers.read().await.iter() {
req = req.header(k.as_str(), v.as_str());
}
for (k, v) in headers.iter() {
req = req.header(k.as_str(), v.as_str());
}
if !body.is_empty() {
req = req.body(body.to_string());
}
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let resp = req.send().await.map_err(|e| {
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
ObscuraNetError::Network(format!("{}: {}", url, e))
})?;
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
let status = resp.status();
for val in resp.headers().get_all("set-cookie") {
if let Ok(s) = val.to_str() {
self.cookie_jar.set_cookie(s, url);
}
}
let response_headers: HashMap<String, String> = resp
.headers()
.iter()
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
.collect();
let resp_body = resp
.bytes()
.await
.map_err(|e| ObscuraNetError::Network(format!("Failed to read body: {}", e)))?
.to_vec();
Ok(Response {
url: url.clone(),
status: status.as_u16(),
headers: response_headers,
body: resp_body,
redirected_from: Vec::new(),
})
}
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) { pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
*self.extra_headers.write().await = headers; *self.extra_headers.write().await = headers;
} }