Merge stealth-overhaul into main
Resolved conflicts: - page.rs: keep both set_blocked_urls (#315) and stealth client wiring - ops.rs: keep SHA-512/224 + SHA-512/256 from stealth branch; keep the unreachable catch-all (vec![]) from #314 instead of silent SHA-256 fallback; updated comment accordingly - bootstrap.js: extend the digest algorithm allowlist to include SHA-512/224 and SHA-512/256 to match the Rust additions
This commit is contained in:
commit
8582fcceca
9 files changed
+471
-88
No files matched your search
@@ -5,7 +5,7 @@ edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
stealth = ["obscura-net/stealth"]
|
||||
stealth = ["obscura-net/stealth", "obscura-js/stealth"]
|
||||
|
||||
[dependencies]
|
||||
obscura-dom = { workspace = true }
|
||||
|
||||
@@ -286,9 +286,11 @@ impl Page {
|
||||
if self.stealth_client.is_some() {
|
||||
rt.set_stealth(true);
|
||||
rt.set_user_agent(obscura_net::STEALTH_USER_AGENT);
|
||||
// Match platform to the Linux UA so navigator.platform and
|
||||
// userAgentData.platform are consistent with navigator.userAgent.
|
||||
rt.set_platform("Linux x86_64", "Linux", "");
|
||||
rt.set_platform(
|
||||
obscura_net::STEALTH_NAVIGATOR_PLATFORM,
|
||||
obscura_net::STEALTH_UA_PLATFORM,
|
||||
obscura_net::STEALTH_UA_PLATFORM_VERSION,
|
||||
);
|
||||
} else {
|
||||
if let Ok(ua) = self.http_client.user_agent.try_read() {
|
||||
rt.set_user_agent(&ua);
|
||||
@@ -317,6 +319,10 @@ impl Page {
|
||||
rt.set_cookie_jar(self.context.cookie_jar.clone());
|
||||
rt.set_http_client(self.http_client.clone());
|
||||
rt.set_blocked_urls(self.blocked_url_patterns.clone());
|
||||
#[cfg(feature = "stealth")]
|
||||
if let Some(ref stealth) = self.stealth_client {
|
||||
rt.set_stealth_client(stealth.clone());
|
||||
}
|
||||
|
||||
if let Some(tx) = &self.intercept_tx {
|
||||
rt.set_intercept_tx(tx.clone());
|
||||
|
||||
@@ -3,6 +3,13 @@ name = "obscura-js"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Builds obscura-net with its stealth (wreq) HTTP client and lets scripted
|
||||
# fetch()/XHR be routed through it so JS-level requests carry the same Chrome
|
||||
# TLS fingerprint and client hints as stealth navigation.
|
||||
stealth = ["obscura-net/stealth"]
|
||||
|
||||
[build-dependencies]
|
||||
deno_core = "0.350"
|
||||
|
||||
|
||||
Vendored
+134
-46
@@ -154,7 +154,9 @@ function _fpNoise(x, y, channel) {
|
||||
var _fpCache = null;
|
||||
function _getFp() {
|
||||
if (_fpCache) return _fpCache;
|
||||
const isMac = (globalThis.__obscura_ua_platform || 'Windows') === 'macOS';
|
||||
const _uaPlat = globalThis.__obscura_ua_platform || 'Windows';
|
||||
const isMac = _uaPlat === 'macOS';
|
||||
const isLinux = _uaPlat === 'Linux';
|
||||
const gpuPool = isMac ? [
|
||||
'ANGLE (Apple, ANGLE Metal Renderer: Apple M1, Unspecified Version)',
|
||||
'ANGLE (Apple, ANGLE Metal Renderer: Apple M1 Pro, Unspecified Version)',
|
||||
@@ -162,6 +164,14 @@ function _getFp() {
|
||||
'ANGLE (Apple, ANGLE Metal Renderer: Apple M2 Pro, Unspecified Version)',
|
||||
'ANGLE (Apple, ANGLE Metal Renderer: Apple M3, Unspecified Version)',
|
||||
'ANGLE (Intel Inc., ANGLE Metal Renderer: Intel(R) Iris(TM) Plus Graphics, Unspecified Version)',
|
||||
] : isLinux ? [
|
||||
'ANGLE (Intel, Mesa Intel(R) UHD Graphics 630 (CFL GT2), OpenGL 4.6)',
|
||||
'ANGLE (Intel, Mesa Intel(R) Iris(R) Xe Graphics (TGL GT2), OpenGL 4.6)',
|
||||
'ANGLE (Intel, Mesa Intel(R) UHD Graphics 770 (RPL-S), OpenGL 4.6)',
|
||||
'ANGLE (AMD, AMD Radeon RX 580 (polaris10, LLVM 15.0.7, DRM 3.54, LLVM 15.0.7), OpenGL 4.6)',
|
||||
'ANGLE (AMD, AMD Radeon RX 6700 XT (navi22, LLVM 16.0.6, DRM 3.54, LLVM 16.0.6), OpenGL 4.6)',
|
||||
'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 OpenGL 4.6)',
|
||||
'ANGLE (NVIDIA, NVIDIA GeForce RTX 4070 OpenGL 4.6)',
|
||||
] : [
|
||||
'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)',
|
||||
'ANGLE (NVIDIA, NVIDIA GeForce GTX 1660 SUPER Direct3D11 vs_5_0 ps_5_0, D3D11)',
|
||||
@@ -180,6 +190,10 @@ function _getFp() {
|
||||
'Google Inc. (Apple)','Google Inc. (Apple)','Google Inc. (Apple)',
|
||||
'Google Inc. (Apple)','Google Inc. (Apple)',
|
||||
'Google Inc. (Intel Inc.)',
|
||||
] : isLinux ? [
|
||||
'Google Inc. (Intel)','Google Inc. (Intel)','Google Inc. (Intel)',
|
||||
'Google Inc. (AMD)','Google Inc. (AMD)',
|
||||
'Google Inc. (NVIDIA)','Google Inc. (NVIDIA)',
|
||||
] : [
|
||||
'Google Inc. (NVIDIA)','Google Inc. (NVIDIA)','Google Inc. (NVIDIA)',
|
||||
'Google Inc. (Intel)','Google Inc. (Intel)',
|
||||
@@ -2577,7 +2591,7 @@ function __currentUrl() {
|
||||
}
|
||||
globalThis.location = {
|
||||
get href() { return __currentUrl(); },
|
||||
set href(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); },
|
||||
set href(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
|
||||
get origin() { try { return new URL(this.href).origin; } catch { return ""; } },
|
||||
get protocol() { try { return new URL(this.href).protocol; } catch { return ""; } },
|
||||
get host() { try { return new URL(this.href).host; } catch { return ""; } },
|
||||
@@ -2587,14 +2601,14 @@ globalThis.location = {
|
||||
get hash() { try { return new URL(this.href).hash; } catch { return ""; } },
|
||||
get port() { try { return new URL(this.href).port; } catch { return ""; } },
|
||||
toString() { return this.href; },
|
||||
assign(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); },
|
||||
reload() {},
|
||||
replace(url) { globalThis.__virtualUrl = null; Deno.core.ops.op_navigate(_resolveUrl(url), 'GET', ''); },
|
||||
assign(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
|
||||
reload() { var r = _resolveUrl(this.href); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
|
||||
replace(url) { var r = _resolveUrl(url); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
|
||||
};
|
||||
const _locationObj = globalThis.location;
|
||||
Object.defineProperty(globalThis, 'location', {
|
||||
get() { return _locationObj; },
|
||||
set(url) { Deno.core.ops.op_navigate(_resolveUrl(String(url)), 'GET', ''); },
|
||||
set(url) { var r = _resolveUrl(String(url)); globalThis.__virtualUrl = r; Deno.core.ops.op_navigate(r, 'GET', ''); },
|
||||
configurable: false,
|
||||
enumerable: true,
|
||||
});
|
||||
@@ -2752,8 +2766,35 @@ globalThis.NetworkInformation = NetworkInformation;
|
||||
|
||||
globalThis.ContentIndex = class ContentIndex {};
|
||||
|
||||
function _chromeMajor() {
|
||||
var m = (globalThis.__obscura_ua || '').match(/Chrome\/(\d+)/);
|
||||
return m ? (m[1] | 0) : 145;
|
||||
}
|
||||
// Chromium derives the sec-ch-ua GREASE brand, version, and brand order
|
||||
// deterministically from the Chrome major version
|
||||
// (components/embedder_support/user_agent_utils.cc). Replicating it keeps
|
||||
// sec-ch-ua and userAgentData exact for every profile version rather than
|
||||
// hardcoding one static token.
|
||||
var _GREASE_CHARS = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_'];
|
||||
var _GREASE_VER = ['8', '99', '24'];
|
||||
var _BRAND_PERMS = [[0,1,2],[0,2,1],[1,0,2],[1,2,0],[2,0,1],[2,1,0]];
|
||||
function _uaBrands() {
|
||||
var seed = _chromeMajor();
|
||||
var grease = {
|
||||
brand: 'Not' + _GREASE_CHARS[seed % 11] + 'A' + _GREASE_CHARS[(seed + 1) % 11] + 'Brand',
|
||||
version: _GREASE_VER[seed % 3],
|
||||
};
|
||||
var ordered = [
|
||||
grease,
|
||||
{brand: 'Chromium', version: String(seed)},
|
||||
{brand: 'Google Chrome', version: String(seed)},
|
||||
];
|
||||
var p = _BRAND_PERMS[seed % 6];
|
||||
return [ordered[p[0]], ordered[p[1]], ordered[p[2]]];
|
||||
}
|
||||
|
||||
globalThis.navigator = {
|
||||
get userAgent() { return globalThis.__obscura_ua || "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; },
|
||||
get userAgent() { return globalThis.__obscura_ua || "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"; },
|
||||
get appVersion() { return this.userAgent.replace('Mozilla/', ''); },
|
||||
language: "en-US", languages: ["en-US","en"], get platform() { return globalThis.__obscura_platform || "Win32"; },
|
||||
onLine: true, cookieEnabled: true, hardwareConcurrency: 8,
|
||||
@@ -2779,24 +2820,22 @@ globalThis.navigator = {
|
||||
]);
|
||||
},
|
||||
userAgentData: {
|
||||
brands: [
|
||||
{brand: "Google Chrome", version: "145"},
|
||||
{brand: "Chromium", version: "145"},
|
||||
{brand: "Not;A=Brand", version: "24"},
|
||||
],
|
||||
mobile: false,
|
||||
get brands() { return _uaBrands(); },
|
||||
get platform() { return globalThis.__obscura_ua_platform || "Windows"; },
|
||||
getHighEntropyValues(hints) {
|
||||
var brands = _uaBrands();
|
||||
return Promise.resolve({
|
||||
architecture: "x86",
|
||||
bitness: "64",
|
||||
brands: [{brand:"Google Chrome",version:"145"},{brand:"Chromium",version:"145"},{brand:"Not;A=Brand",version:"24"}],
|
||||
fullVersionList: [{brand:"Google Chrome",version:"145.0.0.0"},{brand:"Chromium",version:"145.0.0.0"},{brand:"Not;A=Brand",version:"24.0.0.0"}],
|
||||
brands: brands,
|
||||
fullVersionList: brands.map(function(b) { return {brand: b.brand, version: b.version + ".0.0.0"}; }),
|
||||
mobile: false,
|
||||
model: "",
|
||||
platform: globalThis.__obscura_ua_platform || "Windows",
|
||||
platformVersion: globalThis.__obscura_ua_platform_version || "15.0.0",
|
||||
uaFullVersion: "145.0.0.0",
|
||||
uaFullVersion: _chromeMajor() + ".0.0.0",
|
||||
wow64: false,
|
||||
});
|
||||
},
|
||||
toJSON() { return {brands:this.brands,mobile:this.mobile,platform:this.platform}; },
|
||||
@@ -2817,8 +2856,12 @@ globalThis.navigator = {
|
||||
},
|
||||
clipboard: { writeText(){return Promise.resolve();}, readText(){return Promise.resolve("");} },
|
||||
permissions: { query(params){
|
||||
if (params?.name === 'notifications') return Promise.resolve({state:"prompt",onchange:null});
|
||||
return Promise.resolve({state:"granted"});
|
||||
var n = params && params.name;
|
||||
// Chrome defaults privacy-sensitive permissions to "prompt", not "granted";
|
||||
// returning "granted" for camera/microphone is a bot tell.
|
||||
if (n === 'notifications') return Promise.resolve({state: (globalThis.Notification && Notification.permission === 'granted') ? 'granted' : 'prompt', onchange: null});
|
||||
if (n === 'geolocation' || n === 'camera' || n === 'microphone' || n === 'midi') return Promise.resolve({state: 'prompt', onchange: null});
|
||||
return Promise.resolve({state: 'granted', onchange: null});
|
||||
} },
|
||||
getBattery() { return Promise.resolve({ charging: _fp('batteryCharging'), chargingTime: _fp('batteryCharging') ? 0 : Infinity, dischargingTime: _fp('batteryCharging') ? Infinity : Math.floor(3600 + _fpRand(250) * 7200), level: _fp('batteryLevel'), addEventListener(){} }); },
|
||||
getGamepads() { return []; },
|
||||
@@ -3003,6 +3046,51 @@ function _installWasmStreamingFallback() {
|
||||
}
|
||||
_installWasmStreamingFallback();
|
||||
|
||||
// Serialize a FormData into a multipart/form-data body the way a browser does
|
||||
// when it is passed as fetch()/XHR body. The previous shim did String(body),
|
||||
// so a FormData became the literal "[object Object]" and the multipart payload
|
||||
// (with its boundary) was lost; servers replied "Invalid boundary for
|
||||
// multipart/form-data" (e.g. the AWS WAF challenge /mp_verify POST).
|
||||
function _formDataToMultipart(fd) {
|
||||
const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
|
||||
let bnd = '----WebKitFormBoundary';
|
||||
for (let i = 0; i < 16; i++) bnd += chars[Math.floor(Math.random() * chars.length)];
|
||||
let out = '';
|
||||
const entries = fd._d || [];
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
const k = entries[i][0], v = entries[i][1];
|
||||
out += '--' + bnd + '\r\n';
|
||||
if (v != null && typeof v === 'object' && v._bytes != null) {
|
||||
out += 'Content-Disposition: form-data; name="' + k + '"; filename="' + (v.name || 'blob') + '"\r\n';
|
||||
out += 'Content-Type: ' + (v.type || 'application/octet-stream') + '\r\n\r\n';
|
||||
try { out += new TextDecoder().decode(v._bytes); } catch (e) {}
|
||||
out += '\r\n';
|
||||
} else {
|
||||
out += 'Content-Disposition: form-data; name="' + k + '"\r\n\r\n' + String(v) + '\r\n';
|
||||
}
|
||||
}
|
||||
out += '--' + bnd + '--\r\n';
|
||||
return { boundary: bnd, body: out };
|
||||
}
|
||||
|
||||
// Coerce a fetch()/XHR body into the string op_fetch_url expects, attaching a
|
||||
// Content-Type header for body types that need one (FormData, URLSearchParams).
|
||||
function _serializeBody(initBody, headers) {
|
||||
if (initBody == null || initBody === '') return '';
|
||||
if (initBody instanceof FormData) {
|
||||
const mp = _formDataToMultipart(initBody);
|
||||
headers['Content-Type'] = 'multipart/form-data; boundary=' + mp.boundary;
|
||||
return mp.body;
|
||||
}
|
||||
if (initBody instanceof URLSearchParams) {
|
||||
if (!Object.keys(headers).some(k => k.toLowerCase() === 'content-type')) {
|
||||
headers['Content-Type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
|
||||
}
|
||||
return initBody.toString();
|
||||
}
|
||||
return typeof initBody === 'string' ? initBody : String(initBody);
|
||||
}
|
||||
|
||||
globalThis.fetch = async (input, init = {}) => {
|
||||
let url = typeof input === "string"
|
||||
? input
|
||||
@@ -3016,8 +3104,9 @@ globalThis.fetch = async (input, init = {}) => {
|
||||
} catch(e) { /* keep as-is if URL resolution fails */ }
|
||||
}
|
||||
const method = init.method || (input instanceof Request ? input.method : "GET");
|
||||
const hdrs = JSON.stringify(init.headers instanceof Headers ? Object.fromEntries(init.headers.entries()) : init.headers || {});
|
||||
const body = init.body ? String(init.body) : "";
|
||||
let _h = init.headers instanceof Headers ? Object.fromEntries(init.headers.entries()) : (init.headers || {});
|
||||
const body = _serializeBody(init.body, _h);
|
||||
const hdrs = JSON.stringify(_h);
|
||||
const fetchMode = init.mode || (input instanceof Request ? input.mode : "cors");
|
||||
const pageOrigin = (function() { try { const u = new URL(_domParse("document_url") || "about:blank"); return u.origin; } catch(e) { return ""; } })();
|
||||
const raw = await Deno.core.ops.op_fetch_url(url, method, hdrs, body, pageOrigin, fetchMode);
|
||||
@@ -6328,11 +6417,25 @@ globalThis.Worker = class Worker {
|
||||
};
|
||||
|
||||
globalThis.__blobStore = globalThis.__blobStore || {};
|
||||
const _origCreateObjectURL = URL.createObjectURL;
|
||||
URL.createObjectURL = function(blob) {
|
||||
if (blob && typeof blob.text === 'function') {
|
||||
if (blob) {
|
||||
const id = 'blob:obscura/' + Math.random().toString(36).substring(2);
|
||||
blob.text().then(text => { globalThis.__blobStore[id] = text; });
|
||||
// Store synchronously so a Worker built from the blob URL in the same
|
||||
// tick sees its source. Blob-URL Worker construction is synchronous in
|
||||
// real browsers; the previous async blob.text().then() store raced the
|
||||
// Worker constructor, so new Worker(blobURL) fell through to fetch() and
|
||||
// failed (net::ERR_FAILED), which broke AWS WAF's proof-of-work worker.
|
||||
// The obscura Blob materializes _bytes in its constructor; fall back to
|
||||
// the async text() store only for foreign Blob shims without _bytes.
|
||||
if (blob._bytes) {
|
||||
let text = '';
|
||||
try { text = new TextDecoder().decode(blob._bytes); } catch (e) {}
|
||||
globalThis.__blobStore[id] = text;
|
||||
} else if (typeof blob.text === 'function') {
|
||||
blob.text().then(text => { globalThis.__blobStore[id] = text; });
|
||||
} else {
|
||||
globalThis.__blobStore[id] = '';
|
||||
}
|
||||
return id;
|
||||
}
|
||||
return 'blob:obscura/fallback';
|
||||
@@ -6421,7 +6524,8 @@ if (!globalThis.crypto.subtle) {
|
||||
globalThis.crypto.subtle = {
|
||||
async digest(algorithm, data) {
|
||||
const name = (typeof algorithm === 'string' ? algorithm : algorithm?.name || '').toUpperCase().replace('_', '-');
|
||||
if (name !== 'SHA-1' && name !== 'SHA-256' && name !== 'SHA-384' && name !== 'SHA-512') {
|
||||
if (name !== 'SHA-1' && name !== 'SHA-256' && name !== 'SHA-384' && name !== 'SHA-512' &&
|
||||
name !== 'SHA-512/224' && name !== 'SHA-512/256') {
|
||||
throw new DOMException('Unrecognized algorithm name', 'NotSupportedError');
|
||||
}
|
||||
let bytes;
|
||||
@@ -6840,6 +6944,10 @@ if (typeof ShadowRoot !== 'undefined' && !ShadowRoot.prototype.elementFromPoint)
|
||||
globalThis.__obscura_init = function() {
|
||||
_fpSeed = Date.now() ^ (Math.random() * 0xFFFFFFFF >>> 0);
|
||||
_fpCache = null;
|
||||
// A real navigation just completed (this runs after set_url), so drop any
|
||||
// URL a location setter previewed synchronously and let document_url drive
|
||||
// location.href again, including any redirect target.
|
||||
globalThis.__virtualUrl = null;
|
||||
_installWasmStreamingFallback();
|
||||
|
||||
globalThis.document = new Document(+_dom("document_node_id"));
|
||||
@@ -6868,29 +6976,9 @@ globalThis.__obscura_init = function() {
|
||||
};
|
||||
globalThis.Notification.permission = "default";
|
||||
|
||||
if (globalThis.__obscura_stealth) {
|
||||
var _stealthBrands = [
|
||||
{brand: "Not:A-Brand", version: "99"},
|
||||
{brand: "Google Chrome", version: "145"},
|
||||
{brand: "Chromium", version: "145"},
|
||||
];
|
||||
globalThis.navigator.userAgentData.brands = _stealthBrands;
|
||||
globalThis.navigator.userAgentData.getHighEntropyValues = function(hints) {
|
||||
return Promise.resolve({
|
||||
architecture: "x86", bitness: "64",
|
||||
brands: _stealthBrands,
|
||||
fullVersionList: [
|
||||
{brand:"Not:A-Brand",version:"99.0.0.0"},
|
||||
{brand:"Google Chrome",version:"145.0.0.0"},
|
||||
{brand:"Chromium",version:"145.0.0.0"},
|
||||
],
|
||||
mobile: false, model: "",
|
||||
platform: "Windows",
|
||||
platformVersion: "10.0.0",
|
||||
uaFullVersion: "145.0.0.0",
|
||||
});
|
||||
};
|
||||
}
|
||||
// userAgentData brands and getHighEntropyValues now derive the Chrome
|
||||
// version from navigator.userAgent and read the platform from the page
|
||||
// globals, so every stealth surface agrees without a per-mode override.
|
||||
|
||||
// Hide internals (_*, obscura, Obscura). The set of keys is static at
|
||||
// snapshot-build time, so we precompute it ONCE below (after this
|
||||
|
||||
@@ -9,6 +9,8 @@ use deno_core::Extension;
|
||||
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
|
||||
use obscura_dom::{DomTree, NodeData, NodeId};
|
||||
use obscura_net::{CookieJar, ObscuraHttpClient};
|
||||
#[cfg(feature = "stealth")]
|
||||
use obscura_net::StealthHttpClient;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
pub type InterceptCallback = Arc<Mutex<Option<Box<dyn Fn(String, String, String) -> Option<(u16, String, String)> + Send + Sync>>>>;
|
||||
@@ -55,6 +57,11 @@ pub struct ObscuraState {
|
||||
pub blocked_urls: Vec<String>,
|
||||
pub cookie_jar: Option<Arc<CookieJar>>,
|
||||
pub http_client: Option<Arc<ObscuraHttpClient>>,
|
||||
/// When set (stealth mode), scripted fetch()/XHR is routed through the wreq
|
||||
/// client so the request carries the Chrome TLS fingerprint and client
|
||||
/// hints instead of the rustls ClientHello op_fetch_url would otherwise send.
|
||||
#[cfg(feature = "stealth")]
|
||||
pub stealth_client: Option<Arc<StealthHttpClient>>,
|
||||
pub pending_navigation: Option<(String, String, String)>,
|
||||
pub intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<InterceptedRequest>>,
|
||||
pub intercept_counter: u64,
|
||||
@@ -82,6 +89,8 @@ impl ObscuraState {
|
||||
blocked_urls: Vec::new(),
|
||||
cookie_jar: None,
|
||||
http_client: None,
|
||||
#[cfg(feature = "stealth")]
|
||||
stealth_client: None,
|
||||
pending_navigation: None,
|
||||
intercept_tx: None,
|
||||
intercept_counter: 0,
|
||||
@@ -707,6 +716,34 @@ async fn op_fetch_url(
|
||||
let custom_headers: std::collections::HashMap<String, String> =
|
||||
serde_json::from_str(&headers_json).unwrap_or_default();
|
||||
|
||||
// Stealth mode: route the scripted request through the wreq client so its
|
||||
// TLS fingerprint and Chrome client hints match the main navigation. The
|
||||
// rustls ClientHello plus missing client hints that op_fetch_url's reqwest
|
||||
// path sends otherwise read as a non-browser script to bot managers (the
|
||||
// AWS WAF challenge verify call, Akamai sensors, etc.).
|
||||
#[cfg(feature = "stealth")]
|
||||
{
|
||||
let stealth = {
|
||||
let st = state.borrow();
|
||||
let gs = st.borrow::<SharedState>().clone();
|
||||
let client = gs.borrow().stealth_client.clone();
|
||||
client
|
||||
};
|
||||
if let Some(stealth) = stealth {
|
||||
return stealth_fetch_all(
|
||||
stealth,
|
||||
url.clone(),
|
||||
req_method.as_str().to_string(),
|
||||
custom_headers.clone(),
|
||||
body.clone(),
|
||||
page_origin.clone(),
|
||||
is_cross_origin,
|
||||
mode.clone(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
let needs_preflight = is_cross_origin
|
||||
&& mode == "cors"
|
||||
&& (req_method != reqwest::Method::GET
|
||||
@@ -946,6 +983,120 @@ async fn op_fetch_url(
|
||||
.to_string())
|
||||
}
|
||||
|
||||
/// Stealth-mode scripted fetch()/XHR: mirrors op_fetch_url's redirect, SSRF,
|
||||
/// and CORS semantics but sends every hop through the wreq stealth client so
|
||||
/// the request carries the Chrome TLS fingerprint and client hints. Cookie
|
||||
/// handling lives inside StealthHttpClient::send_single, which shares the
|
||||
/// context jar. Response bodies are not mirrored into the CDP
|
||||
/// Network.getResponseBody buffer here; that is a follow-up for stealth fetches.
|
||||
#[cfg(feature = "stealth")]
|
||||
async fn stealth_fetch_all(
|
||||
stealth: Arc<StealthHttpClient>,
|
||||
url: String,
|
||||
method: String,
|
||||
custom_headers: HashMap<String, String>,
|
||||
body: String,
|
||||
page_origin: String,
|
||||
is_cross_origin: bool,
|
||||
mode: String,
|
||||
) -> Result<String, deno_error::JsErrorBox> {
|
||||
let mut current_url = url.clone();
|
||||
let mut current_method = method;
|
||||
let mut current_body = body;
|
||||
let mut redirects_followed: usize = 0;
|
||||
|
||||
let (status, resp_headers, resp_bytes): (u16, HashMap<String, String>, Vec<u8>) = loop {
|
||||
let parsed_current = match url::Url::parse(¤t_url) {
|
||||
Ok(u) => u,
|
||||
Err(_) => {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0, "body": "", "url": current_url, "headers": {},
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
};
|
||||
|
||||
let mut req_headers: HashMap<String, String> = HashMap::new();
|
||||
if is_cross_origin {
|
||||
req_headers.insert("origin".to_string(), page_origin.clone());
|
||||
}
|
||||
for (k, v) in &custom_headers {
|
||||
req_headers.insert(k.to_lowercase(), v.clone());
|
||||
}
|
||||
|
||||
let r = stealth
|
||||
.send_single(¤t_method, &parsed_current, &req_headers, ¤t_body)
|
||||
.await
|
||||
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?;
|
||||
|
||||
if !(300..400).contains(&r.status) {
|
||||
break (r.status, r.headers, r.body);
|
||||
}
|
||||
let Some(location) = r.headers.get("location").cloned() else {
|
||||
break (r.status, r.headers, r.body);
|
||||
};
|
||||
let next_url = match parsed_current.join(&location) {
|
||||
Ok(u) => u,
|
||||
Err(_) => break (r.status, r.headers, r.body),
|
||||
};
|
||||
// Re-validate every redirect target against the SSRF policy, matching
|
||||
// op_fetch_url (GHSA-8v6v-g4rh-jmcm).
|
||||
if let Err(reason) = validate_fetch_url(&next_url) {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0, "body": "", "url": next_url.to_string(), "headers": {},
|
||||
"blocked": true,
|
||||
"error": format!("Redirect to forbidden URL blocked: {}", reason),
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
redirects_followed += 1;
|
||||
if redirects_followed > FETCH_REDIRECT_LIMIT {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0, "body": "", "url": next_url.to_string(), "headers": {},
|
||||
"blocked": true,
|
||||
"error": format!("Too many redirects (>{})", FETCH_REDIRECT_LIMIT),
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
// Browser semantics: 301/302/303 downgrade to GET with no body.
|
||||
if r.status == 301 || r.status == 302 || r.status == 303 {
|
||||
current_method = "GET".to_string();
|
||||
current_body.clear();
|
||||
}
|
||||
current_url = next_url.to_string();
|
||||
};
|
||||
|
||||
if is_cross_origin && mode == "cors" {
|
||||
let allowed = resp_headers
|
||||
.get("access-control-allow-origin")
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or("");
|
||||
if allowed != "*" && allowed != page_origin {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0, "body": "", "url": url, "headers": {},
|
||||
"corsBlocked": true,
|
||||
"corsError": format!(
|
||||
"CORS error: Origin '{}' not in Access-Control-Allow-Origin '{}'",
|
||||
page_origin, allowed
|
||||
),
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
|
||||
let resp_body_base64 = BASE64.encode(&resp_bytes);
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"status": status,
|
||||
"body": resp_body,
|
||||
"bodyBase64": resp_body_base64,
|
||||
"url": url,
|
||||
"headers": resp_headers,
|
||||
})
|
||||
.to_string())
|
||||
}
|
||||
|
||||
fn glob_match(pattern: &str, url: &str) -> bool {
|
||||
if pattern == "*" {
|
||||
return true;
|
||||
@@ -1107,8 +1258,9 @@ fn op_binding_called(state: &OpState, #[string] name: &str, #[string] payload: &
|
||||
}
|
||||
|
||||
/// Real WebCrypto `crypto.subtle.digest`. `algorithm` is the SubtleCrypto
|
||||
/// algorithm name (`SHA-1` / `SHA-256` / `SHA-384` / `SHA-512`). The JS shim
|
||||
/// validates the name before calling this op; any other value is unreachable.
|
||||
/// algorithm name (`SHA-1` / `SHA-256` / `SHA-384` / `SHA-512`, plus the
|
||||
/// FIPS 180-4 truncated variants `SHA-512/224` and `SHA-512/256`). The JS
|
||||
/// shim validates the name; any other value is unreachable.
|
||||
/// Returns the raw digest bytes so the JS shim can hand them back as an ArrayBuffer.
|
||||
#[op2]
|
||||
#[buffer]
|
||||
@@ -1120,6 +1272,8 @@ fn op_subtle_digest(#[string] algorithm: &str, #[buffer] data: &[u8]) -> Vec<u8>
|
||||
"SHA-256" => sha2::Sha256::digest(data).to_vec(),
|
||||
"SHA-384" => sha2::Sha384::digest(data).to_vec(),
|
||||
"SHA-512" => sha2::Sha512::digest(data).to_vec(),
|
||||
"SHA-512/224" => sha2::Sha512_224::digest(data).to_vec(),
|
||||
"SHA-512/256" => sha2::Sha512_256::digest(data).to_vec(),
|
||||
_ => vec![],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,6 +153,13 @@ impl ObscuraJsRuntime {
|
||||
self.state.borrow_mut().http_client = Some(client);
|
||||
}
|
||||
|
||||
/// Install the stealth (wreq) HTTP client so scripted fetch()/XHR is routed
|
||||
/// through it in stealth mode (see op_fetch_url / stealth_fetch_all).
|
||||
#[cfg(feature = "stealth")]
|
||||
pub fn set_stealth_client(&self, client: std::sync::Arc<obscura_net::StealthHttpClient>) {
|
||||
self.state.borrow_mut().stealth_client = Some(client);
|
||||
}
|
||||
|
||||
pub fn set_dom(&self, dom: DomTree) {
|
||||
self.state.borrow_mut().dom = Some(dom);
|
||||
}
|
||||
|
||||
@@ -273,6 +273,46 @@ pub struct ObscuraHttpClient {
|
||||
pub allow_private_network: bool,
|
||||
}
|
||||
|
||||
/// Derive the sec-ch-ua and sec-ch-ua-platform client-hint header values from a
|
||||
/// User-Agent string, using Chromium's per-major-version GREASE algorithm so
|
||||
/// the non-stealth HTTP path agrees with navigator.userAgentData instead of
|
||||
/// shipping a fixed Linux/Chrome-145 hint that contradicts a Windows profile.
|
||||
fn chrome_client_hints(ua: &str) -> (String, String) {
|
||||
let major: usize = ua
|
||||
.split("Chrome/")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split('.').next())
|
||||
.and_then(|s| s.parse::<usize>().ok())
|
||||
.unwrap_or(145);
|
||||
const GREASE_CHARS: [char; 11] = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_'];
|
||||
const GREASE_VER: [&str; 3] = ["8", "99", "24"];
|
||||
const PERMS: [[usize; 3]; 6] = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]];
|
||||
let grease_brand = format!(
|
||||
"Not{}A{}Brand",
|
||||
GREASE_CHARS[major % 11],
|
||||
GREASE_CHARS[(major + 1) % 11]
|
||||
);
|
||||
let brands = [
|
||||
(grease_brand, GREASE_VER[major % 3].to_string()),
|
||||
("Chromium".to_string(), major.to_string()),
|
||||
("Google Chrome".to_string(), major.to_string()),
|
||||
];
|
||||
let p = PERMS[major % 6];
|
||||
let sec_ch_ua = p
|
||||
.iter()
|
||||
.map(|&i| format!("\"{}\";v=\"{}\"", brands[i].0, brands[i].1))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
let platform = if ua.contains("Windows NT") {
|
||||
"\"Windows\""
|
||||
} else if ua.contains("Macintosh") {
|
||||
"\"macOS\""
|
||||
} else {
|
||||
"\"Linux\""
|
||||
};
|
||||
(sec_ch_ua, platform.to_string())
|
||||
}
|
||||
|
||||
impl ObscuraHttpClient {
|
||||
pub fn new() -> Self {
|
||||
Self::with_cookie_jar(Arc::new(CookieJar::new()))
|
||||
@@ -407,50 +447,38 @@ impl ObscuraHttpClient {
|
||||
}
|
||||
|
||||
let ua = self.user_agent.read().await.clone();
|
||||
let (sec_ch_ua, sec_ch_ua_platform) = chrome_client_hints(&ua);
|
||||
let mut headers = HeaderMap::new();
|
||||
// Chrome's top-level navigation header order. (reqwest appends
|
||||
// accept-encoding/host after these, so accept-encoding lands after
|
||||
// accept-language rather than before it; the rest matches Chrome.)
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua"),
|
||||
HeaderValue::from_str(&sec_ch_ua)
|
||||
.unwrap_or_else(|_| HeaderValue::from_static("\"Not:A-Brand\";v=\"99\", \"Google Chrome\";v=\"145\", \"Chromium\";v=\"145\"")),
|
||||
);
|
||||
headers.insert(HeaderName::from_static("sec-ch-ua-mobile"), HeaderValue::from_static("?0"));
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua-platform"),
|
||||
HeaderValue::from_str(&sec_ch_ua_platform)
|
||||
.unwrap_or_else(|_| HeaderValue::from_static("\"Windows\"")),
|
||||
);
|
||||
headers.insert(HeaderName::from_static("upgrade-insecure-requests"), HeaderValue::from_static("1"));
|
||||
headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap_or_else(|_| {
|
||||
HeaderValue::from_static("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36")
|
||||
HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36")
|
||||
}));
|
||||
headers.insert(
|
||||
reqwest::header::ACCEPT,
|
||||
HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"),
|
||||
);
|
||||
headers.insert(HeaderName::from_static("sec-fetch-site"), HeaderValue::from_static("none"));
|
||||
headers.insert(HeaderName::from_static("sec-fetch-mode"), HeaderValue::from_static("navigate"));
|
||||
headers.insert(HeaderName::from_static("sec-fetch-user"), HeaderValue::from_static("?1"));
|
||||
headers.insert(HeaderName::from_static("sec-fetch-dest"), HeaderValue::from_static("document"));
|
||||
headers.insert(
|
||||
reqwest::header::ACCEPT_LANGUAGE,
|
||||
HeaderValue::from_static("en-US,en;q=0.9"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua"),
|
||||
HeaderValue::from_static("\"Chromium\";v=\"145\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"145\""),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua-mobile"),
|
||||
HeaderValue::from_static("?0"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua-platform"),
|
||||
HeaderValue::from_static("\"Linux\""),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-dest"),
|
||||
HeaderValue::from_static("document"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-mode"),
|
||||
HeaderValue::from_static("navigate"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-site"),
|
||||
HeaderValue::from_static("none"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-user"),
|
||||
HeaderValue::from_static("?1"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("upgrade-insecure-requests"),
|
||||
HeaderValue::from_static("1"),
|
||||
);
|
||||
|
||||
let cookie_header = self.cookie_jar.get_cookie_header(¤t_url);
|
||||
tracing::debug!(
|
||||
|
||||
@@ -19,4 +19,7 @@ pub use encoding::{
|
||||
pub use robots::RobotsCache;
|
||||
pub use blocklist::is_blocked as is_tracker_blocked;
|
||||
#[cfg(feature = "stealth")]
|
||||
pub use wreq_client::{StealthHttpClient, STEALTH_USER_AGENT};
|
||||
pub use wreq_client::{
|
||||
StealthHttpClient, STEALTH_NAVIGATOR_PLATFORM, STEALTH_UA_PLATFORM,
|
||||
STEALTH_UA_PLATFORM_VERSION, STEALTH_USER_AGENT,
|
||||
};
|
||||
@@ -19,7 +19,18 @@ use crate::client::{Response, ObscuraNetError};
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
pub const STEALTH_USER_AGENT: &str =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
|
||||
|
||||
// The wreq emulation (Profile::Chrome145, Platform::Windows) sends this exact
|
||||
// UA and sec-ch-ua-platform "Windows" on the wire. navigator has to report the
|
||||
// same identity, otherwise the TLS/HTTP layer and the JS layer disagree and a
|
||||
// site cross-checks the mismatch as a bot signal.
|
||||
#[cfg(feature = "stealth")]
|
||||
pub const STEALTH_NAVIGATOR_PLATFORM: &str = "Win32";
|
||||
#[cfg(feature = "stealth")]
|
||||
pub const STEALTH_UA_PLATFORM: &str = "Windows";
|
||||
#[cfg(feature = "stealth")]
|
||||
pub const STEALTH_UA_PLATFORM_VERSION: &str = "15.0.0";
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
pub struct StealthHttpClient {
|
||||
@@ -38,7 +49,7 @@ impl StealthHttpClient {
|
||||
pub fn with_proxy(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
|
||||
let emulation_opts = wreq_util::Emulation::builder()
|
||||
.profile(wreq_util::Profile::Chrome145)
|
||||
.platform(wreq_util::Platform::Linux)
|
||||
.platform(wreq_util::Platform::Windows)
|
||||
.build();
|
||||
|
||||
let mut builder = wreq::Client::builder()
|
||||
@@ -143,6 +154,85 @@ impl StealthHttpClient {
|
||||
Err(ObscuraNetError::TooManyRedirects(url.to_string()))
|
||||
}
|
||||
|
||||
/// One request with no redirect following, for scripted fetch()/XHR. Reads
|
||||
/// the cookie jar for the Cookie header and stores Set-Cookie back into it,
|
||||
/// so the caller only owns redirect hops and SSRF re-validation. Used in
|
||||
/// stealth mode so JS-level requests carry the same Chrome TLS fingerprint
|
||||
/// and client hints as the main navigation instead of the rustls ClientHello
|
||||
/// that op_fetch_url would otherwise send (which bot managers read as a
|
||||
/// non-browser script and reject, e.g. the AWS WAF challenge verify call).
|
||||
pub async fn send_single(
|
||||
&self,
|
||||
method: &str,
|
||||
url: &Url,
|
||||
headers: &HashMap<String, String>,
|
||||
body: &str,
|
||||
) -> Result<Response, ObscuraNetError> {
|
||||
if let Some(host) = url.host_str() {
|
||||
if crate::blocklist::is_blocked(host) {
|
||||
tracing::debug!("Blocked tracker: {}", url);
|
||||
return Ok(Response {
|
||||
status: 0,
|
||||
url: url.clone(),
|
||||
headers: HashMap::new(),
|
||||
body: Vec::new(),
|
||||
redirected_from: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let req_method = method
|
||||
.parse::<wreq::Method>()
|
||||
.map_err(|e| ObscuraNetError::Network(format!("invalid method '{}': {}", method, e)))?;
|
||||
let mut req = self.client.request(req_method, url.as_str());
|
||||
|
||||
let cookie_header = self.cookie_jar.get_cookie_header(url);
|
||||
if !cookie_header.is_empty() {
|
||||
req = req.header("cookie", &cookie_header);
|
||||
}
|
||||
for (k, v) in self.extra_headers.read().await.iter() {
|
||||
req = req.header(k.as_str(), v.as_str());
|
||||
}
|
||||
for (k, v) in headers.iter() {
|
||||
req = req.header(k.as_str(), v.as_str());
|
||||
}
|
||||
if !body.is_empty() {
|
||||
req = req.body(body.to_string());
|
||||
}
|
||||
|
||||
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let resp = req.send().await.map_err(|e| {
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
ObscuraNetError::Network(format!("{}: {}", url, e))
|
||||
})?;
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let status = resp.status();
|
||||
for val in resp.headers().get_all("set-cookie") {
|
||||
if let Ok(s) = val.to_str() {
|
||||
self.cookie_jar.set_cookie(s, url);
|
||||
}
|
||||
}
|
||||
let response_headers: HashMap<String, String> = resp
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
let resp_body = resp
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| ObscuraNetError::Network(format!("Failed to read body: {}", e)))?
|
||||
.to_vec();
|
||||
|
||||
Ok(Response {
|
||||
url: url.clone(),
|
||||
status: status.as_u16(),
|
||||
headers: response_headers,
|
||||
body: resp_body,
|
||||
redirected_from: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
|
||||
*self.extra_headers.write().await = headers;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user