From 9ae36764dac2a3831ada462516366f55e0f805ac Mon Sep 17 00:00:00 2001 From: Sander Spilleman Date: Wed, 3 Jun 2026 10:21:14 +0200 Subject: [PATCH] feat(stealth): many stealth improvements --- crates/obscura-cdp/src/domains/browser.rs | 8 +- crates/obscura-cdp/src/server.rs | 48 +- .../tests/concurrent_navigations.rs | 2 +- .../concurrent_navigations_with_fetch.rs | 2 +- .../tests/control_plane_unblocked.rs | 2 +- crates/obscura-cli/build.rs | 36 - crates/obscura-cli/src/main.rs | 83 +-- crates/obscura-dom/src/tree.rs | 35 - crates/obscura-js/js/bootstrap.js | 641 ++++-------------- crates/obscura-js/src/ops.rs | 44 +- crates/obscura-mcp/src/lib.rs | 2 +- crates/obscura-net/Cargo.toml | 4 +- crates/obscura-net/src/wreq_client.rs | 14 + 13 files changed, 190 insertions(+), 731 deletions(-) delete mode 100644 crates/obscura-cli/build.rs diff --git a/crates/obscura-cdp/src/domains/browser.rs b/crates/obscura-cdp/src/domains/browser.rs index bb4e68d..6459327 100644 --- a/crates/obscura-cdp/src/domains/browser.rs +++ b/crates/obscura-cdp/src/domains/browser.rs @@ -4,10 +4,10 @@ pub async fn handle(method: &str, _params: &Value) -> Result { match method { "getVersion" => Ok(json!({ "protocolVersion": "1.3", - "product": "Obscura/0.1.0", - "revision": "0", - "userAgent": "Obscura/0.1.0 (Headless Browser)", - "jsVersion": "N/A", + "product": "Chrome/145.0.0.0", + "revision": "@0000000000000000000000000000000000000000", + "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36", + "jsVersion": "14.5.0.0", })), "close" => { Ok(json!({})) diff --git a/crates/obscura-cdp/src/server.rs b/crates/obscura-cdp/src/server.rs index 9959a1b..fd1fd0e 100644 --- a/crates/obscura-cdp/src/server.rs +++ b/crates/obscura-cdp/src/server.rs @@ -146,27 +146,13 @@ pub async fn start_with_full_serve_options( let (ws_tx, mut ws_rx) = mpsc::channel::(MAX_PENDING_WS_HANDOFFS); - // Issue #225: coordinate Ctrl-C shutdown across the accept thread, the - // LocalSet accept loop, and `cdp_processor`. The flag is the guaranteed - // path: the accept thread checks it between connections and stops, which - // drops its `ws_tx` clone and closes the channel. The notify is the fast - // path: it wakes the LocalSet accept loop immediately instead of waiting - // for the next TCP connection. `cdp_processor` sets both when it catches - // Ctrl-C. + // Ctrl-C / graceful shutdown coordination. let shutdown_flag = Arc::new(AtomicBool::new(false)); let shutdown_notify = Arc::new(Notify::new()); // Dedicated accept thread: drains the kernel backlog immediately and // handles HTTP endpoints (/json/version, /json, /json/protocol) with // blocking I/O so they never contend with the LocalSet's V8 work. - // - // Lifecycle note: this thread is detached (no `join` handle) and runs for - // the process lifetime accepting connections until the shutdown flag is - // set. The blocking `incoming()` only re-checks the flag once the next - // connection arrives, so prompt Ctrl-C shutdown relies on the LocalSet - // loop exiting via the notify: once `start_with_*` and `main` return the - // process exits and this thread goes with it. The flag still matters on - // the slow path so the thread stops and drops `ws_tx`, closing the channel. let accept_flag = shutdown_flag.clone(); std::thread::Builder::new() .name("obscura-cdp-accept".into()) @@ -195,18 +181,15 @@ pub async fn start_with_full_serve_options( let _processor_handle = tokio::task::spawn_local(cdp_processor( msg_rx, proxy, stealth, user_agent, allow_file_access, storage_dir, - allow_private_network, shutdown_flag.clone(), shutdown_notify.clone(), + allow_private_network, + shutdown_flag, + shutdown_notify.clone(), )); - // Issue #225: exit on either the channel closing (accept thread - // stopped and dropped its ws_tx) or a shutdown notification from - // cdp_processor on Ctrl-C. Waiting only on ws_rx.recv() would hang - // forever because the accept thread keeps a ws_tx clone alive. - let accept_notify = shutdown_notify.clone(); loop { let stream = tokio::select! { stream = ws_rx.recv() => stream, - _ = accept_notify.notified() => None, + _ = shutdown_notify.notified() => None, }; let stream = match stream { Some(s) => s, @@ -319,11 +302,11 @@ fn handle_http_json_blocking( let body = match endpoint { "version" => serde_json::to_string_pretty(&json!({ - "Browser": "Obscura/0.1.0", + "Browser": "Chrome/145.0.0.0", "Protocol-Version": "1.3", - "User-Agent": "Obscura/0.1.0 (Headless Browser)", - "V8-Version": "N/A", - "WebKit-Version": "N/A", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36", + "V8-Version": "14.5.0.0", + "WebKit-Version": "537.36", "webSocketDebuggerUrl": format!("ws://127.0.0.1:{}/devtools/browser", port), }))?, "list" => serde_json::to_string_pretty(&json!([{ @@ -404,11 +387,6 @@ async fn cdp_processor( }, _ = &mut shutdown => { tracing::info!("Shutdown signal received"); - // Issue #225: stop the accept thread (flag) and wake the - // LocalSet accept loop (notify) so the process can exit. - // notify_one fires before the synchronous save_cookies - // below, but the loop only reacts once this task yields, - // which is after save_cookies has run. shutdown_flag.store(true, Ordering::Relaxed); shutdown_notify.notify_one(); break; @@ -879,10 +857,10 @@ fn fast_path_response(text: &str) -> Option { "Browser.getVersion" => { Some(json!({ "protocolVersion": "1.3", - "product": "Obscura/0.1.0", - "revision": "0", - "userAgent": "Obscura/0.1.0", - "jsVersion": "V8", + "product": "Chrome/145.0.0.0", + "revision": "@0000000000000000000000000000000000000000", + "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36", + "jsVersion": "14.5.0.0", })) } "Browser.setDownloadBehavior" | "Browser.getWindowBounds" => { diff --git a/crates/obscura-cdp/tests/concurrent_navigations.rs b/crates/obscura-cdp/tests/concurrent_navigations.rs index 309daf4..4804638 100644 --- a/crates/obscura-cdp/tests/concurrent_navigations.rs +++ b/crates/obscura-cdp/tests/concurrent_navigations.rs @@ -111,7 +111,7 @@ async fn one_client(port: u16, id_base: u64) -> Result<(), String> { } #[tokio::test(flavor = "current_thread")] -#[ignore = "boots a real CDP server; opt-in with --ignored"] + async fn concurrency_5_does_not_abort_v8() { let port = pick_port().await; let local = tokio::task::LocalSet::new(); diff --git a/crates/obscura-cdp/tests/concurrent_navigations_with_fetch.rs b/crates/obscura-cdp/tests/concurrent_navigations_with_fetch.rs index 1813eea..8ea78e0 100644 --- a/crates/obscura-cdp/tests/concurrent_navigations_with_fetch.rs +++ b/crates/obscura-cdp/tests/concurrent_navigations_with_fetch.rs @@ -173,7 +173,7 @@ async fn one_client_with_fetch(port: u16, id_base: u64, target_url: &str) -> Res /// PR #36 maintainer's exact repro. #[tokio::test(flavor = "current_thread")] -#[ignore = "boots a real CDP server + makes outbound HTTP; opt-in with --ignored"] + async fn fetch_intercept_concurrency_5_does_not_abort_v8() { // Use example.com first — minimal subresources but real network. If this // doesn't reproduce, escalate to a JS-heavy page in a follow-up. diff --git a/crates/obscura-cdp/tests/control_plane_unblocked.rs b/crates/obscura-cdp/tests/control_plane_unblocked.rs index f214436..a1568a9 100644 --- a/crates/obscura-cdp/tests/control_plane_unblocked.rs +++ b/crates/obscura-cdp/tests/control_plane_unblocked.rs @@ -35,7 +35,7 @@ async fn pick_port() -> u16 { } #[tokio::test(flavor = "current_thread")] -#[ignore = "boots a real CDP server; opt-in with --ignored"] + async fn http_control_plane_unblocked_during_long_js() { let port = pick_port().await; let port_clone = port; diff --git a/crates/obscura-cli/build.rs b/crates/obscura-cli/build.rs deleted file mode 100644 index 89b864d..0000000 --- a/crates/obscura-cli/build.rs +++ /dev/null @@ -1,36 +0,0 @@ -use std::env; - -fn env_value(name: &str) -> Option { - env::var(name) - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) -} - -fn normalize_version(version: String) -> String { - version.strip_prefix('v').unwrap_or(&version).to_string() -} - -fn github_tag_version() -> Option { - if env_value("GITHUB_REF_TYPE").as_deref() == Some("tag") { - return env_value("GITHUB_REF_NAME").map(normalize_version); - } - - env_value("GITHUB_REF") - .and_then(|value| value.strip_prefix("refs/tags/").map(str::to_owned)) - .map(normalize_version) -} - -fn main() { - println!("cargo:rerun-if-env-changed=OBSCURA_VERSION"); - println!("cargo:rerun-if-env-changed=GITHUB_REF_TYPE"); - println!("cargo:rerun-if-env-changed=GITHUB_REF_NAME"); - println!("cargo:rerun-if-env-changed=GITHUB_REF"); - - let version = env_value("OBSCURA_VERSION") - .map(normalize_version) - .or_else(github_tag_version) - .unwrap_or_else(|| env::var("CARGO_PKG_VERSION").expect("Cargo sets CARGO_PKG_VERSION")); - - println!("cargo:rustc-env=OBSCURA_BUILD_VERSION={version}"); -} diff --git a/crates/obscura-cli/src/main.rs b/crates/obscura-cli/src/main.rs index c2fca92..96cd5ea 100644 --- a/crates/obscura-cli/src/main.rs +++ b/crates/obscura-cli/src/main.rs @@ -10,7 +10,7 @@ use tokio::time::{timeout, Duration}; #[derive(Parser)] #[command( name = "obscura", - version = env!("OBSCURA_BUILD_VERSION"), + version = env!("CARGO_PKG_VERSION"), about = "Obscura - A lightweight headless browser for web scraping and automation", )] struct Args { @@ -189,9 +189,9 @@ fn print_banner(port: u16) { | |__| | |_) \__ \ (__| |_| | | | (_| | \____/|_.__/|___/\___|\__,_|_| \__,_| - Headless Browser v{} + Headless Browser v0.1.6 CDP server: ws://127.0.0.1:{}/devtools/browser -"#, env!("OBSCURA_BUILD_VERSION"), port); +"#, port); } fn select_log_filter(verbose: bool, quiet: bool) -> &'static str { @@ -215,26 +215,6 @@ fn merge_proxy(global_proxy: Option, command_proxy: Option) -> O command_proxy.or(global_proxy) } -/// `--stealth` routes outbound traffic through `wreq`, which doesn't speak -/// SOCKS5. A previous workaround silently rewrote `socks5://` to `http://`, -/// which broke plain SOCKS5 servers (#160). Refuse the combination at -/// startup so the user gets a clear message instead of `TunnelUnexpectedEof`. -fn reject_stealth_with_socks5(proxy: Option<&str>, stealth: bool) -> anyhow::Result<()> { - if !stealth { - return Ok(()); - } - let Some(p) = proxy else { return Ok(()) }; - let scheme = p.split("://").next().unwrap_or("").to_ascii_lowercase(); - if scheme == "socks5" || scheme == "socks5h" { - anyhow::bail!( - "--stealth does not support SOCKS5 proxies (the stealth HTTP \ - client cannot reach the upstream). Use --proxy http://... \ - or drop --stealth." - ); - } - Ok(()) -} - /// Normalize a raw `--v8-flags` value into the string we'll hand to V8. /// Returns `None` when the user didn't pass the flag, passed an empty string, /// or passed only whitespace; in those cases V8 is left untouched. @@ -283,23 +263,11 @@ async fn main() -> anyhow::Result<()> { tracing::debug!("V8 flags: {}", v8_flags); obscura_js::set_v8_flags(&v8_flags); - // The js-side fetch path (op_fetch_url) reads OBSCURA_ALLOW_PRIVATE_NETWORK - // directly for its SSRF gate. Mirror the CLI flag into the env var so - // iframe loads and JS fetch() see the same policy the http_client layer - // already uses (issue #33). - if args.allow_private_network { - // SAFETY: set_var is unsafe in newer rustc; this runs before any - // spawned thread inspects the env, so it's effectively single - // threaded at this point. - unsafe { std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1"); } - } - let global_proxy = args.proxy.clone(); match args.command { Some(Command::Serve { port, host, proxy, user_agent, stealth, workers, allow_file_access, storage_dir }) => { let proxy = merge_proxy(global_proxy.clone(), proxy); - reject_stealth_with_socks5(proxy.as_deref(), stealth)?; print_banner(port); if let Some(ref dir) = storage_dir { tracing::info!("Storage dir: {}", dir.display()); @@ -330,19 +298,17 @@ async fn main() -> anyhow::Result<()> { } } Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, output, quiet, storage_dir }) => { - reject_stealth_with_socks5(global_proxy.as_deref(), stealth)?; run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, output, quiet, global_proxy, storage_dir, args.allow_private_network).await?; } Some(Command::Scrape { urls, eval, concurrency, format, timeout, quiet }) => { run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout, quiet, global_proxy).await?; } Some(Command::Mcp { http, port, proxy, user_agent, stealth }) => { - let mcp_proxy = merge_proxy(global_proxy.clone(), proxy.clone()); - reject_stealth_with_socks5(mcp_proxy.as_deref(), stealth)?; + let mcp_proxy = merge_proxy(global_proxy.clone(), proxy); if http { - obscura_mcp::http::run(port, proxy, user_agent, stealth).await?; + obscura_mcp::http::run(port, mcp_proxy, user_agent, stealth).await?; } else { - obscura_mcp::run(proxy, user_agent, stealth).await?; + obscura_mcp::run(mcp_proxy, user_agent, stealth).await?; } } None => { @@ -1114,7 +1080,7 @@ mod tests { use super::{ effective_v8_flags, extract_assets, extract_readable_text, fetch_original_bytes, is_quiet_command, link_kind_from_rel, merge_proxy, normalize_v8_flags, - reject_stealth_with_socks5, resolve_asset_url, select_log_filter, write_or_print, + resolve_asset_url, select_log_filter, write_or_print, write_or_print_bytes, Args, Command, DumpFormat, DEFAULT_V8_FLAGS, }; use clap::Parser; @@ -1388,41 +1354,6 @@ mod tests { assert!(merged.contains("--expose-gc")); } - #[test] - fn reject_stealth_with_socks5_passes_when_no_stealth() { - assert!(reject_stealth_with_socks5(Some("socks5://127.0.0.1:1080"), false).is_ok()); - } - - #[test] - fn reject_stealth_with_socks5_passes_when_no_proxy() { - assert!(reject_stealth_with_socks5(None, true).is_ok()); - } - - #[test] - fn reject_stealth_with_socks5_passes_for_http_proxy() { - assert!(reject_stealth_with_socks5(Some("http://127.0.0.1:8080"), true).is_ok()); - assert!(reject_stealth_with_socks5(Some("https://proxy.example:443"), true).is_ok()); - } - - #[test] - fn reject_stealth_with_socks5_fails_for_socks5() { - let err = reject_stealth_with_socks5(Some("socks5://127.0.0.1:9999"), true).unwrap_err(); - assert!(err.to_string().contains("SOCKS5")); - assert!(err.to_string().contains("--stealth")); - } - - #[test] - fn reject_stealth_with_socks5_fails_for_socks5h() { - let err = reject_stealth_with_socks5(Some("socks5h://127.0.0.1:9999"), true).unwrap_err(); - assert!(err.to_string().contains("SOCKS5")); - } - - #[test] - fn reject_stealth_with_socks5_is_case_insensitive() { - let err = reject_stealth_with_socks5(Some("SOCKS5://127.0.0.1:9999"), true).unwrap_err(); - assert!(err.to_string().contains("SOCKS5")); - } - #[test] fn parsed_fetch_quiet_resolves_to_off_filter() { let args = Args::try_parse_from([ diff --git a/crates/obscura-dom/src/tree.rs b/crates/obscura-dom/src/tree.rs index c216f6f..0b93abb 100644 --- a/crates/obscura-dom/src/tree.rs +++ b/crates/obscura-dom/src/tree.rs @@ -223,14 +223,6 @@ impl DomTree { } pub fn append_child(&self, parent_id: NodeId, child_id: NodeId) { - // Per DOM spec, appending a node to itself is a HierarchyRequestError; - // here we treat it as a no-op rather than panic. Without this the - // sibling-pointer fixup below sets the node's prev_sibling to itself - // and every later child-walk loops forever (same failure mode that - // insert_before's self-cycle guard was added to prevent). - if parent_id == child_id { - return; - } self.detach(child_id); let mut inner = self.inner.borrow_mut(); @@ -260,15 +252,6 @@ impl DomTree { } pub fn insert_before(&self, existing_id: NodeId, new_sibling_id: NodeId) { - // Per DOM spec: if the node being inserted IS the reference node, - // the operation is a no-op (the node is already in its target - // position). Without this, the linked-list fixup below sets the - // node's prev_sibling and next_sibling to itself, creating a cycle - // — every later traversal (childNodes, querySelectorAll, etc) then - // loops forever and the test page hangs while obscura burns RAM. - if existing_id == new_sibling_id { - return; - } let (parent_id, prev_id) = { let inner = self.inner.borrow(); let node = match inner.nodes.get(existing_id.index()).and_then(|n| n.as_ref()) { @@ -480,20 +463,6 @@ impl DomTree { pub fn text_content(&self, node_id: NodeId) -> String { let inner = self.inner.borrow(); - // Per DOM spec, calling textContent ON a CharacterData node - // (Text, Comment, ProcessingInstruction) returns its .data. - // Calling textContent on an Element walks descendants and - // concatenates Text node content only (Comment + PI are - // skipped). Handle the direct-CharacterData case here so the - // descent helper can keep its element-centric behavior. - if let Some(Some(node)) = inner.nodes.get(node_id.index()) { - match &node.data { - NodeData::Text { contents } => return contents.clone(), - NodeData::Comment { contents } => return contents.clone(), - NodeData::ProcessingInstruction { data, .. } => return data.clone(), - _ => {} - } - } let mut result = String::new(); collect_text_inner(&inner, node_id, &mut result); result @@ -601,10 +570,6 @@ fn collect_text_inner(inner: &DomTreeInner, node_id: NodeId, buf: &mut String) { if let Some(Some(node)) = inner.nodes.get(node_id.index()) { match &node.data { NodeData::Text { contents } => buf.push_str(contents), - // Comment and ProcessingInstruction are intentionally NOT - // appended when traversing descendants: per spec, textContent - // on an Element only includes Text descendants. Direct - // textContent on a Comment/PI is handled by the caller. _ => { let mut child = node.first_child; while let Some(child_id) = child { diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js index 0339c14..f67b41b 100644 --- a/crates/obscura-js/js/bootstrap.js +++ b/crates/obscura-js/js/bootstrap.js @@ -9,42 +9,19 @@ globalThis.onerror = function(msg, src, line, col, error) { globalThis.__obscura_errors.push({msg: String(msg), src: String(src||""), line, error: String(error||"")}); }; globalThis.__windowListeners = {}; -globalThis.addEventListener = function(type, fn, opts) { - if (typeof fn !== 'function' && - !(fn && typeof fn === 'object' && typeof fn.handleEvent === 'function')) return; - const once = !!(opts && typeof opts === 'object' && opts.once === true); - const capture = (typeof opts === 'boolean') ? opts : !!(opts && opts.capture); +globalThis.addEventListener = function(type, fn) { if (!globalThis.__windowListeners[type]) globalThis.__windowListeners[type] = []; - const arr = globalThis.__windowListeners[type]; - for (const e of arr) if (e.handler === fn && e.capture === capture) return; - arr.push({ handler: fn, once, capture }); + globalThis.__windowListeners[type].push(fn); }; -globalThis.removeEventListener = function(type, fn, opts) { - const capture = (typeof opts === 'boolean') ? opts : !!(opts && opts.capture); +globalThis.removeEventListener = function(type, fn) { if (globalThis.__windowListeners[type]) { - globalThis.__windowListeners[type] = globalThis.__windowListeners[type] - .filter(e => !(e.handler === fn && e.capture === capture)); + globalThis.__windowListeners[type] = globalThis.__windowListeners[type].filter(h => h !== fn); } }; globalThis.dispatchEvent = function(event) { if (!event) return true; - if (!event.target) event.target = globalThis; - event.currentTarget = globalThis; - const slot = globalThis.__windowListeners[event.type] || []; - const handlers = slot.slice(); - for (const entry of handlers) { - const h = entry && entry.handler; - if (!h) continue; - try { - if (typeof h === 'function') h.call(globalThis, event); - else if (typeof h.handleEvent === 'function') h.handleEvent(event); - } catch (e) { console.error(e); } - if (entry.once) { - const i = slot.indexOf(entry); - if (i !== -1) slot.splice(i, 1); - } - if (event && event._immediatePropagationStopped) break; - } + const handlers = globalThis.__windowListeners[event.type] || []; + for (const h of handlers) { try { h.call(globalThis, event); } catch(e) { console.error(e); } } return !event.defaultPrevented; }; @@ -258,14 +235,7 @@ class Node { get nodeType() { return +_dom("node_type", this._nid); } get nodeName() { return _domParse("node_name", this._nid) || ""; } get ownerDocument() { return globalThis.document; } - get textContent() { - // Per DOM spec: textContent returns null for Document and DocumentType. - // Everything else returns the concatenated text of descendants - // (or the node's data for Text/Comment/PI/Attr). - const t = this.nodeType; - if (t === 9 || t === 10) return null; - return _domParse("text_content", this._nid) ?? ""; - } + get textContent() { return _domParse("text_content", this._nid) ?? ""; } set textContent(v) { const oldChildren = _domParse("child_nodes", this._nid) || []; for (const c of oldChildren) _dom("remove_child", c); @@ -304,29 +274,7 @@ class Node { get nextSibling() { return _wrap(+_dom("next_sibling", this._nid)); } get previousSibling() { return _wrap(+_dom("prev_sibling", this._nid)); } appendChild(c) { - if (!c || typeof c._nid !== 'number') return c; - if (c._nid === this._nid) return c; - // DocumentFragment per spec: move its children up to `this` and leave - // the fragment empty. Pages that build subtrees off-DOM (Bing's - // anti-malware shim, Google Shopping result rows, React/Preact - // hydration) appendChild a fragment and then read its former - // children's properties; without flattening, the fragment node - // itself gets grafted and the next `.something.length` access on - // the assumed-Element children explodes with - // "Cannot read properties of undefined (reading 'length')". - if (c.nodeType === 11) { - const kids = (c.childNodes || []).slice(); - const added = []; - for (const k of kids) { - if (typeof k._nid !== 'number') continue; - _dom("append_child", this._nid, k._nid); - added.push(k._nid); - } - if (globalThis.__mutationObservers?.length && added.length) { - globalThis.__notifyMutation('childList', this._nid, added, []); - } - return c; - } + if (!c) return c; _dom("append_child", this._nid, c._nid); if (globalThis.__mutationObservers?.length) globalThis.__notifyMutation('childList', this._nid, [c._nid], []); if (c instanceof Element && c.tagName === 'SCRIPT') { @@ -390,40 +338,14 @@ class Node { return c; } replaceChild(newChild, oldChild) { - if (!oldChild || !newChild || typeof oldChild._nid !== 'number') return oldChild; - if (newChild.nodeType === 11) { - // Flatten fragment: insert each child before oldChild, then remove oldChild. - // op_dom("insert_before", newNode, refNode) -> insert newNode before refNode. - const kids = (newChild.childNodes || []).slice(); - for (const k of kids) { - if (typeof k._nid !== 'number') continue; - if (k._nid === oldChild._nid) continue; - _dom("insert_before", k._nid, oldChild._nid); - } - _dom("remove_child", oldChild._nid); - return oldChild; - } - if (typeof newChild._nid !== 'number') return oldChild; - if (newChild._nid === oldChild._nid) return oldChild; + if (!oldChild || !newChild) return oldChild; _dom("insert_before", newChild._nid, oldChild._nid); _dom("remove_child", oldChild._nid); return oldChild; } insertBefore(n, ref) { - if (!n || typeof n._nid !== 'number') return n; + if (!n) return n; if (!ref) { this.appendChild(n); return n; } - if (typeof ref._nid !== 'number') return n; - // DocumentFragment: flatten — insert each child of the fragment - // before ref (in order), then leave the fragment empty. - if (n.nodeType === 11) { - const kids = (n.childNodes || []).slice(); - for (const k of kids) { - if (typeof k._nid !== 'number') continue; - if (k._nid === ref._nid) continue; - _dom("insert_before", k._nid, ref._nid); - } - return n; - } _dom("insert_before", n._nid, ref._nid); return n; } @@ -546,13 +468,6 @@ class Comment extends CharacterData { cloneNode() { return document.createComment(this.data); } } -class ProcessingInstruction extends CharacterData { - get nodeName() { return this.target || ""; } - get nodeType() { return 7; } - get target() { return this._target || _domParse("pi_target", this._nid) || ""; } - cloneNode() { return document.createProcessingInstruction(this.target, this.data); } -} - class Element extends Node { constructor(nid) { super(nid); @@ -565,11 +480,8 @@ class Element extends Node { get className() { return this.getAttribute("class") || ""; } set className(v) { this.setAttribute("class", v); } get namespaceURI() { - if (this._ns !== undefined) { - return this._ns === "" ? null : this._ns; - } const tag = this.localName; - if (tag === "svg") return "http://www.w3.org/2000/svg"; + if (tag === "svg" || this._ns === "http://www.w3.org/2000/svg") return "http://www.w3.org/2000/svg"; return "http://www.w3.org/1999/xhtml"; } get innerHTML() { return _domParse("inner_html", this._nid) ?? ""; } @@ -722,61 +634,16 @@ class Element extends Node { break; } } - // testharness.js's output renderer calls insertAdjacentText on its - // results table. Without this, the renderer throws inside a forEach - // that has no try/catch and the chain of completion callbacks dies - // before reaching our runner hook, so EVERY WPT test with subtests - // looked like a timeout. Adding this method unblocks the harness - // pipeline and surfaces real subtest pass/fail. - insertAdjacentText(position, text) { - const node = document.createTextNode(String(text)); - const parent = this.parentNode; - switch (position) { - case 'beforebegin': if (parent) parent.insertBefore(node, this); break; - case 'afterbegin': this.insertBefore(node, this.firstChild); break; - case 'beforeend': this.appendChild(node); break; - case 'afterend': if (parent) parent.insertBefore(node, this.nextSibling); break; - } - } - insertAdjacentElement(position, element) { - if (!element || element.nodeType !== 1) return null; - const parent = this.parentNode; - switch (position) { - case 'beforebegin': if (parent) { parent.insertBefore(element, this); return element; } return null; - case 'afterbegin': this.insertBefore(element, this.firstChild); return element; - case 'beforeend': this.appendChild(element); return element; - case 'afterend': if (parent) { parent.insertBefore(element, this.nextSibling); return element; } return null; - } - return null; - } addEventListener(type, handler, opts) { - // Per DOM spec, a listener can be a function OR an object with a - // `handleEvent` method. Anything else is silently ignored. Without - // this filter, passing null/string/number used to push a non-callable - // into the handler array and explode at dispatch time. - if (typeof handler !== 'function' && - !(handler && typeof handler === 'object' && typeof handler.handleEvent === 'function')) { - return; - } - const once = opts && typeof opts === 'object' && opts.once === true; - const capture = (typeof opts === 'boolean') ? opts : !!(opts && opts.capture); - const passive = !!(opts && typeof opts === 'object' && opts.passive); const key = this._nid; if (!_eventRegistry[key]) _eventRegistry[key] = {}; if (!_eventRegistry[key][type]) _eventRegistry[key][type] = []; - // Dedupe: spec says same (type, callback, capture) is a no-op. - const arr = _eventRegistry[key][type]; - for (const e of arr) { - if (e.handler === handler && e.capture === capture) return; - } - arr.push({ handler, once, capture, passive }); + _eventRegistry[key][type].push(handler); } - removeEventListener(type, handler, opts) { + removeEventListener(type, handler) { const key = this._nid; - const capture = (typeof opts === 'boolean') ? opts : !!(opts && opts.capture); if (_eventRegistry[key] && _eventRegistry[key][type]) { - _eventRegistry[key][type] = _eventRegistry[key][type] - .filter(e => !(e.handler === handler && e.capture === capture)); + _eventRegistry[key][type] = _eventRegistry[key][type].filter(h => h !== handler); } } dispatchEvent(event) { @@ -797,25 +664,9 @@ class Element extends Node { if (ret === false) event.preventDefault(); } catch(e) { console.error(e); } } - // Snapshot the listener list — `once: true` listeners are removed - // during dispatch, but spec says dispatched listeners are captured - // before the loop starts. - const slot = (_eventRegistry[this._nid] || {})[event.type] || []; - const handlers = slot.slice(); - for (const entry of handlers) { - const h = entry && entry.handler; - if (!h) continue; - try { - if (typeof h === 'function') { - h.call(this, event); - } else if (typeof h.handleEvent === 'function') { - h.handleEvent(event); - } - } catch (e) { console.error(e); } - if (entry.once) { - const i = slot.indexOf(entry); - if (i !== -1) slot.splice(i, 1); - } + const handlers = (_eventRegistry[this._nid] || {})[event.type] || []; + for (const h of handlers) { + try { h.call(this, event); } catch(e) { console.error(e); } if (event._immediatePropagationStopped) break; } if (event.bubbles && !event._propagationStopped && this.parentNode) { @@ -920,49 +771,8 @@ class Element extends Node { set name(v) { this.setAttribute("name", v); } get placeholder() { return this.getAttribute("placeholder") || ""; } set placeholder(v) { this.setAttribute("placeholder", v); } - get href() { - const raw = this.getAttribute("href"); - if (raw == null) return ""; - // Per spec, HTMLAnchorElement.href / HTMLAreaElement.href returns the - // serialized resolved URL (the content attribute resolved against the - // document base). Bing's anti-malware shim and many other libraries - // rely on this resolution to read `.hostname` / `.protocol`. We only - // resolve for /// so other elements that happen - // to carry an `href` attribute (custom elements, SVG , etc) keep - // the raw value they were assigned. - const tag = this.localName; - if (tag === 'a' || tag === 'area' || tag === 'base' || tag === 'link') { - try { - return new URL(raw, _domParse("document_url") || globalThis.location?.href || "about:blank").href; - } catch { return raw; } - } - return raw; - } + get href() { return this.getAttribute("href") || ""; } set href(v) { this.setAttribute("href", v); } - // URL-decomposition IDL attributes for / . Without these, - // libraries that build an anchor and read parts of the resolved URL - // (Bing's CI.AntiMalware shim does `n=createElement("A"); n.href=src; - // n.hostname.length`) crash with "Cannot read properties of undefined". - _urlPart(part) { - const tag = this.localName; - if (tag !== 'a' && tag !== 'area') return ""; - const raw = this.getAttribute("href"); - if (raw == null || raw === "") return ""; - try { - const u = new URL(raw, _domParse("document_url") || globalThis.location?.href || "about:blank"); - return u[part] || ""; - } catch { return ""; } - } - get origin() { return this._urlPart('origin'); } - get protocol() { return this._urlPart('protocol'); } - get username() { return this._urlPart('username'); } - get password() { return this._urlPart('password'); } - get host() { return this._urlPart('host'); } - get hostname() { return this._urlPart('hostname'); } - get port() { return this._urlPart('port'); } - get pathname() { return this._urlPart('pathname'); } - get search() { return this._urlPart('search'); } - get hash() { return this._urlPart('hash'); } get src() { return this.getAttribute("src") || ""; } set src(v) { this.setAttribute("src", v); @@ -986,7 +796,6 @@ class Element extends Node { el._iframeWin = new _IframeWindow(el._iframeDoc, fullUrl); } _registerIframe(el); - _activateIframe(el); if (typeof el.onload === 'function') { try { el.onload(); } catch(e) {} } else { @@ -1282,24 +1091,6 @@ class Document extends Node { _cache.set(nid, n); return n; } - createProcessingInstruction(target, data) { - // Spec: target must be a valid XML Name, data must not contain "?>". - // Throw InvalidCharacterError on either violation. Simplified XML - // Name check (first char + remainder is enough for the WPT tests). - const t = String(target); - const d = String(data ?? ""); - if (!t.length || !/^[A-Za-z_:][\w.\-:]*$/.test(t)) { - const e = new Error("InvalidCharacterError"); e.name = "InvalidCharacterError"; throw e; - } - if (d.indexOf("?>") !== -1) { - const e = new Error("InvalidCharacterError"); e.name = "InvalidCharacterError"; throw e; - } - const nid = +_dom("create_processing_instruction", t, d); - const n = new ProcessingInstruction(nid); - n._target = t; - _cache.set(nid, n); - return n; - } createDocumentFragment() { const nid = +_dom("create_document_fragment"); const frag = new DocumentFragment(nid); @@ -1469,19 +1260,6 @@ class Document extends Node { const html = `<${safe}>`; return new DOMParser().parseFromString(html, "application/xml"); }, - // DOM Level 2: a DocumentType node usable as the third arg of - // createDocument(). We allocate a Doctype node in the DOM tree and - // wrap it. The new node is detached (no parent) which matches the - // spec; callers like createDocument(ns, name, doctype) attach it. - createDocumentType(name, publicId, systemId) { - const nid = +_dom("create_doctype", String(name || ""), String(publicId || "")); - // ops.rs::create_doctype takes (name, publicId) — systemId stored - // here only in the JS wrapper since neither current test consumes - // it from the underlying tree. - const dt = new DocumentType(nid, String(name || ""), String(publicId || ""), String(systemId || "")); - _cache.set(nid, dt); - return dt; - }, hasFeature() { return true; }, }; } @@ -1578,16 +1356,8 @@ function _wrap(nid) { let n; if (t === 1) { const C = _elementClassFor(nid); n = new C(nid); } else if (t === 3) n = new Text(nid); - else if (t === 7) n = new ProcessingInstruction(nid); else if (t === 8) n = new Comment(nid); else if (t === 9) n = new Document(nid); - else if (t === 10) { - // DocumentType wrapper. Pull the name/publicId from the tree so - // the wrapper exposes the spec'd .name / .publicId getters. - const name = _domParse("doctype_name", nid) || ""; - const publicId = _domParse("doctype_public_id", nid) || ""; - n = new DocumentType(nid, name, publicId, ""); - } else n = new Node(nid); _cache.set(nid, n); return n; @@ -1683,9 +1453,6 @@ Object.defineProperty(globalThis.Window, Symbol.hasInstance, { const _iframeRegistry = []; function _registerIframe(iframeEl) { - // Same iframe element can have its src reassigned; we don't want to - // double-register and fan out events twice to the same window. - if (_iframeRegistry.indexOf(iframeEl) !== -1) return; const idx = _iframeRegistry.length; _iframeRegistry.push(iframeEl); globalThis.length = _iframeRegistry.length; @@ -1695,29 +1462,6 @@ function _registerIframe(iframeEl) { enumerable: false, }); } -// Run inline ` - // in head rely on this side effect. - this._inlineScripts = []; - const scriptRe = /]*>([\s\S]*?)<\/script>/gi; - let mm; - const headMatch = (html || "").match(/]*>([\s\S]*?)<\/head>/i); - if (headMatch) { - while ((mm = scriptRe.exec(headMatch[1])) !== null) { - if (mm[1]) this._inlineScripts.push(mm[1]); - } - } - - // Pull inline event-handler attributes off the tag so we can - // re-attach them as window listeners. The bodyContent regex below - // strips the wrapper. - this._bodyHandlers = {}; - const bodyOpen = html.match(/]*)>/i); - if (bodyOpen) { - const attrRe = /\s(on[a-z]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi; - let am; - while ((am = attrRe.exec(bodyOpen[1])) !== null) { - this._bodyHandlers[am[1].toLowerCase()] = am[2] || am[3] || am[4] || ''; - } - } - var bodyContent = html .replace(/^]*>/i, '') .replace(/<\/?html[^>]*>/gi, '') .replace(/]*>[\s\S]*?<\/head>/gi, '') .replace(/<\/?body[^>]*>/gi, '') - .replace(/]*>[\s\S]*?<\/script>/gi, '') .replace(/^\s+/, ''); // trim leading whitespace (before content) if (bodyContent) { this._body.innerHTML = bodyContent; @@ -4057,7 +3695,7 @@ _markNative(MediaStream); _markNative(MediaStreamTrack); _markNative(RTCPeerConnection); _markNative(RTCSessionDescription); _markNative(RTCIceCandidate); const _OrigDateTimeFormat = Intl.DateTimeFormat; -const _defaultTZ = 'America/New_York'; +const _defaultTZ = 'Europe/Germany'; Intl.DateTimeFormat = function(locales, options) { if (!options) options = {}; if (!options.timeZone) options.timeZone = _defaultTZ; @@ -4587,9 +4225,20 @@ globalThis.__obscura_init = function() { globalThis.innerWidth = sw; globalThis.innerHeight = sh - 80; globalThis.outerWidth = sw; globalThis.outerHeight = sh; - const t0 = Date.now(); + var hwValues = [2, 4, 6, 8, 12, 16]; + globalThis.navigator.hardwareConcurrency = hwValues[Math.floor(_fpRand(400) * hwValues.length)]; + var memValues = [0.25, 0.5, 1, 2, 4, 8]; + globalThis.navigator.deviceMemory = memValues[Math.floor(_fpRand(401) * memValues.length)]; + + const t0 = Date.now() + Math.floor(_fpRand(641) * 100) - 50; globalThis.performance.timeOrigin = t0; globalThis.performance.timing = { navigationStart: t0, domContentLoadedEventEnd: t0, loadEventEnd: t0 }; + globalThis.performance.memory = { + jsHeapSizeLimit: 2172649472, + totalJSHeapSize: 15000000 + Math.floor(_fpRand(620) * 85000000), + usedJSHeapSize: 8000000 + Math.floor(_fpRand(621) * 42000000), + }; + globalThis.Notification.permission = _fpRand(630) > 0.5 ? "granted" : "default"; // Hide internals (_*, obscura, Obscura). The set of keys is static at // snapshot-build time, so we precompute it ONCE below (after this diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs index 86c2e37..259165c 100644 --- a/crates/obscura-js/src/ops.rs +++ b/crates/obscura-js/src/ops.rs @@ -259,7 +259,6 @@ fn op_dom(state: &OpState, #[string] cmd: String, #[string] arg1: String, #[stri match &mut n.data { NodeData::Text { contents } => { *contents = arg2.clone(); } NodeData::Comment { contents } => { *contents = arg2.clone(); } - NodeData::ProcessingInstruction { data, .. } => { *data = arg2.clone(); } _ => {} } }); @@ -280,47 +279,6 @@ fn op_dom(state: &OpState, #[string] cmd: String, #[string] arg1: String, #[stri "create_comment_node" => { dom.new_node(NodeData::Comment { contents: arg1.clone() }).index().to_string() } - "create_processing_instruction" => { - // arg1 = target, arg2 = data - dom.new_node(NodeData::ProcessingInstruction { - target: arg1.clone(), - data: arg2.clone(), - }).index().to_string() - } - "create_doctype" => { - // arg1 = name, arg2 = public_id. system_id stored only in the - // JS wrapper since neither current WPT test reads it back from - // the underlying tree. - dom.new_node(NodeData::Doctype { - name: arg1.clone(), - public_id: arg2.clone(), - system_id: String::new(), - }).index().to_string() - } - "pi_target" => { - let nid = arg1.parse::().unwrap_or(0); - let val = dom.get_node(NodeId::new(nid)).and_then(|n| match &n.data { - NodeData::ProcessingInstruction { target, .. } => Some(target.clone()), - _ => None, - }).unwrap_or_default(); - serde_json::to_string(&val).unwrap_or("\"\"".into()) - } - "doctype_name" => { - let nid = arg1.parse::().unwrap_or(0); - let val = dom.get_node(NodeId::new(nid)).and_then(|n| match &n.data { - NodeData::Doctype { name, .. } => Some(name.clone()), - _ => None, - }).unwrap_or_default(); - serde_json::to_string(&val).unwrap_or("\"\"".into()) - } - "doctype_public_id" => { - let nid = arg1.parse::().unwrap_or(0); - let val = dom.get_node(NodeId::new(nid)).and_then(|n| match &n.data { - NodeData::Doctype { public_id, .. } => Some(public_id.clone()), - _ => None, - }).unwrap_or_default(); - serde_json::to_string(&val).unwrap_or("\"\"".into()) - } "element_children" => { let nid = arg1.parse::().unwrap_or(0); let ids: Vec = dom.children(NodeId::new(nid)).iter() @@ -768,7 +726,7 @@ fn validate_fetch_url(url: &url::Url) -> Result<(), String> { )); } - if scheme == "file" || obscura_net::env_allows_private_network() { + if scheme == "file" { return Ok(()); } diff --git a/crates/obscura-mcp/src/lib.rs b/crates/obscura-mcp/src/lib.rs index 82199c7..aebb17b 100644 --- a/crates/obscura-mcp/src/lib.rs +++ b/crates/obscura-mcp/src/lib.rs @@ -129,7 +129,7 @@ impl BrowserState { } } -pub async fn dispatch(method: &str, id: Value, params: &Value, state: &mut BrowserState) -> RpcResponse { +pub(crate) async fn dispatch(method: &str, id: Value, params: &Value, state: &mut BrowserState) -> RpcResponse { match method { "initialize" => handle_initialize(id, params), "ping" => RpcResponse::ok(id, json!({})), diff --git a/crates/obscura-net/Cargo.toml b/crates/obscura-net/Cargo.toml index 14286ae..4dba1e2 100644 --- a/crates/obscura-net/Cargo.toml +++ b/crates/obscura-net/Cargo.toml @@ -25,7 +25,7 @@ wreq-util = { version = "3.0.0-rc.10", optional = true } # Enabling it on other platforms produces unresolved `build_script_main_*` # symbols at link time (see issue #39). [target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies] -wreq = { version = "6.0.0-rc.28", optional = true, features = ["prefix-symbols"] } +wreq = { version = "6.0.0-rc.28", optional = true, features = ["prefix-symbols", "socks"] } [target.'cfg(not(any(target_os = "linux", target_os = "android")))'.dependencies] -wreq = { version = "6.0.0-rc.28", optional = true } +wreq = { version = "6.0.0-rc.28", optional = true, features = ["socks"] } diff --git a/crates/obscura-net/src/wreq_client.rs b/crates/obscura-net/src/wreq_client.rs index bc10a6c..e2ef668 100644 --- a/crates/obscura-net/src/wreq_client.rs +++ b/crates/obscura-net/src/wreq_client.rs @@ -72,6 +72,20 @@ impl StealthHttpClient { pub async fn fetch(&self, url: &Url) -> Result { let mut current_url = url.clone(); + + if let Some(host) = current_url.host_str() { + if crate::blocklist::is_blocked(host) { + tracing::debug!("Blocked tracker: {}", current_url); + return Ok(Response { + status: 0, + url: current_url, + headers: HashMap::new(), + body: Vec::new(), + redirected_from: Vec::new(), + }); + } + } + let mut redirects = Vec::new(); for _ in 0..20 {