diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs index 0a47482..51e75ea 100644 --- a/crates/obscura-browser/src/page.rs +++ b/crates/obscura-browser/src/page.rs @@ -1211,6 +1211,12 @@ impl Page { self.dom.as_ref().map(f) } + /// Absolute URLs the page pulled in via fetch()/XHR (issue #301). Empty + /// when the page has no live JS runtime. + pub fn fetched_urls(&self) -> Vec { + self.js.as_ref().map(|js| js.fetched_urls()).unwrap_or_default() + } + pub fn dom(&self) -> Option<&DomTree> { self.dom.as_ref() } diff --git a/crates/obscura-cdp/src/domains/input.rs b/crates/obscura-cdp/src/domains/input.rs index 1166fde..e633d73 100644 --- a/crates/obscura-cdp/src/domains/input.rs +++ b/crates/obscura-cdp/src/domains/input.rs @@ -23,9 +23,9 @@ pub async fn handle( var target = (document.elementFromPoint && document.elementFromPoint({x},{y})) || globalThis.__obscura_click_target || document.activeElement || document.body;\ if (!target) return;\ globalThis.__obscura_click_target = target;\ - var evt = new MouseEvent('mousedown', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}});\ + var evt = globalThis.__obscura_markTrusted(new MouseEvent('mousedown', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}}));\ target.dispatchEvent(evt);\ - var click = new MouseEvent('click', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}});\ + var click = globalThis.__obscura_markTrusted(new MouseEvent('click', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}}));\ var cancelled = !target.dispatchEvent(click);\ if (!cancelled) {{\ var link = target.closest ? target.closest('a[href]') : null;\ @@ -46,7 +46,7 @@ pub async fn handle( if (form2 && typeof form2.submit === 'function') {{ try {{ form2.submit(target); }} catch(e) {{}} }}\ }} else if (tag === 'INPUT' && (type === 'checkbox' || type === 'radio')) {{\ target.checked = !target.checked;\ - try {{ target.dispatchEvent(new Event('change', {{bubbles:true}})); }} catch(e) {{}}\ + try {{ target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('change', {{bubbles:true}}))); }} catch(e) {{}}\ }}\ }}\ }}\ @@ -62,7 +62,7 @@ pub async fn handle( "(function() {{\ var target = (document.elementFromPoint && document.elementFromPoint({x},{y})) || globalThis.__obscura_click_target || document.activeElement || document.body;\ if (!target) return;\ - var evt = new MouseEvent('mouseup', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}});\ + var evt = globalThis.__obscura_markTrusted(new MouseEvent('mouseup', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}}));\ target.dispatchEvent(evt);\ }})()", x = x, y = y, @@ -85,7 +85,7 @@ pub async fn handle( let js = format!( "(function() {{\ var target = document.activeElement || document.body;\ - var evt = new KeyboardEvent('keydown', {{bubbles:true,cancelable:true,key:'{key}',code:'{code}'}});\ + var evt = globalThis.__obscura_markTrusted(new KeyboardEvent('keydown', {{bubbles:true,cancelable:true,key:'{key}',code:'{code}'}}));\ target.dispatchEvent(evt);\ }})()", key = key.replace('\'', "\\'"), @@ -102,7 +102,7 @@ pub async fn handle( var target = document.activeElement;\ if (target && (target.localName === 'input' || target.localName === 'textarea')) {{\ target.value = (target.value || '') + '{text}';\ - target.dispatchEvent(new Event('input', {{bubbles:true}}));\ + target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {{bubbles:true}})));\ }}\ }})()", text = escaped_text, @@ -118,10 +118,10 @@ pub async fn handle( let js = "(function() {\ var target = document.activeElement;\ if (!target) return;\ - target.dispatchEvent(new KeyboardEvent('keypress', {bubbles:true,key:'Enter',code:'Enter'}));\ + target.dispatchEvent(globalThis.__obscura_markTrusted(new KeyboardEvent('keypress', {bubbles:true,key:'Enter',code:'Enter'})));\ if (target.localName === 'textarea') {\ target.value = (target.value || '') + '\\n';\ - target.dispatchEvent(new Event('input', {bubbles:true}));\ + target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {bubbles:true})));\ } else {\ var form = target.form || (target.closest && target.closest('form'));\ if (form && typeof form.submit === 'function') form.submit();\ @@ -135,7 +135,7 @@ pub async fn handle( var target = document.activeElement;\ if (target && (target.localName === 'input' || target.localName === 'textarea')) {\ target.value = target.value.slice(0, -1);\ - target.dispatchEvent(new Event('input', {bubbles:true}));\ + target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {bubbles:true})));\ }\ })()"; page.evaluate(js); @@ -145,7 +145,7 @@ pub async fn handle( let js = format!( "(function() {{\ var target = document.activeElement || document.body;\ - var evt = new KeyboardEvent('keyup', {{bubbles:true,key:'{key}',code:'{code}'}});\ + var evt = globalThis.__obscura_markTrusted(new KeyboardEvent('keyup', {{bubbles:true,key:'{key}',code:'{code}'}}));\ target.dispatchEvent(evt);\ }})()", key = key.replace('\'', "\\'"), @@ -160,7 +160,7 @@ pub async fn handle( var target = document.activeElement;\ if (target && (target.localName === 'input' || target.localName === 'textarea')) {{\ target.value = (target.value || '') + '{text}';\ - target.dispatchEvent(new Event('input', {{bubbles:true}}));\ + target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {{bubbles:true}})));\ }}\ }})()", text = text.replace('\'', "\\'").replace('\\', "\\\\"), diff --git a/crates/obscura-cli/src/main.rs b/crates/obscura-cli/src/main.rs index e088053..d2ab8dd 100644 --- a/crates/obscura-cli/src/main.rs +++ b/crates/obscura-cli/src/main.rs @@ -1160,7 +1160,30 @@ fn extract_assets(dom: &obscura_dom::DomTree, base_url: Option<&url::Url>) -> St fn dump_assets(page: &Page) -> String { let base_url = page.url.clone(); - page.with_dom(|dom| extract_assets(dom, base_url.as_ref())).unwrap_or_default() + let dom_ndjson = page + .with_dom(|dom| extract_assets(dom, base_url.as_ref())) + .unwrap_or_default(); + + let mut lines: Vec = + dom_ndjson.lines().filter(|l| !l.is_empty()).map(|l| l.to_string()).collect(); + + // URLs already listed from static DOM attributes, so a resource the script + // fetches that the markup also references is not emitted twice. + let mut seen: std::collections::HashSet = lines + .iter() + .filter_map(|l| serde_json::from_str::(l).ok()) + .filter_map(|v| v.get("url").and_then(|u| u.as_str()).map(|s| s.to_string())) + .collect(); + + // Resources pulled in by JS fetch()/XHR, which leave no static DOM tag + // (issue #301). + for url in page.fetched_urls() { + if seen.insert(url.clone()) { + lines.push(serde_json::json!({ "url": url, "type": "fetch" }).to_string()); + } + } + + lines.join("\n") } #[cfg(test)] diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js index bee612e..c1fb240 100644 --- a/crates/obscura-js/js/bootstrap.js +++ b/crates/obscura-js/js/bootstrap.js @@ -14,7 +14,7 @@ '__obscura_errors', '__obscura_init', '__obscura_hide_list', '__obscura_objects', '__obscura_oid', '__obscura_ua', '__obscura_platform', '__obscura_ua_platform', '__obscura_ua_platform_version', - '__obscura_stealth', + '__obscura_stealth', '__obscura_markTrusted', '__documentReadyState__', '__currentUrl', // internal helpers (var-declared throughout the file) '__processDynScriptQueue', '_markNative', '_fpRand', '_fpNoise', @@ -4009,9 +4009,18 @@ globalThis.DOMException = (function () { } return DOMException; })(); +// Per the UI Events spec, only events the user agent dispatches (real or +// automation-synthesized input) are trusted; events page script builds with +// `new Event(...)` must report isTrusted === false (issue #303). Returning true +// for everything is a trivial bot-detection tell. Trusted events are tracked in +// a closure-private WeakSet so page JS can neither read nor forge the flag. +// obscura's CDP input pipeline marks its synthetic events via the +// non-enumerable __obscura_markTrusted helper. +const _trustedEvents = new WeakSet(); +globalThis.__obscura_markTrusted = function(ev) { try { if (ev) _trustedEvents.add(ev); } catch (_e) {} return ev; }; globalThis.Event = class Event { constructor(t,o={}) { this.type=t;this.bubbles=!!o.bubbles;this.cancelable=!!o.cancelable;this.composed=!!o.composed;this.defaultPrevented=false;this.target=null;this.currentTarget=null;this.eventPhase=0;this.timeStamp=Date.now();this._propagationStopped=false;this._immediatePropagationStopped=false; } - get isTrusted() { return true; } + get isTrusted() { return _trustedEvents.has(this); } preventDefault() { if (this.cancelable) this.defaultPrevented=true; } stopPropagation(){ this._propagationStopped=true; } stopImmediatePropagation(){ this._propagationStopped=true; this._immediatePropagationStopped=true; } initEvent(type,bubbles,cancelable) { if (arguments.length < 1) throw new TypeError("Failed to execute 'initEvent' on 'Event': 1 argument required, but only 0 present."); this.type=String(type);this.bubbles=!!bubbles;this.cancelable=!!cancelable;this.defaultPrevented=false;this._propagationStopped=false;this._immediatePropagationStopped=false; } composedPath() { @@ -4023,6 +4032,7 @@ globalThis.Event = class Event { return path; } }; +_markNative(Event); globalThis.CustomEvent = class extends Event { constructor(t,o={}) { super(t,o);this.detail=o.detail; } // Legacy DOM Level 2 init; some libraries (Starbucks China bundle, older diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs index 7fb09f9..1975b72 100644 --- a/crates/obscura-js/src/ops.rs +++ b/crates/obscura-js/src/ops.rs @@ -66,6 +66,10 @@ pub struct ObscuraState { pub network_response_bodies: HashMap, pub network_response_body_order: VecDeque, pub network_response_body_counter: u64, + // Absolute URLs requested via JS fetch() / XHR (op_fetch_url), in request + // order. Surfaced by `--dump assets` so resources pulled in by script, not + // just static DOM attributes, are listed (issue #301). + pub fetched_urls: Vec, } impl ObscuraState { @@ -86,6 +90,7 @@ impl ObscuraState { network_response_bodies: HashMap::new(), network_response_body_order: VecDeque::new(), network_response_body_counter: 0, + fetched_urls: Vec::new(), } } } @@ -620,6 +625,10 @@ async fn op_fetch_url( }).to_string()); } } + // Record the resource the page pulled in via fetch()/XHR so `--dump + // assets` can list it (issue #301). URL is already absolute here, since + // reqwest needs an absolute URL to send the request. + gs.fetched_urls.push(url.clone()); let jar = gs.cookie_jar.clone(); let in_flight = gs.http_client.as_ref().map(|c| c.in_flight.clone()); // #139: thread the configured proxy through to the per-request diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs index e0bdbcc..dcd7c96 100644 --- a/crates/obscura-js/src/runtime.rs +++ b/crates/obscura-js/src/runtime.rs @@ -957,6 +957,12 @@ impl ObscuraJsRuntime { state.dom.as_ref().map(f) } + /// Absolute URLs the page requested via fetch()/XHR, in request order + /// (issue #301). Backs `--dump assets`. + pub fn fetched_urls(&self) -> Vec { + self.state.borrow().fetched_urls.clone() + } + pub fn dom_ref(&self) -> Option>> { let r = self.state.borrow(); if r.dom.is_some() {