Merge branch 'main' of https://github.com/h4ckf0r0day/obscura

crates/obscura-browser/src/page.rs

@@ -1211,6 +1211,12 @@ impl Page {
         self.dom.as_ref().map(f)
     }
 
+    /// Absolute URLs the page pulled in via fetch()/XHR (issue #301). Empty
+    /// when the page has no live JS runtime.
+    pub fn fetched_urls(&self) -> Vec<String> {
+        self.js.as_ref().map(|js| js.fetched_urls()).unwrap_or_default()
+    }
+
     pub fn dom(&self) -> Option<&DomTree> {
         self.dom.as_ref()
     }

crates/obscura-cdp/src/domains/input.rs

@@ -23,9 +23,9 @@ pub async fn handle(
                             var target = (document.elementFromPoint && document.elementFromPoint({x},{y})) || globalThis.__obscura_click_target || document.activeElement || document.body;\
                             if (!target) return;\
                             globalThis.__obscura_click_target = target;\
-                            var evt = new MouseEvent('mousedown', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}});\
+                            var evt = globalThis.__obscura_markTrusted(new MouseEvent('mousedown', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}}));\
                             target.dispatchEvent(evt);\
-                            var click = new MouseEvent('click', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}});\
+                            var click = globalThis.__obscura_markTrusted(new MouseEvent('click', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}}));\
                             var cancelled = !target.dispatchEvent(click);\
                             if (!cancelled) {{\
                                 var link = target.closest ? target.closest('a[href]') : null;\
@@ -46,7 +46,7 @@ pub async fn handle(
                                         if (form2 && typeof form2.submit === 'function') {{ try {{ form2.submit(target); }} catch(e) {{}} }}\
                                     }} else if (tag === 'INPUT' && (type === 'checkbox' || type === 'radio')) {{\
                                         target.checked = !target.checked;\
-                                        try {{ target.dispatchEvent(new Event('change', {{bubbles:true}})); }} catch(e) {{}}\
+                                        try {{ target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('change', {{bubbles:true}}))); }} catch(e) {{}}\
                                     }}\
                                 }}\
                             }}\
@@ -62,7 +62,7 @@ pub async fn handle(
                         "(function() {{\
                             var target = (document.elementFromPoint && document.elementFromPoint({x},{y})) || globalThis.__obscura_click_target || document.activeElement || document.body;\
                             if (!target) return;\
-                            var evt = new MouseEvent('mouseup', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}});\
+                            var evt = globalThis.__obscura_markTrusted(new MouseEvent('mouseup', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}}));\
                             target.dispatchEvent(evt);\
                         }})()",
                         x = x, y = y,
@@ -85,7 +85,7 @@ pub async fn handle(
                         let js = format!(
                             "(function() {{\
                                 var target = document.activeElement || document.body;\
-                                var evt = new KeyboardEvent('keydown', {{bubbles:true,cancelable:true,key:'{key}',code:'{code}'}});\
+                                var evt = globalThis.__obscura_markTrusted(new KeyboardEvent('keydown', {{bubbles:true,cancelable:true,key:'{key}',code:'{code}'}}));\
                                 target.dispatchEvent(evt);\
                             }})()",
                             key = key.replace('\'', "\\'"),
@@ -102,7 +102,7 @@ pub async fn handle(
                                     var target = document.activeElement;\
                                     if (target && (target.localName === 'input' || target.localName === 'textarea')) {{\
                                         target.value = (target.value || '') + '{text}';\
-                                        target.dispatchEvent(new Event('input', {{bubbles:true}}));\
+                                        target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {{bubbles:true}})));\
                                     }}\
                                 }})()",
                                 text = escaped_text,
@@ -118,10 +118,10 @@ pub async fn handle(
                             let js = "(function() {\
                                 var target = document.activeElement;\
                                 if (!target) return;\
-                                target.dispatchEvent(new KeyboardEvent('keypress', {bubbles:true,key:'Enter',code:'Enter'}));\
+                                target.dispatchEvent(globalThis.__obscura_markTrusted(new KeyboardEvent('keypress', {bubbles:true,key:'Enter',code:'Enter'})));\
                                 if (target.localName === 'textarea') {\
                                     target.value = (target.value || '') + '\\n';\
-                                    target.dispatchEvent(new Event('input', {bubbles:true}));\
+                                    target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {bubbles:true})));\
                                 } else {\
                                     var form = target.form || (target.closest && target.closest('form'));\
                                     if (form && typeof form.submit === 'function') form.submit();\
@@ -135,7 +135,7 @@ pub async fn handle(
                                 var target = document.activeElement;\
                                 if (target && (target.localName === 'input' || target.localName === 'textarea')) {\
                                     target.value = target.value.slice(0, -1);\
-                                    target.dispatchEvent(new Event('input', {bubbles:true}));\
+                                    target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {bubbles:true})));\
                                 }\
                             })()";
                             page.evaluate(js);
@@ -145,7 +145,7 @@ pub async fn handle(
                         let js = format!(
                             "(function() {{\
                                 var target = document.activeElement || document.body;\
-                                var evt = new KeyboardEvent('keyup', {{bubbles:true,key:'{key}',code:'{code}'}});\
+                                var evt = globalThis.__obscura_markTrusted(new KeyboardEvent('keyup', {{bubbles:true,key:'{key}',code:'{code}'}}));\
                                 target.dispatchEvent(evt);\
                             }})()",
                             key = key.replace('\'', "\\'"),
@@ -160,7 +160,7 @@ pub async fn handle(
                                     var target = document.activeElement;\
                                     if (target && (target.localName === 'input' || target.localName === 'textarea')) {{\
                                         target.value = (target.value || '') + '{text}';\
-                                        target.dispatchEvent(new Event('input', {{bubbles:true}}));\
+                                        target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {{bubbles:true}})));\
                                     }}\
                                 }})()",
                                 text = text.replace('\'', "\\'").replace('\\', "\\\\"),

crates/obscura-cli/src/main.rs

@@ -1160,7 +1160,30 @@ fn extract_assets(dom: &obscura_dom::DomTree, base_url: Option<&url::Url>) -> St
 
 fn dump_assets(page: &Page) -> String {
     let base_url = page.url.clone();
-    page.with_dom(|dom| extract_assets(dom, base_url.as_ref())).unwrap_or_default()
+    let dom_ndjson = page
+        .with_dom(|dom| extract_assets(dom, base_url.as_ref()))
+        .unwrap_or_default();
+
+    let mut lines: Vec<String> =
+        dom_ndjson.lines().filter(|l| !l.is_empty()).map(|l| l.to_string()).collect();
+
+    // URLs already listed from static DOM attributes, so a resource the script
+    // fetches that the markup also references is not emitted twice.
+    let mut seen: std::collections::HashSet<String> = lines
+        .iter()
+        .filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok())
+        .filter_map(|v| v.get("url").and_then(|u| u.as_str()).map(|s| s.to_string()))
+        .collect();
+
+    // Resources pulled in by JS fetch()/XHR, which leave no static DOM tag
+    // (issue #301).
+    for url in page.fetched_urls() {
+        if seen.insert(url.clone()) {
+            lines.push(serde_json::json!({ "url": url, "type": "fetch" }).to_string());
+        }
+    }
+
+    lines.join("\n")
 }
 
 #[cfg(test)]

crates/obscura-js/js/bootstrap.js

@@ -14,7 +14,7 @@
     '__obscura_errors', '__obscura_init', '__obscura_hide_list',
     '__obscura_objects', '__obscura_oid', '__obscura_ua',
     '__obscura_platform', '__obscura_ua_platform', '__obscura_ua_platform_version',
-    '__obscura_stealth',
+    '__obscura_stealth', '__obscura_markTrusted',
     '__documentReadyState__', '__currentUrl',
     // internal helpers (var-declared throughout the file)
     '__processDynScriptQueue', '_markNative', '_fpRand', '_fpNoise',
@@ -4009,9 +4009,18 @@ globalThis.DOMException = (function () {
   }
   return DOMException;
 })();
+// Per the UI Events spec, only events the user agent dispatches (real or
+// automation-synthesized input) are trusted; events page script builds with
+// `new Event(...)` must report isTrusted === false (issue #303). Returning true
+// for everything is a trivial bot-detection tell. Trusted events are tracked in
+// a closure-private WeakSet so page JS can neither read nor forge the flag.
+// obscura's CDP input pipeline marks its synthetic events via the
+// non-enumerable __obscura_markTrusted helper.
+const _trustedEvents = new WeakSet();
+globalThis.__obscura_markTrusted = function(ev) { try { if (ev) _trustedEvents.add(ev); } catch (_e) {} return ev; };
 globalThis.Event = class Event {
   constructor(t,o={}) { this.type=t;this.bubbles=!!o.bubbles;this.cancelable=!!o.cancelable;this.composed=!!o.composed;this.defaultPrevented=false;this.target=null;this.currentTarget=null;this.eventPhase=0;this.timeStamp=Date.now();this._propagationStopped=false;this._immediatePropagationStopped=false; }
-  get isTrusted() { return true; }
+  get isTrusted() { return _trustedEvents.has(this); }
   preventDefault() { if (this.cancelable) this.defaultPrevented=true; } stopPropagation(){ this._propagationStopped=true; } stopImmediatePropagation(){ this._propagationStopped=true; this._immediatePropagationStopped=true; }
   initEvent(type,bubbles,cancelable) { if (arguments.length < 1) throw new TypeError("Failed to execute 'initEvent' on 'Event': 1 argument required, but only 0 present."); this.type=String(type);this.bubbles=!!bubbles;this.cancelable=!!cancelable;this.defaultPrevented=false;this._propagationStopped=false;this._immediatePropagationStopped=false; }
   composedPath() {
@@ -4023,6 +4032,7 @@ globalThis.Event = class Event {
     return path;
   }
 };
+_markNative(Event);
 globalThis.CustomEvent = class extends Event {
   constructor(t,o={}) { super(t,o);this.detail=o.detail; }
   // Legacy DOM Level 2 init; some libraries (Starbucks China bundle, older

crates/obscura-js/src/ops.rs

@@ -66,6 +66,10 @@ pub struct ObscuraState {
     pub network_response_bodies: HashMap<String, StoredNetworkResponseBody>,
     pub network_response_body_order: VecDeque<String>,
     pub network_response_body_counter: u64,
+    // Absolute URLs requested via JS fetch() / XHR (op_fetch_url), in request
+    // order. Surfaced by `--dump assets` so resources pulled in by script, not
+    // just static DOM attributes, are listed (issue #301).
+    pub fetched_urls: Vec<String>,
 }
 
 impl ObscuraState {
@@ -86,6 +90,7 @@ impl ObscuraState {
             network_response_bodies: HashMap::new(),
             network_response_body_order: VecDeque::new(),
             network_response_body_counter: 0,
+            fetched_urls: Vec::new(),
         }
     }
 }
@@ -620,6 +625,10 @@ async fn op_fetch_url(
                 }).to_string());
             }
         }
+        // Record the resource the page pulled in via fetch()/XHR so `--dump
+        // assets` can list it (issue #301). URL is already absolute here, since
+        // reqwest needs an absolute URL to send the request.
+        gs.fetched_urls.push(url.clone());
         let jar = gs.cookie_jar.clone();
         let in_flight = gs.http_client.as_ref().map(|c| c.in_flight.clone());
         // #139: thread the configured proxy through to the per-request

crates/obscura-js/src/runtime.rs

@@ -957,6 +957,12 @@ impl ObscuraJsRuntime {
         state.dom.as_ref().map(f)
     }
 
+    /// Absolute URLs the page requested via fetch()/XHR, in request order
+    /// (issue #301). Backs `--dump assets`.
+    pub fn fetched_urls(&self) -> Vec<String> {
+        self.state.borrow().fetched_urls.clone()
+    }
+
     pub fn dom_ref(&self) -> Option<std::cell::Ref<'_, Option<DomTree>>> {
         let r = self.state.borrow();
         if r.dom.is_some() {