From ab27e9153dbecb2be4e1ce7267c98040c57a9f03 Mon Sep 17 00:00:00 2001
From: SGavrl <277843151+SGavrl@users.noreply.github.com>
Date: Wed, 17 Jun 2026 16:36:08 +0000
Subject: [PATCH] cli: list fetch()/XHR resources in --dump assets

--dump assets only walked static DOM attributes (script[src], img[src], ...),
so resources a page pulled in with fetch() or XHR never showed up (issue #301).

Record each absolute URL requested through op_fetch_url in ObscuraState,
expose it via the runtime and Page, and append those URLs to --dump assets as
type "fetch", deduped against URLs already listed from the DOM.
---
 crates/obscura-browser/src/page.rs |  6 ++++++
 crates/obscura-cli/src/main.rs     | 25 ++++++++++++++++++++++++-
 crates/obscura-js/src/ops.rs       |  9 +++++++++
 crates/obscura-js/src/runtime.rs   |  6 ++++++
 4 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs
index 0a47482..51e75ea 100644
--- a/crates/obscura-browser/src/page.rs
+++ b/crates/obscura-browser/src/page.rs
@@ -1211,6 +1211,12 @@ impl Page {
         self.dom.as_ref().map(f)
     }
 
+    /// Absolute URLs the page pulled in via fetch()/XHR (issue #301). Empty
+    /// when the page has no live JS runtime.
+    pub fn fetched_urls(&self) -> Vec<String> {
+        self.js.as_ref().map(|js| js.fetched_urls()).unwrap_or_default()
+    }
+
     pub fn dom(&self) -> Option<&DomTree> {
         self.dom.as_ref()
     }
diff --git a/crates/obscura-cli/src/main.rs b/crates/obscura-cli/src/main.rs
index e088053..d2ab8dd 100644
--- a/crates/obscura-cli/src/main.rs
+++ b/crates/obscura-cli/src/main.rs
@@ -1160,7 +1160,30 @@ fn extract_assets(dom: &obscura_dom::DomTree, base_url: Option<&url::Url>) -> St
 
 fn dump_assets(page: &Page) -> String {
     let base_url = page.url.clone();
-    page.with_dom(|dom| extract_assets(dom, base_url.as_ref())).unwrap_or_default()
+    let dom_ndjson = page
+        .with_dom(|dom| extract_assets(dom, base_url.as_ref()))
+        .unwrap_or_default();
+
+    let mut lines: Vec<String> =
+        dom_ndjson.lines().filter(|l| !l.is_empty()).map(|l| l.to_string()).collect();
+
+    // URLs already listed from static DOM attributes, so a resource the script
+    // fetches that the markup also references is not emitted twice.
+    let mut seen: std::collections::HashSet<String> = lines
+        .iter()
+        .filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok())
+        .filter_map(|v| v.get("url").and_then(|u| u.as_str()).map(|s| s.to_string()))
+        .collect();
+
+    // Resources pulled in by JS fetch()/XHR, which leave no static DOM tag
+    // (issue #301).
+    for url in page.fetched_urls() {
+        if seen.insert(url.clone()) {
+            lines.push(serde_json::json!({ "url": url, "type": "fetch" }).to_string());
+        }
+    }
+
+    lines.join("\n")
 }
 
 #[cfg(test)]
diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs
index 7fb09f9..1975b72 100644
--- a/crates/obscura-js/src/ops.rs
+++ b/crates/obscura-js/src/ops.rs
@@ -66,6 +66,10 @@ pub struct ObscuraState {
     pub network_response_bodies: HashMap<String, StoredNetworkResponseBody>,
     pub network_response_body_order: VecDeque<String>,
     pub network_response_body_counter: u64,
+    // Absolute URLs requested via JS fetch() / XHR (op_fetch_url), in request
+    // order. Surfaced by `--dump assets` so resources pulled in by script, not
+    // just static DOM attributes, are listed (issue #301).
+    pub fetched_urls: Vec<String>,
 }
 
 impl ObscuraState {
@@ -86,6 +90,7 @@ impl ObscuraState {
             network_response_bodies: HashMap::new(),
             network_response_body_order: VecDeque::new(),
             network_response_body_counter: 0,
+            fetched_urls: Vec::new(),
         }
     }
 }
@@ -620,6 +625,10 @@ async fn op_fetch_url(
                 }).to_string());
             }
         }
+        // Record the resource the page pulled in via fetch()/XHR so `--dump
+        // assets` can list it (issue #301). URL is already absolute here, since
+        // reqwest needs an absolute URL to send the request.
+        gs.fetched_urls.push(url.clone());
         let jar = gs.cookie_jar.clone();
         let in_flight = gs.http_client.as_ref().map(|c| c.in_flight.clone());
         // #139: thread the configured proxy through to the per-request
diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs
index e0bdbcc..dcd7c96 100644
--- a/crates/obscura-js/src/runtime.rs
+++ b/crates/obscura-js/src/runtime.rs
@@ -957,6 +957,12 @@ impl ObscuraJsRuntime {
         state.dom.as_ref().map(f)
     }
 
+    /// Absolute URLs the page requested via fetch()/XHR, in request order
+    /// (issue #301). Backs `--dump assets`.
+    pub fn fetched_urls(&self) -> Vec<String> {
+        self.state.borrow().fetched_urls.clone()
+    }
+
     pub fn dom_ref(&self) -> Option<std::cell::Ref<'_, Option<DomTree>>> {
         let r = self.state.borrow();
         if r.dom.is_some() {