cli: list fetch()/XHR resources in --dump assets

--dump assets only walked static DOM attributes (script[src], img[src], ...),
so resources a page pulled in with fetch() or XHR never showed up (issue #301).

Record each absolute URL requested through op_fetch_url in ObscuraState,
expose it via the runtime and Page, and append those URLs to --dump assets as
type "fetch", deduped against URLs already listed from the DOM.
This commit is contained in:
SGavrl committed 2026-06-17 16:36:08 +00:00
1 parent d526a4da7d
commit ab27e9153d
4 files changed
+45 -1

No files matched your search

+6
View File
@@ -1211,6 +1211,12 @@ impl Page {
self.dom.as_ref().map(f)
}
/// Absolute URLs the page pulled in via fetch()/XHR (issue #301). Empty
/// when the page has no live JS runtime.
pub fn fetched_urls(&self) -> Vec<String> {
self.js.as_ref().map(|js| js.fetched_urls()).unwrap_or_default()
}
pub fn dom(&self) -> Option<&DomTree> {
self.dom.as_ref()
}
+24 -1
View File
@@ -1160,7 +1160,30 @@ fn extract_assets(dom: &obscura_dom::DomTree, base_url: Option<&url::Url>) -> St
fn dump_assets(page: &Page) -> String {
let base_url = page.url.clone();
page.with_dom(|dom| extract_assets(dom, base_url.as_ref())).unwrap_or_default()
let dom_ndjson = page
.with_dom(|dom| extract_assets(dom, base_url.as_ref()))
.unwrap_or_default();
let mut lines: Vec<String> =
dom_ndjson.lines().filter(|l| !l.is_empty()).map(|l| l.to_string()).collect();
// URLs already listed from static DOM attributes, so a resource the script
// fetches that the markup also references is not emitted twice.
let mut seen: std::collections::HashSet<String> = lines
.iter()
.filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok())
.filter_map(|v| v.get("url").and_then(|u| u.as_str()).map(|s| s.to_string()))
.collect();
// Resources pulled in by JS fetch()/XHR, which leave no static DOM tag
// (issue #301).
for url in page.fetched_urls() {
if seen.insert(url.clone()) {
lines.push(serde_json::json!({ "url": url, "type": "fetch" }).to_string());
}
}
lines.join("\n")
}
#[cfg(test)]
+9
View File
@@ -66,6 +66,10 @@ pub struct ObscuraState {
pub network_response_bodies: HashMap<String, StoredNetworkResponseBody>,
pub network_response_body_order: VecDeque<String>,
pub network_response_body_counter: u64,
// Absolute URLs requested via JS fetch() / XHR (op_fetch_url), in request
// order. Surfaced by `--dump assets` so resources pulled in by script, not
// just static DOM attributes, are listed (issue #301).
pub fetched_urls: Vec<String>,
}
impl ObscuraState {
@@ -86,6 +90,7 @@ impl ObscuraState {
network_response_bodies: HashMap::new(),
network_response_body_order: VecDeque::new(),
network_response_body_counter: 0,
fetched_urls: Vec::new(),
}
}
}
@@ -620,6 +625,10 @@ async fn op_fetch_url(
}).to_string());
}
}
// Record the resource the page pulled in via fetch()/XHR so `--dump
// assets` can list it (issue #301). URL is already absolute here, since
// reqwest needs an absolute URL to send the request.
gs.fetched_urls.push(url.clone());
let jar = gs.cookie_jar.clone();
let in_flight = gs.http_client.as_ref().map(|c| c.in_flight.clone());
// #139: thread the configured proxy through to the per-request
+6
View File
@@ -957,6 +957,12 @@ impl ObscuraJsRuntime {
state.dom.as_ref().map(f)
}
/// Absolute URLs the page requested via fetch()/XHR, in request order
/// (issue #301). Backs `--dump assets`.
pub fn fetched_urls(&self) -> Vec<String> {
self.state.borrow().fetched_urls.clone()
}
pub fn dom_ref(&self) -> Option<std::cell::Ref<'_, Option<DomTree>>> {
let r = self.state.borrow();
if r.dom.is_some() {