cli: list fetch()/XHR resources in --dump assets

--dump assets only walked static DOM attributes (script[src], img[src], ...),
so resources a page pulled in with fetch() or XHR never showed up (issue #301).

Record each absolute URL requested through op_fetch_url in ObscuraState,
expose it via the runtime and Page, and append those URLs to --dump assets as
type "fetch", deduped against URLs already listed from the DOM.
This commit is contained in:
SGavrl committed 2026-06-17 16:36:08 +00:00
1 parent d526a4da7d
commit ab27e9153d
4 files changed
+45 -1

No files matched your search

+24 -1
View File
@@ -1160,7 +1160,30 @@ fn extract_assets(dom: &obscura_dom::DomTree, base_url: Option<&url::Url>) -> St
fn dump_assets(page: &Page) -> String {
let base_url = page.url.clone();
page.with_dom(|dom| extract_assets(dom, base_url.as_ref())).unwrap_or_default()
let dom_ndjson = page
.with_dom(|dom| extract_assets(dom, base_url.as_ref()))
.unwrap_or_default();
let mut lines: Vec<String> =
dom_ndjson.lines().filter(|l| !l.is_empty()).map(|l| l.to_string()).collect();
// URLs already listed from static DOM attributes, so a resource the script
// fetches that the markup also references is not emitted twice.
let mut seen: std::collections::HashSet<String> = lines
.iter()
.filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok())
.filter_map(|v| v.get("url").and_then(|u| u.as_str()).map(|s| s.to_string()))
.collect();
// Resources pulled in by JS fetch()/XHR, which leave no static DOM tag
// (issue #301).
for url in page.fetched_urls() {
if seen.insert(url.clone()) {
lines.push(serde_json::json!({ "url": url, "type": "fetch" }).to_string());
}
}
lines.join("\n")
}
#[cfg(test)]