cli: list fetch()/XHR resources in --dump assets
--dump assets only walked static DOM attributes (script[src], img[src], ...), so resources a page pulled in with fetch() or XHR never showed up (issue #301). Record each absolute URL requested through op_fetch_url in ObscuraState, expose it via the runtime and Page, and append those URLs to --dump assets as type "fetch", deduped against URLs already listed from the DOM.
This commit is contained in:
1 parent
d526a4da7d
commit
ab27e9153d
4 files changed
+45
-1
No files matched your search
@@ -1211,6 +1211,12 @@ impl Page {
|
||||
self.dom.as_ref().map(f)
|
||||
}
|
||||
|
||||
/// Absolute URLs the page pulled in via fetch()/XHR (issue #301). Empty
|
||||
/// when the page has no live JS runtime.
|
||||
pub fn fetched_urls(&self) -> Vec<String> {
|
||||
self.js.as_ref().map(|js| js.fetched_urls()).unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn dom(&self) -> Option<&DomTree> {
|
||||
self.dom.as_ref()
|
||||
}
|
||||
|
||||
@@ -1160,7 +1160,30 @@ fn extract_assets(dom: &obscura_dom::DomTree, base_url: Option<&url::Url>) -> St
|
||||
|
||||
fn dump_assets(page: &Page) -> String {
|
||||
let base_url = page.url.clone();
|
||||
page.with_dom(|dom| extract_assets(dom, base_url.as_ref())).unwrap_or_default()
|
||||
let dom_ndjson = page
|
||||
.with_dom(|dom| extract_assets(dom, base_url.as_ref()))
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut lines: Vec<String> =
|
||||
dom_ndjson.lines().filter(|l| !l.is_empty()).map(|l| l.to_string()).collect();
|
||||
|
||||
// URLs already listed from static DOM attributes, so a resource the script
|
||||
// fetches that the markup also references is not emitted twice.
|
||||
let mut seen: std::collections::HashSet<String> = lines
|
||||
.iter()
|
||||
.filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok())
|
||||
.filter_map(|v| v.get("url").and_then(|u| u.as_str()).map(|s| s.to_string()))
|
||||
.collect();
|
||||
|
||||
// Resources pulled in by JS fetch()/XHR, which leave no static DOM tag
|
||||
// (issue #301).
|
||||
for url in page.fetched_urls() {
|
||||
if seen.insert(url.clone()) {
|
||||
lines.push(serde_json::json!({ "url": url, "type": "fetch" }).to_string());
|
||||
}
|
||||
}
|
||||
|
||||
lines.join("\n")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -66,6 +66,10 @@ pub struct ObscuraState {
|
||||
pub network_response_bodies: HashMap<String, StoredNetworkResponseBody>,
|
||||
pub network_response_body_order: VecDeque<String>,
|
||||
pub network_response_body_counter: u64,
|
||||
// Absolute URLs requested via JS fetch() / XHR (op_fetch_url), in request
|
||||
// order. Surfaced by `--dump assets` so resources pulled in by script, not
|
||||
// just static DOM attributes, are listed (issue #301).
|
||||
pub fetched_urls: Vec<String>,
|
||||
}
|
||||
|
||||
impl ObscuraState {
|
||||
@@ -86,6 +90,7 @@ impl ObscuraState {
|
||||
network_response_bodies: HashMap::new(),
|
||||
network_response_body_order: VecDeque::new(),
|
||||
network_response_body_counter: 0,
|
||||
fetched_urls: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -620,6 +625,10 @@ async fn op_fetch_url(
|
||||
}).to_string());
|
||||
}
|
||||
}
|
||||
// Record the resource the page pulled in via fetch()/XHR so `--dump
|
||||
// assets` can list it (issue #301). URL is already absolute here, since
|
||||
// reqwest needs an absolute URL to send the request.
|
||||
gs.fetched_urls.push(url.clone());
|
||||
let jar = gs.cookie_jar.clone();
|
||||
let in_flight = gs.http_client.as_ref().map(|c| c.in_flight.clone());
|
||||
// #139: thread the configured proxy through to the per-request
|
||||
|
||||
@@ -957,6 +957,12 @@ impl ObscuraJsRuntime {
|
||||
state.dom.as_ref().map(f)
|
||||
}
|
||||
|
||||
/// Absolute URLs the page requested via fetch()/XHR, in request order
|
||||
/// (issue #301). Backs `--dump assets`.
|
||||
pub fn fetched_urls(&self) -> Vec<String> {
|
||||
self.state.borrow().fetched_urls.clone()
|
||||
}
|
||||
|
||||
pub fn dom_ref(&self) -> Option<std::cell::Ref<'_, Option<DomTree>>> {
|
||||
let r = self.state.borrow();
|
||||
if r.dom.is_some() {
|
||||
|
||||
Reference in new issue
Block a user