cli: list fetch()/XHR resources in --dump assets
--dump assets only walked static DOM attributes (script[src], img[src], ...), so resources a page pulled in with fetch() or XHR never showed up (issue #301). Record each absolute URL requested through op_fetch_url in ObscuraState, expose it via the runtime and Page, and append those URLs to --dump assets as type "fetch", deduped against URLs already listed from the DOM.
This commit is contained in:
1 parent
d526a4da7d
commit
ab27e9153d
4 files changed
+45
-1
No files matched your search
@@ -1211,6 +1211,12 @@ impl Page {
|
|||||||
self.dom.as_ref().map(f)
|
self.dom.as_ref().map(f)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Absolute URLs the page pulled in via fetch()/XHR (issue #301). Empty
|
||||||
|
/// when the page has no live JS runtime.
|
||||||
|
pub fn fetched_urls(&self) -> Vec<String> {
|
||||||
|
self.js.as_ref().map(|js| js.fetched_urls()).unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn dom(&self) -> Option<&DomTree> {
|
pub fn dom(&self) -> Option<&DomTree> {
|
||||||
self.dom.as_ref()
|
self.dom.as_ref()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1160,7 +1160,30 @@ fn extract_assets(dom: &obscura_dom::DomTree, base_url: Option<&url::Url>) -> St
|
|||||||
|
|
||||||
fn dump_assets(page: &Page) -> String {
|
fn dump_assets(page: &Page) -> String {
|
||||||
let base_url = page.url.clone();
|
let base_url = page.url.clone();
|
||||||
page.with_dom(|dom| extract_assets(dom, base_url.as_ref())).unwrap_or_default()
|
let dom_ndjson = page
|
||||||
|
.with_dom(|dom| extract_assets(dom, base_url.as_ref()))
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let mut lines: Vec<String> =
|
||||||
|
dom_ndjson.lines().filter(|l| !l.is_empty()).map(|l| l.to_string()).collect();
|
||||||
|
|
||||||
|
// URLs already listed from static DOM attributes, so a resource the script
|
||||||
|
// fetches that the markup also references is not emitted twice.
|
||||||
|
let mut seen: std::collections::HashSet<String> = lines
|
||||||
|
.iter()
|
||||||
|
.filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok())
|
||||||
|
.filter_map(|v| v.get("url").and_then(|u| u.as_str()).map(|s| s.to_string()))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
// Resources pulled in by JS fetch()/XHR, which leave no static DOM tag
|
||||||
|
// (issue #301).
|
||||||
|
for url in page.fetched_urls() {
|
||||||
|
if seen.insert(url.clone()) {
|
||||||
|
lines.push(serde_json::json!({ "url": url, "type": "fetch" }).to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
lines.join("\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -66,6 +66,10 @@ pub struct ObscuraState {
|
|||||||
pub network_response_bodies: HashMap<String, StoredNetworkResponseBody>,
|
pub network_response_bodies: HashMap<String, StoredNetworkResponseBody>,
|
||||||
pub network_response_body_order: VecDeque<String>,
|
pub network_response_body_order: VecDeque<String>,
|
||||||
pub network_response_body_counter: u64,
|
pub network_response_body_counter: u64,
|
||||||
|
// Absolute URLs requested via JS fetch() / XHR (op_fetch_url), in request
|
||||||
|
// order. Surfaced by `--dump assets` so resources pulled in by script, not
|
||||||
|
// just static DOM attributes, are listed (issue #301).
|
||||||
|
pub fetched_urls: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ObscuraState {
|
impl ObscuraState {
|
||||||
@@ -86,6 +90,7 @@ impl ObscuraState {
|
|||||||
network_response_bodies: HashMap::new(),
|
network_response_bodies: HashMap::new(),
|
||||||
network_response_body_order: VecDeque::new(),
|
network_response_body_order: VecDeque::new(),
|
||||||
network_response_body_counter: 0,
|
network_response_body_counter: 0,
|
||||||
|
fetched_urls: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -620,6 +625,10 @@ async fn op_fetch_url(
|
|||||||
}).to_string());
|
}).to_string());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Record the resource the page pulled in via fetch()/XHR so `--dump
|
||||||
|
// assets` can list it (issue #301). URL is already absolute here, since
|
||||||
|
// reqwest needs an absolute URL to send the request.
|
||||||
|
gs.fetched_urls.push(url.clone());
|
||||||
let jar = gs.cookie_jar.clone();
|
let jar = gs.cookie_jar.clone();
|
||||||
let in_flight = gs.http_client.as_ref().map(|c| c.in_flight.clone());
|
let in_flight = gs.http_client.as_ref().map(|c| c.in_flight.clone());
|
||||||
// #139: thread the configured proxy through to the per-request
|
// #139: thread the configured proxy through to the per-request
|
||||||
|
|||||||
@@ -957,6 +957,12 @@ impl ObscuraJsRuntime {
|
|||||||
state.dom.as_ref().map(f)
|
state.dom.as_ref().map(f)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Absolute URLs the page requested via fetch()/XHR, in request order
|
||||||
|
/// (issue #301). Backs `--dump assets`.
|
||||||
|
pub fn fetched_urls(&self) -> Vec<String> {
|
||||||
|
self.state.borrow().fetched_urls.clone()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn dom_ref(&self) -> Option<std::cell::Ref<'_, Option<DomTree>>> {
|
pub fn dom_ref(&self) -> Option<std::cell::Ref<'_, Option<DomTree>>> {
|
||||||
let r = self.state.borrow();
|
let r = self.state.borrow();
|
||||||
if r.dom.is_some() {
|
if r.dom.is_some() {
|
||||||
|
|||||||
Reference in new issue
Block a user