From d0b6fc4ffd52eea3cb323bc042531172c6059886 Mon Sep 17 00:00:00 2001 From: SGavrl <277843151+SGavrl@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:23:55 +0000 Subject: [PATCH] fix(cdp): emit Network events for script-initiated requests (#415) Requests made by page JS (fetch/XHR/dynamic script and stylesheet loads) run in the V8 op layer and never produced a NetworkEvent, so CDP emitted Network.requestWillBeSent / responseReceived only for the static navigation subresources. Puppeteer/Playwright page.on('request') and on('response') therefore missed every XHR/JSON response, which breaks response capture (and is the root cause of the Aviasales half of #394). op_fetch_url now records a JsNetworkEvent keyed by the same fetch-{N} id under which it already stores the response body. The Page drains these into its network_events before the CDP layer emits, so they flow through emit_navigation_events as requestWillBeSent / responseReceived / loadingFinished alongside the document subresources, and Network.getResponseBody resolves for them via the shared id. Applied to both navigation paths (server process_navigation and in-process do_navigate). The queue is capped at 4096 events per page. Closes #406. --- crates/obscura-browser/src/page.rs | 26 +++ crates/obscura-cdp/src/domains/page.rs | 3 + crates/obscura-cdp/src/server.rs | 4 + .../tests/js_fetch_emits_network_events.rs | 185 ++++++++++++++++++ crates/obscura-js/src/ops.rs | 45 +++++ crates/obscura-js/src/runtime.rs | 7 + 6 files changed, 270 insertions(+) create mode 100644 crates/obscura-cdp/tests/js_fetch_emits_network_events.rs diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs index e1f5538..0336588 100644 --- a/crates/obscura-browser/src/page.rs +++ b/crates/obscura-browser/src/page.rs @@ -1256,6 +1256,32 @@ impl Page { self.js.as_ref().map(|js| js.fetched_urls()).unwrap_or_default() } + /// Move network events recorded for script-initiated requests + /// (fetch/XHR/dynamic resource) from the JS runtime into this page's + /// network_events, so the CDP layer emits Network.requestWillBeSent / + /// responseReceived for them (issue #406). Idempotent: the runtime's queue + /// is drained, so calling this repeatedly does not duplicate events. The + /// fetch-{N} request id is preserved so Network.getResponseBody resolves. + pub fn sync_js_network_events(&mut self) { + let events = match self.js.as_ref() { + Some(js) => js.take_js_network_events(), + None => return, + }; + for ev in events { + self.network_events.push(NetworkEvent { + request_id: ev.request_id, + url: ev.url, + method: ev.method, + resource_type: "Fetch".to_string(), + status: ev.status, + headers: std::collections::HashMap::new(), + response_headers: Arc::new(ev.response_headers), + body_size: ev.body_size, + timestamp: ev.timestamp, + }); + } + } + pub fn dom(&self) -> Option<&DomTree> { self.dom.as_ref() } diff --git a/crates/obscura-cdp/src/domains/page.rs b/crates/obscura-cdp/src/domains/page.rs index bd06e9a..ed048d5 100644 --- a/crates/obscura-cdp/src/domains/page.rs +++ b/crates/obscura-cdp/src/domains/page.rs @@ -263,6 +263,9 @@ async fn do_navigate( } let reached_network_idle = page.lifecycle.is_network_idle(); + // Fold in script-initiated requests (fetch/XHR/dynamic resource) so they + // emit as Network events alongside static subresources (#406). + page.sync_js_network_events(); let network_events: Vec<_> = page.network_events.drain(..).collect(); let page_url = page.url_string(); let page_id = page.id.clone(); diff --git a/crates/obscura-cdp/src/server.rs b/crates/obscura-cdp/src/server.rs index c6fc646..67f5dbb 100644 --- a/crates/obscura-cdp/src/server.rs +++ b/crates/obscura-cdp/src/server.rs @@ -778,6 +778,10 @@ async fn process_with_interception( let mut page = page_back.expect("navigation task should return the page"); + // Fold in network events for script-initiated requests (fetch/XHR/dynamic + // resource) so they emit as Network.requestWillBeSent / responseReceived + // alongside the static navigation subresources (#406). + page.sync_js_network_events(); let network_events: Vec<_> = page.network_events.drain(..).collect(); let page_url = page.url_string(); let page_id_for_events = page.id.clone(); diff --git a/crates/obscura-cdp/tests/js_fetch_emits_network_events.rs b/crates/obscura-cdp/tests/js_fetch_emits_network_events.rs new file mode 100644 index 0000000..d9694e4 --- /dev/null +++ b/crates/obscura-cdp/tests/js_fetch_emits_network_events.rs @@ -0,0 +1,185 @@ +// Regression for issue #406: requests initiated by page JS (fetch/XHR/dynamic +// resource) must emit Network.requestWillBeSent / responseReceived so +// Puppeteer/Playwright `page.on('request'|'response')` observe them. On main +// only the static navigation subresources surfaced; a `fetch()` fired from the +// page produced no CDP Network event, so clients captured zero XHR/JSON +// responses (this is also the root cause of the Aviasales half of #394). + +use obscura_cdp::dispatch::{dispatch, CdpContext}; +use obscura_cdp::types::CdpRequest; +use serde_json::{json, Value}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; + +// Serves an HTML page that, on load, fetches /api/data.json, plus that JSON. +async fn serve() -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + for _ in 0..6 { + let (mut socket, _) = listener.accept().await.unwrap(); + tokio::spawn(async move { + let mut buf = [0u8; 2048]; + let _ = socket.read(&mut buf).await.unwrap(); + let req = String::from_utf8_lossy(&buf[..]); + let (ct, body) = if req.starts_with("GET /api/data.json") { + ("application/json", "{\"value\":42}") + } else { + ( + "text/html", + r#" +
stage1
+ +"#, + ) + }; + let resp = format!( + "HTTP/1.1 200 OK\r\nContent-Type: {ct}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = socket.write_all(resp.as_bytes()).await; + }); + } + }); + format!("http://{addr}/") +} + +async fn cdp(ctx: &mut CdpContext, id: u64, method: &str, params: Value, session_id: &str) -> Value { + let resp = dispatch( + &CdpRequest { + id, + method: method.to_string(), + params, + session_id: Some(session_id.to_string()), + }, + ctx, + ) + .await; + assert!(resp.error.is_none(), "CDP {method} failed: {:?}", resp.error); + resp.result.unwrap_or_else(|| json!({})) +} + +// Collect the request URLs from every Network.requestWillBeSent currently +// queued in ctx.pending_events, then clear the queue. +fn drain_request_urls(ctx: &mut CdpContext) -> Vec { + let urls = ctx + .pending_events + .iter() + .filter(|e| e.method == "Network.requestWillBeSent") + .filter_map(|e| e.params.get("request").and_then(|r| r.get("url")).and_then(|u| u.as_str()).map(str::to_string)) + .collect(); + ctx.pending_events.clear(); + urls +} + +// The requestId that Network.responseReceived reported for the given URL. +fn response_request_id(ctx: &CdpContext, url_needle: &str) -> Option { + ctx.pending_events + .iter() + .find(|e| { + e.method == "Network.responseReceived" + && e.params + .get("response") + .and_then(|r| r.get("url")) + .and_then(|u| u.as_str()) + .map(|u| u.contains(url_needle)) + .unwrap_or(false) + }) + .and_then(|e| e.params.get("requestId").and_then(|v| v.as_str()).map(str::to_string)) +} + +#[tokio::test(flavor = "current_thread")] +async fn js_fetch_emits_network_request_and_response() { + std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1"); + let base = serve().await; + let mut ctx = CdpContext::new(); + let page_id = ctx.create_page(); + let session_id = "session-1"; + ctx.sessions.insert(session_id.to_string(), page_id.clone()); + + // Navigate with waitUntil:load so the after-load fetch() runs and settles + // before the navigation emits its Network events. + cdp( + &mut ctx, + 1, + "Page.navigate", + json!({"url": base, "waitUntil": "load"}), + session_id, + ) + .await; + + // The fetched JSON URL must appear as a requestWillBeSent event. + let request_urls = ctx + .pending_events + .iter() + .filter(|e| e.method == "Network.requestWillBeSent") + .filter_map(|e| e.params.get("request").and_then(|r| r.get("url")).and_then(|u| u.as_str()).map(str::to_string)) + .collect::>(); + assert!( + request_urls.iter().any(|u| u.contains("/api/data.json")), + "script-initiated fetch must emit Network.requestWillBeSent; saw {request_urls:?}" + ); + + // And its response body must be resolvable via the same requestId, so a + // client can read the captured JSON. + let request_id = response_request_id(&ctx, "/api/data.json") + .expect("fetch must emit Network.responseReceived with a requestId"); + let body = cdp( + &mut ctx, + 2, + "Network.getResponseBody", + json!({"requestId": request_id}), + session_id, + ) + .await; + assert_eq!( + body.get("body").and_then(|b| b.as_str()), + Some("{\"value\":42}"), + "Network.getResponseBody must return the script-fetched JSON" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn navigation_without_script_fetch_is_unaffected() { + // A page that issues no script fetch must still emit exactly its document + // request, proving the #406 change adds nothing spurious. + std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1"); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buf = [0u8; 1024]; + let _ = socket.read(&mut buf).await.unwrap(); + let body = "plain"; + let resp = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = socket.write_all(resp.as_bytes()).await; + }); + let base = format!("http://{addr}/"); + + let mut ctx = CdpContext::new(); + let page_id = ctx.create_page(); + let session_id = "session-1"; + ctx.sessions.insert(session_id.to_string(), page_id.clone()); + + cdp(&mut ctx, 1, "Page.navigate", json!({"url": base, "waitUntil": "load"}), session_id).await; + + let urls = drain_request_urls(&mut ctx); + assert!( + urls.iter().any(|u| u == &base || u.starts_with(&base)), + "the document request must still be emitted; saw {urls:?}" + ); + assert!( + !urls.iter().any(|u| u.contains("/api/")), + "no spurious script-fetch events for a page that makes none; saw {urls:?}" + ); +} diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs index 79e4ea1..5b800cd 100644 --- a/crates/obscura-js/src/ops.rs +++ b/crates/obscura-js/src/ops.rs @@ -46,6 +46,24 @@ pub struct StoredNetworkResponseBody { pub base64_encoded: bool, } +/// A network request made from page JS (fetch()/XHR/dynamic resource) recorded +/// so the CDP layer can emit Network.requestWillBeSent / responseReceived for +/// it. Static navigation subresources go through Page::record_network_event; +/// this is the parallel channel for script-initiated requests, which run in the +/// V8 op layer and would otherwise never surface as CDP Network events (#406). +#[derive(Debug, Clone)] +pub struct JsNetworkEvent { + /// Matches the `fetch-{N}` id under which the body is stored, so CDP + /// Network.getResponseBody resolves for the same request. + pub request_id: String, + pub url: String, + pub method: String, + pub status: u16, + pub response_headers: HashMap, + pub body_size: usize, + pub timestamp: f64, +} + pub struct ObscuraState { pub dom: Option, pub url: String, @@ -77,6 +95,10 @@ pub struct ObscuraState { // order. Surfaced by `--dump assets` so resources pulled in by script, not // just static DOM attributes, are listed (issue #301). pub fetched_urls: Vec, + // Network events for script-initiated requests (fetch/XHR/dynamic resource), + // drained by the Page into its network_events so the CDP layer emits + // Network.requestWillBeSent / responseReceived for them (issue #406). + pub js_network_events: Vec, } impl ObscuraState { @@ -100,6 +122,7 @@ impl ObscuraState { network_response_body_order: VecDeque::new(), network_response_body_counter: 0, fetched_urls: Vec::new(), + js_network_events: Vec::new(), } } } @@ -1043,6 +1066,28 @@ async fn op_fetch_url( } } } + // Record a network event so the CDP layer emits requestWillBeSent / + // responseReceived for this script-initiated request (#406). Keyed by + // the same fetch-{N} id as the stored body so Network.getResponseBody + // resolves. Capped to keep a long-lived page from growing unbounded. + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(); + gs.js_network_events.push(JsNetworkEvent { + request_id: request_id.clone(), + url: url.clone(), + method: method.clone(), + status, + response_headers: resp_headers.clone(), + body_size: resp_bytes.len(), + timestamp, + }); + const MAX_JS_NETWORK_EVENTS: usize = 4096; + if gs.js_network_events.len() > MAX_JS_NETWORK_EVENTS { + let overflow = gs.js_network_events.len() - MAX_JS_NETWORK_EVENTS; + gs.js_network_events.drain(0..overflow); + } request_id }; diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs index 2a8361b..e558105 100644 --- a/crates/obscura-js/src/runtime.rs +++ b/crates/obscura-js/src/runtime.rs @@ -988,6 +988,13 @@ impl ObscuraJsRuntime { self.state.borrow().fetched_urls.clone() } + /// Drain the network events recorded for script-initiated requests + /// (fetch/XHR/dynamic resource). The Page moves these into its own + /// network_events so the CDP layer emits Network events for them (#406). + pub fn take_js_network_events(&self) -> Vec { + std::mem::take(&mut self.state.borrow_mut().js_network_events) + } + pub fn dom_ref(&self) -> Option>> { let r = self.state.borrow(); if r.dom.is_some() {