fix(cdp): emit Network events for script-initiated requests (#415)

Requests made by page JS (fetch/XHR/dynamic script and stylesheet loads)
run in the V8 op layer and never produced a NetworkEvent, so CDP emitted
Network.requestWillBeSent / responseReceived only for the static
navigation subresources. Puppeteer/Playwright page.on('request') and
on('response') therefore missed every XHR/JSON response, which breaks
response capture (and is the root cause of the Aviasales half of #394).

op_fetch_url now records a JsNetworkEvent keyed by the same fetch-{N} id
under which it already stores the response body. The Page drains these
into its network_events before the CDP layer emits, so they flow through
emit_navigation_events as requestWillBeSent / responseReceived /
loadingFinished alongside the document subresources, and
Network.getResponseBody resolves for them via the shared id. Applied to
both navigation paths (server process_navigation and in-process
do_navigate). The queue is capped at 4096 events per page.

Closes #406.

crates/obscura-js/src/ops.rs

@@ -46,6 +46,24 @@ pub struct StoredNetworkResponseBody {
     pub base64_encoded: bool,
 }
 
+/// A network request made from page JS (fetch()/XHR/dynamic resource) recorded
+/// so the CDP layer can emit Network.requestWillBeSent / responseReceived for
+/// it. Static navigation subresources go through Page::record_network_event;
+/// this is the parallel channel for script-initiated requests, which run in the
+/// V8 op layer and would otherwise never surface as CDP Network events (#406).
+#[derive(Debug, Clone)]
+pub struct JsNetworkEvent {
+    /// Matches the `fetch-{N}` id under which the body is stored, so CDP
+    /// Network.getResponseBody resolves for the same request.
+    pub request_id: String,
+    pub url: String,
+    pub method: String,
+    pub status: u16,
+    pub response_headers: HashMap<String, String>,
+    pub body_size: usize,
+    pub timestamp: f64,
+}
+
 pub struct ObscuraState {
     pub dom: Option<DomTree>,
     pub url: String,
@@ -77,6 +95,10 @@ pub struct ObscuraState {
     // order. Surfaced by `--dump assets` so resources pulled in by script, not
     // just static DOM attributes, are listed (issue #301).
     pub fetched_urls: Vec<String>,
+    // Network events for script-initiated requests (fetch/XHR/dynamic resource),
+    // drained by the Page into its network_events so the CDP layer emits
+    // Network.requestWillBeSent / responseReceived for them (issue #406).
+    pub js_network_events: Vec<JsNetworkEvent>,
 }
 
 impl ObscuraState {
@@ -100,6 +122,7 @@ impl ObscuraState {
             network_response_body_order: VecDeque::new(),
             network_response_body_counter: 0,
             fetched_urls: Vec::new(),
+            js_network_events: Vec::new(),
         }
     }
 }
@@ -1043,6 +1066,28 @@ async fn op_fetch_url(
                 }
             }
         }
+        // Record a network event so the CDP layer emits requestWillBeSent /
+        // responseReceived for this script-initiated request (#406). Keyed by
+        // the same fetch-{N} id as the stored body so Network.getResponseBody
+        // resolves. Capped to keep a long-lived page from growing unbounded.
+        let timestamp = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_secs_f64();
+        gs.js_network_events.push(JsNetworkEvent {
+            request_id: request_id.clone(),
+            url: url.clone(),
+            method: method.clone(),
+            status,
+            response_headers: resp_headers.clone(),
+            body_size: resp_bytes.len(),
+            timestamp,
+        });
+        const MAX_JS_NETWORK_EVENTS: usize = 4096;
+        if gs.js_network_events.len() > MAX_JS_NETWORK_EVENTS {
+            let overflow = gs.js_network_events.len() - MAX_JS_NETWORK_EVENTS;
+            gs.js_network_events.drain(0..overflow);
+        }
         request_id
     };
 

crates/obscura-js/src/runtime.rs

@@ -988,6 +988,13 @@ impl ObscuraJsRuntime {
         self.state.borrow().fetched_urls.clone()
     }
 
+    /// Drain the network events recorded for script-initiated requests
+    /// (fetch/XHR/dynamic resource). The Page moves these into its own
+    /// network_events so the CDP layer emits Network events for them (#406).
+    pub fn take_js_network_events(&self) -> Vec<crate::ops::JsNetworkEvent> {
+        std::mem::take(&mut self.state.borrow_mut().js_network_events)
+    }
+
     pub fn dom_ref(&self) -> Option<std::cell::Ref<'_, Option<DomTree>>> {
         let r = self.state.borrow();
         if r.dom.is_some() {