fix(cdp): emit Network events for script-initiated requests (#415)

Requests made by page JS (fetch/XHR/dynamic script and stylesheet loads)
run in the V8 op layer and never produced a NetworkEvent, so CDP emitted
Network.requestWillBeSent / responseReceived only for the static
navigation subresources. Puppeteer/Playwright page.on('request') and
on('response') therefore missed every XHR/JSON response, which breaks
response capture (and is the root cause of the Aviasales half of #394).

op_fetch_url now records a JsNetworkEvent keyed by the same fetch-{N} id
under which it already stores the response body. The Page drains these
into its network_events before the CDP layer emits, so they flow through
emit_navigation_events as requestWillBeSent / responseReceived /
loadingFinished alongside the document subresources, and
Network.getResponseBody resolves for them via the shared id. Applied to
both navigation paths (server process_navigation and in-process
do_navigate). The queue is capped at 4096 events per page.

Closes #406.

crates/obscura-browser/src/page.rs

@@ -1256,6 +1256,32 @@ impl Page {
         self.js.as_ref().map(|js| js.fetched_urls()).unwrap_or_default()
     }
 
+    /// Move network events recorded for script-initiated requests
+    /// (fetch/XHR/dynamic resource) from the JS runtime into this page's
+    /// network_events, so the CDP layer emits Network.requestWillBeSent /
+    /// responseReceived for them (issue #406). Idempotent: the runtime's queue
+    /// is drained, so calling this repeatedly does not duplicate events. The
+    /// fetch-{N} request id is preserved so Network.getResponseBody resolves.
+    pub fn sync_js_network_events(&mut self) {
+        let events = match self.js.as_ref() {
+            Some(js) => js.take_js_network_events(),
+            None => return,
+        };
+        for ev in events {
+            self.network_events.push(NetworkEvent {
+                request_id: ev.request_id,
+                url: ev.url,
+                method: ev.method,
+                resource_type: "Fetch".to_string(),
+                status: ev.status,
+                headers: std::collections::HashMap::new(),
+                response_headers: Arc::new(ev.response_headers),
+                body_size: ev.body_size,
+                timestamp: ev.timestamp,
+            });
+        }
+    }
+
     pub fn dom(&self) -> Option<&DomTree> {
         self.dom.as_ref()
     }

crates/obscura-cdp/src/domains/page.rs

@@ -263,6 +263,9 @@ async fn do_navigate(
         }
 
         let reached_network_idle = page.lifecycle.is_network_idle();
+        // Fold in script-initiated requests (fetch/XHR/dynamic resource) so they
+        // emit as Network events alongside static subresources (#406).
+        page.sync_js_network_events();
         let network_events: Vec<_> = page.network_events.drain(..).collect();
         let page_url = page.url_string();
         let page_id = page.id.clone();

crates/obscura-cdp/src/server.rs

@@ -778,6 +778,10 @@ async fn process_with_interception(
 
     let mut page = page_back.expect("navigation task should return the page");
 
+    // Fold in network events for script-initiated requests (fetch/XHR/dynamic
+    // resource) so they emit as Network.requestWillBeSent / responseReceived
+    // alongside the static navigation subresources (#406).
+    page.sync_js_network_events();
     let network_events: Vec<_> = page.network_events.drain(..).collect();
     let page_url = page.url_string();
     let page_id_for_events = page.id.clone();

crates/obscura-cdp/tests/js_fetch_emits_network_events.rs

@@ -0,0 +1,185 @@
+// Regression for issue #406: requests initiated by page JS (fetch/XHR/dynamic
+// resource) must emit Network.requestWillBeSent / responseReceived so
+// Puppeteer/Playwright `page.on('request'|'response')` observe them. On main
+// only the static navigation subresources surfaced; a `fetch()` fired from the
+// page produced no CDP Network event, so clients captured zero XHR/JSON
+// responses (this is also the root cause of the Aviasales half of #394).
+
+use obscura_cdp::dispatch::{dispatch, CdpContext};
+use obscura_cdp::types::CdpRequest;
+use serde_json::{json, Value};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tokio::net::TcpListener;
+
+// Serves an HTML page that, on load, fetches /api/data.json, plus that JSON.
+async fn serve() -> String {
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let addr = listener.local_addr().unwrap();
+    tokio::spawn(async move {
+        for _ in 0..6 {
+            let (mut socket, _) = listener.accept().await.unwrap();
+            tokio::spawn(async move {
+                let mut buf = [0u8; 2048];
+                let _ = socket.read(&mut buf).await.unwrap();
+                let req = String::from_utf8_lossy(&buf[..]);
+                let (ct, body) = if req.starts_with("GET /api/data.json") {
+                    ("application/json", "{\"value\":42}")
+                } else {
+                    (
+                        "text/html",
+                        r#"<html><head></head><body>
+<div id="r">stage1</div>
+<script>
+window.__done = new Promise(function (resolve) {
+  fetch("/api/data.json")
+    .then(function (r) { return r.json(); })
+    .then(function (d) { document.getElementById("r").textContent = "got:" + d.value; resolve("ok"); })
+    .catch(function (e) { resolve("err:" + e); });
+});
+</script>
+</body></html>"#,
+                    )
+                };
+                let resp = format!(
+                    "HTTP/1.1 200 OK\r\nContent-Type: {ct}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+                    body.len()
+                );
+                let _ = socket.write_all(resp.as_bytes()).await;
+            });
+        }
+    });
+    format!("http://{addr}/")
+}
+
+async fn cdp(ctx: &mut CdpContext, id: u64, method: &str, params: Value, session_id: &str) -> Value {
+    let resp = dispatch(
+        &CdpRequest {
+            id,
+            method: method.to_string(),
+            params,
+            session_id: Some(session_id.to_string()),
+        },
+        ctx,
+    )
+    .await;
+    assert!(resp.error.is_none(), "CDP {method} failed: {:?}", resp.error);
+    resp.result.unwrap_or_else(|| json!({}))
+}
+
+// Collect the request URLs from every Network.requestWillBeSent currently
+// queued in ctx.pending_events, then clear the queue.
+fn drain_request_urls(ctx: &mut CdpContext) -> Vec<String> {
+    let urls = ctx
+        .pending_events
+        .iter()
+        .filter(|e| e.method == "Network.requestWillBeSent")
+        .filter_map(|e| e.params.get("request").and_then(|r| r.get("url")).and_then(|u| u.as_str()).map(str::to_string))
+        .collect();
+    ctx.pending_events.clear();
+    urls
+}
+
+// The requestId that Network.responseReceived reported for the given URL.
+fn response_request_id(ctx: &CdpContext, url_needle: &str) -> Option<String> {
+    ctx.pending_events
+        .iter()
+        .find(|e| {
+            e.method == "Network.responseReceived"
+                && e.params
+                    .get("response")
+                    .and_then(|r| r.get("url"))
+                    .and_then(|u| u.as_str())
+                    .map(|u| u.contains(url_needle))
+                    .unwrap_or(false)
+        })
+        .and_then(|e| e.params.get("requestId").and_then(|v| v.as_str()).map(str::to_string))
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn js_fetch_emits_network_request_and_response() {
+    std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1");
+    let base = serve().await;
+    let mut ctx = CdpContext::new();
+    let page_id = ctx.create_page();
+    let session_id = "session-1";
+    ctx.sessions.insert(session_id.to_string(), page_id.clone());
+
+    // Navigate with waitUntil:load so the after-load fetch() runs and settles
+    // before the navigation emits its Network events.
+    cdp(
+        &mut ctx,
+        1,
+        "Page.navigate",
+        json!({"url": base, "waitUntil": "load"}),
+        session_id,
+    )
+    .await;
+
+    // The fetched JSON URL must appear as a requestWillBeSent event.
+    let request_urls = ctx
+        .pending_events
+        .iter()
+        .filter(|e| e.method == "Network.requestWillBeSent")
+        .filter_map(|e| e.params.get("request").and_then(|r| r.get("url")).and_then(|u| u.as_str()).map(str::to_string))
+        .collect::<Vec<_>>();
+    assert!(
+        request_urls.iter().any(|u| u.contains("/api/data.json")),
+        "script-initiated fetch must emit Network.requestWillBeSent; saw {request_urls:?}"
+    );
+
+    // And its response body must be resolvable via the same requestId, so a
+    // client can read the captured JSON.
+    let request_id = response_request_id(&ctx, "/api/data.json")
+        .expect("fetch must emit Network.responseReceived with a requestId");
+    let body = cdp(
+        &mut ctx,
+        2,
+        "Network.getResponseBody",
+        json!({"requestId": request_id}),
+        session_id,
+    )
+    .await;
+    assert_eq!(
+        body.get("body").and_then(|b| b.as_str()),
+        Some("{\"value\":42}"),
+        "Network.getResponseBody must return the script-fetched JSON"
+    );
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn navigation_without_script_fetch_is_unaffected() {
+    // A page that issues no script fetch must still emit exactly its document
+    // request, proving the #406 change adds nothing spurious.
+    std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1");
+    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let addr = listener.local_addr().unwrap();
+    tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.unwrap();
+        let mut buf = [0u8; 1024];
+        let _ = socket.read(&mut buf).await.unwrap();
+        let body = "<html><body>plain</body></html>";
+        let resp = format!(
+            "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+            body.len()
+        );
+        let _ = socket.write_all(resp.as_bytes()).await;
+    });
+    let base = format!("http://{addr}/");
+
+    let mut ctx = CdpContext::new();
+    let page_id = ctx.create_page();
+    let session_id = "session-1";
+    ctx.sessions.insert(session_id.to_string(), page_id.clone());
+
+    cdp(&mut ctx, 1, "Page.navigate", json!({"url": base, "waitUntil": "load"}), session_id).await;
+
+    let urls = drain_request_urls(&mut ctx);
+    assert!(
+        urls.iter().any(|u| u == &base || u.starts_with(&base)),
+        "the document request must still be emitted; saw {urls:?}"
+    );
+    assert!(
+        !urls.iter().any(|u| u.contains("/api/")),
+        "no spurious script-fetch events for a page that makes none; saw {urls:?}"
+    );
+}

crates/obscura-js/src/ops.rs

@@ -46,6 +46,24 @@ pub struct StoredNetworkResponseBody {
     pub base64_encoded: bool,
 }
 
+/// A network request made from page JS (fetch()/XHR/dynamic resource) recorded
+/// so the CDP layer can emit Network.requestWillBeSent / responseReceived for
+/// it. Static navigation subresources go through Page::record_network_event;
+/// this is the parallel channel for script-initiated requests, which run in the
+/// V8 op layer and would otherwise never surface as CDP Network events (#406).
+#[derive(Debug, Clone)]
+pub struct JsNetworkEvent {
+    /// Matches the `fetch-{N}` id under which the body is stored, so CDP
+    /// Network.getResponseBody resolves for the same request.
+    pub request_id: String,
+    pub url: String,
+    pub method: String,
+    pub status: u16,
+    pub response_headers: HashMap<String, String>,
+    pub body_size: usize,
+    pub timestamp: f64,
+}
+
 pub struct ObscuraState {
     pub dom: Option<DomTree>,
     pub url: String,
@@ -77,6 +95,10 @@ pub struct ObscuraState {
     // order. Surfaced by `--dump assets` so resources pulled in by script, not
     // just static DOM attributes, are listed (issue #301).
     pub fetched_urls: Vec<String>,
+    // Network events for script-initiated requests (fetch/XHR/dynamic resource),
+    // drained by the Page into its network_events so the CDP layer emits
+    // Network.requestWillBeSent / responseReceived for them (issue #406).
+    pub js_network_events: Vec<JsNetworkEvent>,
 }
 
 impl ObscuraState {
@@ -100,6 +122,7 @@ impl ObscuraState {
             network_response_body_order: VecDeque::new(),
             network_response_body_counter: 0,
             fetched_urls: Vec::new(),
+            js_network_events: Vec::new(),
         }
     }
 }
@@ -1043,6 +1066,28 @@ async fn op_fetch_url(
                 }
             }
         }
+        // Record a network event so the CDP layer emits requestWillBeSent /
+        // responseReceived for this script-initiated request (#406). Keyed by
+        // the same fetch-{N} id as the stored body so Network.getResponseBody
+        // resolves. Capped to keep a long-lived page from growing unbounded.
+        let timestamp = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_secs_f64();
+        gs.js_network_events.push(JsNetworkEvent {
+            request_id: request_id.clone(),
+            url: url.clone(),
+            method: method.clone(),
+            status,
+            response_headers: resp_headers.clone(),
+            body_size: resp_bytes.len(),
+            timestamp,
+        });
+        const MAX_JS_NETWORK_EVENTS: usize = 4096;
+        if gs.js_network_events.len() > MAX_JS_NETWORK_EVENTS {
+            let overflow = gs.js_network_events.len() - MAX_JS_NETWORK_EVENTS;
+            gs.js_network_events.drain(0..overflow);
+        }
         request_id
     };
 

crates/obscura-js/src/runtime.rs

@@ -988,6 +988,13 @@ impl ObscuraJsRuntime {
         self.state.borrow().fetched_urls.clone()
     }
 
+    /// Drain the network events recorded for script-initiated requests
+    /// (fetch/XHR/dynamic resource). The Page moves these into its own
+    /// network_events so the CDP layer emits Network events for them (#406).
+    pub fn take_js_network_events(&self) -> Vec<crate::ops::JsNetworkEvent> {
+        std::mem::take(&mut self.state.borrow_mut().js_network_events)
+    }
+
     pub fn dom_ref(&self) -> Option<std::cell::Ref<'_, Option<DomTree>>> {
         let r = self.state.borrow();
         if r.dom.is_some() {