From 1f5066fc30ffd37f447d0787c3893322cdf083ae Mon Sep 17 00:00:00 2001
From: SGavrl <277843151+SGavrl@users.noreply.github.com>
Date: Wed, 24 Jun 2026 21:36:01 +0000
Subject: [PATCH] api: add request/response interception API to Page (#306)

Expose the interception infrastructure that already existed internally but was
private, so embedders can intercept, observe, mock, or block JS fetch()/XHR
without forking or unsafe casts.

On obscura::Page:
- add_preload_script: inject JS before any page <script> (CDP
  Page.addScriptToEvaluateOnNewDocument). The field already ran the scripts;
  this only adds the method.
- enable_interception: returns a receiver for every JS fetch()/XHR request;
  resolve each through its resolver with InterceptResolution::{Continue,
  Fulfill, Fail} to pass, mock, or block (CDP Fetch.enable equivalent).
- on_request / on_response: passive callbacks. on_response is the main path for
  capturing API response payloads from SPAs.

on_request/on_response previously fired only for navigation; op_fetch_url now
invokes them for JS fetch()/XHR too (both reqwest and stealth paths), gated on
a non-empty callback vector so the common no-callback case pays nothing.

Re-export InterceptedRequest, InterceptResolution, RequestInfo, ResourceType,
RequestCallback, ResponseCallback, Response from the obscura crate.

Integration test tests/interception.rs exercises all three against a local echo
server.
---
 crates/obscura-browser/src/lib.rs    |   3 +
 crates/obscura-browser/src/page.rs   |  42 ++++++++++-
 crates/obscura-js/src/ops.rs         |  69 ++++++++++++++++-
 crates/obscura-net/src/lib.rs        |   4 +-
 crates/obscura/Cargo.toml            |   3 +
 crates/obscura/src/lib.rs            |   4 +
 crates/obscura/src/page.rs           |  36 ++++++++-
 crates/obscura/tests/interception.rs | 107 +++++++++++++++++++++++++++
 8 files changed, 261 insertions(+), 7 deletions(-)
 create mode 100644 crates/obscura/tests/interception.rs

diff --git a/crates/obscura-browser/src/lib.rs b/crates/obscura-browser/src/lib.rs
index 376b816..c715344 100644
--- a/crates/obscura-browser/src/lib.rs
+++ b/crates/obscura-browser/src/lib.rs
@@ -7,3 +7,6 @@ pub use page::{NetworkEvent, Page, PageError};
 pub use context::BrowserContext;
 pub use lifecycle::{LifecycleState, WaitUntil};
 pub use obscura_js::HTML_TO_MARKDOWN_JS;
+// Re-exported so the embeddable `obscura` crate (which depends on obscura-browser,
+// not obscura-js) can surface the interception channel types.
+pub use obscura_js::ops::{InterceptResolution, InterceptedRequest};
diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs
index 393ca9c..9e0079f 100644
--- a/crates/obscura-browser/src/page.rs
+++ b/crates/obscura-browser/src/page.rs
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
 use obscura_dom::{parse_html, DomTree};
 use obscura_js::runtime::ObscuraJsRuntime;
-use obscura_net::{ObscuraHttpClient, ObscuraNetError, Response};
+use obscura_net::{ObscuraHttpClient, ObscuraNetError, RequestCallback, Response, ResponseCallback};
 use url::Url;
 
 use crate::context::BrowserContext;
@@ -1526,6 +1526,46 @@ impl Page {
         self.preload_scripts = scripts;
     }
 
+    /// Append a script that runs in the page before any of the page's own
+    /// `<script>` tags, matching CDP `Page.addScriptToEvaluateOnNewDocument`.
+    /// Takes effect on the next navigation (`goto` / `navigate*`).
+    pub fn add_preload_script(&mut self, script: &str) {
+        self.preload_scripts.push(script.to_string());
+    }
+
+    /// Enable CDP-Fetch-style interception of JS-initiated `fetch()`/XHR.
+    /// Returns a receiver yielding every such request; resolve each through its
+    /// `resolver` with `InterceptResolution::{Continue, Fulfill, Fail}` to pass,
+    /// mock, or block it. Works in stealth and non-stealth. Mirrors how the CDP
+    /// server wires the channel (`obscura-cdp/src/server.rs`).
+    pub fn enable_interception(
+        &mut self,
+    ) -> tokio::sync::mpsc::UnboundedReceiver<obscura_js::ops::InterceptedRequest> {
+        let (tx, rx) =
+            tokio::sync::mpsc::unbounded_channel::<obscura_js::ops::InterceptedRequest>();
+        self.set_intercept_tx(tx);
+        self.enable_intercept(true);
+        rx
+    }
+
+    /// Register a passive callback fired for every JS `fetch()`/XHR (and
+    /// navigation) request, once the method/headers/body are known and before it
+    /// is sent. Non-blocking; use `enable_interception` to mutate or block.
+    pub fn on_request(&mut self, cb: RequestCallback) {
+        if let Ok(mut v) = self.http_client.on_request.try_write() {
+            v.push(cb);
+        }
+    }
+
+    /// Register a passive callback fired with every JS `fetch()`/XHR (and
+    /// navigation) response, including its body. Non-blocking. The main path for
+    /// crawlers that need to capture API response payloads.
+    pub fn on_response(&mut self, cb: ResponseCallback) {
+        if let Ok(mut v) = self.http_client.on_response.try_write() {
+            v.push(cb);
+        }
+    }
+
     pub async fn process_pending_navigation(&mut self) -> Result<bool, PageError> {
         if let Some((url, method, body)) = self.take_pending_navigation() {
             self.navigate_with_wait_post(
diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs
index c33958f..a2b64f1 100644
--- a/crates/obscura-js/src/ops.rs
+++ b/crates/obscura-js/src/ops.rs
@@ -8,7 +8,7 @@ use deno_core::OpState;
 use deno_core::Extension;
 use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
 use obscura_dom::{DomTree, NodeData, NodeId};
-use obscura_net::{CookieJar, ObscuraHttpClient};
+use obscura_net::{CookieJar, ObscuraHttpClient, RequestInfo, ResourceType, Response};
 #[cfg(feature = "stealth")]
 use obscura_net::StealthHttpClient;
 use tokio::sync::Mutex;
@@ -619,7 +619,7 @@ async fn op_fetch_url(
         }
     }
 
-    let (cookie_jar, in_flight, intercept_tx, proxy_url) = {
+    let (cookie_jar, in_flight, intercept_tx, proxy_url, http_client) = {
         let state_borrow = state.borrow();
         let gs = state_borrow.borrow::<SharedState>().clone();
         let mut gs = gs.borrow_mut();
@@ -651,7 +651,7 @@ async fn op_fetch_url(
         } else {
             None
         };
-        (jar, in_flight, itx, proxy_url)
+        (jar, in_flight, itx, proxy_url, gs.http_client.clone())
     };
 
     if let Some((tx, request_id)) = intercept_tx {
@@ -716,6 +716,27 @@ async fn op_fetch_url(
     let custom_headers: std::collections::HashMap<String, String> =
         serde_json::from_str(&headers_json).unwrap_or_default();
 
+    // Passive request observation (non-blocking). Fires for every request that
+    // reaches the network (Fulfill/Fail from the interception channel short-
+    // circuit earlier). on_request/on_response previously fired only for
+    // navigation; this wires them for JS fetch()/XHR too.
+    if let Some(ref hc) = http_client {
+        let cbs = hc.on_request.read().await;
+        if !cbs.is_empty() {
+            if let Ok(parsed) = url::Url::parse(&url) {
+                let info = RequestInfo {
+                    url: parsed,
+                    method: method.clone(),
+                    headers: custom_headers.clone(),
+                    resource_type: ResourceType::Fetch,
+                };
+                for cb in cbs.iter() {
+                    cb(&info);
+                }
+            }
+        }
+    }
+
     // Stealth mode: route the scripted request through the wreq client so its
     // TLS fingerprint and Chrome client hints match the main navigation. The
     // rustls ClientHello plus missing client hints that op_fetch_url's reqwest
@@ -739,6 +760,7 @@ async fn op_fetch_url(
                 page_origin.clone(),
                 is_cross_origin,
                 mode.clone(),
+                http_client.clone(),
             )
             .await;
         }
@@ -944,6 +966,21 @@ async fn op_fetch_url(
         .map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?;
     let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
     let resp_body_base64 = BASE64.encode(&resp_bytes);
+    if let Some(ref hc) = http_client {
+        let cbs = hc.on_response.read().await;
+        if !cbs.is_empty() {
+            let resp = fetch_response(&url, status, resp_headers.clone(), resp_bytes.to_vec());
+            let info = RequestInfo {
+                url: resp.url.clone(),
+                method: method.clone(),
+                headers: resp_headers.clone(),
+                resource_type: ResourceType::Fetch,
+            };
+            for cb in cbs.iter() {
+                cb(&info, &resp);
+            }
+        }
+    }
     let response_request_id = {
         let state_borrow = state.borrow();
         let gs = state_borrow.borrow::<SharedState>().clone();
@@ -989,6 +1026,16 @@ async fn op_fetch_url(
 /// handling lives inside StealthHttpClient::send_single, which shares the
 /// context jar. Response bodies are not mirrored into the CDP
 /// Network.getResponseBody buffer here; that is a follow-up for stealth fetches.
+fn fetch_response(url: &str, status: u16, headers: HashMap<String, String>, body: Vec<u8>) -> Response {
+    Response {
+        url: url::Url::parse(url).unwrap_or_else(|_| url::Url::parse("http://0.0.0.0/").unwrap()),
+        status,
+        headers,
+        body,
+        redirected_from: Vec::new(),
+    }
+}
+
 #[cfg(feature = "stealth")]
 async fn stealth_fetch_all(
     stealth: Arc<StealthHttpClient>,
@@ -999,6 +1046,7 @@ async fn stealth_fetch_all(
     page_origin: String,
     is_cross_origin: bool,
     mode: String,
+    http_client: Option<Arc<ObscuraHttpClient>>,
 ) -> Result<String, deno_error::JsErrorBox> {
     let mut current_url = url.clone();
     let mut current_method = method;
@@ -1086,6 +1134,21 @@ async fn stealth_fetch_all(
 
     let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
     let resp_body_base64 = BASE64.encode(&resp_bytes);
+    if let Some(ref hc) = http_client {
+        let cbs = hc.on_response.read().await;
+        if !cbs.is_empty() {
+            let resp = fetch_response(&url, status, resp_headers.clone(), resp_bytes.clone());
+            let info = RequestInfo {
+                url: resp.url.clone(),
+                method: current_method.clone(),
+                headers: resp_headers.clone(),
+                resource_type: ResourceType::Fetch,
+            };
+            for cb in cbs.iter() {
+                cb(&info, &resp);
+            }
+        }
+    }
 
     Ok(serde_json::json!({
         "status": status,
diff --git a/crates/obscura-net/src/lib.rs b/crates/obscura-net/src/lib.rs
index f9840ad..7542072 100644
--- a/crates/obscura-net/src/lib.rs
+++ b/crates/obscura-net/src/lib.rs
@@ -8,8 +8,8 @@ pub mod blocklist;
 pub mod wreq_client;
 
 pub use client::{
-    env_allows_private_network, is_forbidden_ip, ObscuraHttpClient, ObscuraNetError, RequestInfo,
-    ResourceType, Response, SsrfGuardResolver,
+    env_allows_private_network, is_forbidden_ip, ObscuraHttpClient, ObscuraNetError, RequestCallback,
+    RequestInfo, ResourceType, Response, ResponseCallback, SsrfGuardResolver,
 };
 pub use cookies::{CookieInfo, CookieJar};
 pub use encoding::{
diff --git a/crates/obscura/Cargo.toml b/crates/obscura/Cargo.toml
index 2d7edac..53076ae 100644
--- a/crates/obscura/Cargo.toml
+++ b/crates/obscura/Cargo.toml
@@ -20,3 +20,6 @@ thiserror = "1"
 serde_json = "1"
 serde = { version = "1", features = ["derive"] }
 url = "2"
+
+[dev-dependencies]
+tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time"] }
diff --git a/crates/obscura/src/lib.rs b/crates/obscura/src/lib.rs
index df4c44d..78e366c 100644
--- a/crates/obscura/src/lib.rs
+++ b/crates/obscura/src/lib.rs
@@ -26,3 +26,7 @@ pub use config::BrowserConfig;
 pub use cookie::{Cookie, CookieStore};
 pub use error::Error;
 pub use page::Page;
+
+// Request/response interception types (issue #306).
+pub use obscura_browser::{InterceptedRequest, InterceptResolution};
+pub use obscura_net::{RequestCallback, RequestInfo, ResourceType, Response, ResponseCallback};
diff --git a/crates/obscura/src/page.rs b/crates/obscura/src/page.rs
index 07a9716..d45d7de 100644
--- a/crates/obscura/src/page.rs
+++ b/crates/obscura/src/page.rs
@@ -1,7 +1,8 @@
 use std::time::Duration;
 
 use obscura_browser::lifecycle::WaitUntil;
-use obscura_browser::Page as InnerPage;
+use obscura_browser::{InterceptedRequest, Page as InnerPage};
+use obscura_net::{RequestCallback, ResponseCallback};
 use serde_json::Value;
 
 use crate::error::Error;
@@ -91,6 +92,39 @@ impl Page {
     pub async fn settle(&mut self, max_ms: u64) {
         self.inner.settle(max_ms).await
     }
+
+    /// Register a script that runs before any of the page's own `<script>` tags,
+    /// equivalent to CDP `Page.addScriptToEvaluateOnNewDocument`. Runs on the next
+    /// `goto()` / navigation. Use it to install a fetch()/XHR interceptor or any
+    /// other page-init logic before the page's bootstrap runs.
+    pub fn add_preload_script(&mut self, script: &str) {
+        self.inner.add_preload_script(script);
+    }
+
+    /// Enable CDP-Fetch-style interception of every JS `fetch()`/XHR. Returns a
+    /// receiver yielding each request; resolve it through its `resolver` with
+    /// [`obscura::InterceptResolution`] (`Continue`, `Fulfill`, `Fail`) to pass,
+    /// mock, or block it. Works in stealth and non-stealth.
+    pub fn enable_interception(
+        &mut self,
+    ) -> tokio::sync::mpsc::UnboundedReceiver<InterceptedRequest> {
+        self.inner.enable_interception()
+    }
+
+    /// Register a passive callback fired for every request the page makes
+    /// (navigation and JS `fetch()`/XHR), once its method/headers/body are known
+    /// and before it is sent. Non-blocking; use `enable_interception` to mutate
+    /// or block.
+    pub fn on_request(&mut self, cb: RequestCallback) {
+        self.inner.on_request(cb);
+    }
+
+    /// Register a passive callback fired with every response the page receives
+    /// (navigation and JS `fetch()`/XHR), including its body. Non-blocking. The
+    /// main path for capturing API response payloads from SPAs.
+    pub fn on_response(&mut self, cb: ResponseCallback) {
+        self.inner.on_response(cb);
+    }
 }
 
 /// Handle to a DOM element.
diff --git a/crates/obscura/tests/interception.rs b/crates/obscura/tests/interception.rs
new file mode 100644
index 0000000..acdc80a
--- /dev/null
+++ b/crates/obscura/tests/interception.rs
@@ -0,0 +1,107 @@
+//! End-to-end check for the request/response interception API on `obscura::Page`
+//! (issue #306): preload scripts, the interception channel, and the passive
+//! on_request/on_response callbacks all work for JS-initiated fetch()/XHR.
+
+use std::io::{Read, Write};
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::{Arc, Mutex};
+
+use obscura::{Browser, InterceptResolution, ResourceType};
+
+/// Minimal HTTP/1.1 server: `/` returns HTML that fires `fetch('/api')`; `/api`
+/// returns JSON. Enough to exercise JS fetch() interception + the callbacks.
+fn spawn_echo_server() -> String {
+    let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+    let addr = listener.local_addr().unwrap();
+    std::thread::spawn(move || {
+        for incoming in listener.incoming() {
+            let mut s = match incoming {
+                Ok(s) => s,
+                Err(_) => continue,
+            };
+            let mut buf = [0u8; 2048];
+            let _ = s.read(&mut buf);
+            let req = std::str::from_utf8(&buf).unwrap_or("");
+            let path = req.split_whitespace().nth(1).unwrap_or("/");
+            let (ct, body) = if path.starts_with("/api") {
+                ("application/json", "{\"hello\":\"world\"}".to_string())
+            } else {
+                ("text/html", "<script>fetch('/api');</script>".to_string())
+            };
+            let resp = format!(
+                "HTTP/1.1 200 OK\r\nContent-Type: {}\r\nContent-Length: {}\r\nAccess-Control-Allow-Origin: *\r\nConnection: close\r\n\r\n{}",
+                ct,
+                body.len(),
+                body
+            );
+            let _ = s.write_all(resp.as_bytes());
+            let _ = s.shutdown(std::net::Shutdown::Both);
+        }
+    });
+    format!("http://{}", addr)
+}
+
+#[tokio::test]
+async fn page_intercepts_and_observes_js_fetch() {
+    std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1");
+    let base = spawn_echo_server();
+
+    let browser = Browser::new().unwrap();
+    let mut page = browser.new_page().await.unwrap();
+
+    // Passive on_request counter (fires for navigation + the JS fetch).
+    let req_count = Arc::new(AtomicU32::new(0));
+    let rc = req_count.clone();
+    page.on_request(Arc::new(move |_info| {
+        rc.fetch_add(1, Ordering::SeqCst);
+    }));
+
+    // Passive on_response: capture the /api (Fetch) body.
+    let captured = Arc::new(Mutex::new(String::new()));
+    let cap = captured.clone();
+    page.on_response(Arc::new(move |info, resp| {
+        if info.resource_type == ResourceType::Fetch {
+            *cap.lock().unwrap() = String::from_utf8_lossy(&resp.body).into_owned();
+        }
+    }));
+
+    // Channel-based interception: resolve every request Continue so the fetch
+    // is not blocked.
+    let mut rx = page.enable_interception();
+    tokio::spawn(async move {
+        while let Some(req) = rx.recv().await {
+            if req
+                .resolver
+                .send(InterceptResolution::Continue {
+                    url: None,
+                    method: None,
+                    headers: None,
+                    body: None,
+                })
+                .is_err()
+            {
+                break;
+            }
+        }
+    });
+
+    page.goto(&base).await.unwrap();
+    // Pump the event loop so the inline fetch('/api') resolves.
+    for _ in 0..20 {
+        page.settle(500).await;
+        if captured.lock().unwrap().contains("hello") {
+            break;
+        }
+    }
+
+    assert!(
+        req_count.load(Ordering::SeqCst) >= 1,
+        "on_request never fired for navigation or fetch"
+    );
+    let body = captured.lock().unwrap().clone();
+    assert!(
+        body.contains("hello"),
+        "on_response did not capture the fetch response body: {:?}",
+        body
+    );
+}