thebadking/obscura
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add Network.getResponseBody support

Browse Source
This commit is contained in:
Rohit Madhu committed 2026-06-13 20:26:26 -04:00
1 parent af2f89a03e
commit 570ca1544c
6 files changed
+276 -8
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/obscura-browser/src/page.rs
+104 -5
View File
@@ -140,6 +140,12 @@ pub struct NetworkEvent {
pub timestamp: f64,
}
#[derive(Debug, Clone)]
pub struct StoredResponseBody {
pub body: String,
pub base64_encoded: bool,
}
pub struct Page {
pub id: String,
pub frame_id: String,
@@ -161,6 +167,8 @@ pub struct Page {
pub history: Vec<String>,
pub history_index: usize,
pub network_events: Vec<NetworkEvent>,
response_bodies: std::collections::HashMap<String, StoredResponseBody>,
response_body_order: std::collections::VecDeque<String>,
network_event_counter: u32,
pub intercept_enabled: bool,
pub intercept_block_patterns: Vec<String>,
@@ -213,6 +221,8 @@ impl Page {
history: Vec::new(),
history_index: 0,
network_events: Vec::new(),
response_bodies: std::collections::HashMap::new(),
response_body_order: std::collections::VecDeque::new(),
network_event_counter: 0,
intercept_enabled: false,
intercept_block_patterns: Vec::new(),
@@ -591,7 +601,7 @@ impl Page {
if script.src.is_some() {
if let Some((url, code, resp)) = fetched.remove(&i) {
tracing::info!("Executing script ({} bytes): {}", code.len(), url);
self.record_network_event(&url, "GET", "Script", resp.status, &resp.headers, resp.body.len());
self.record_network_event_with_body(&url, "GET", "Script", resp.status, &resp.headers, &resp.body, false);
if let Some(js) = &mut self.js {
let _ = js.execute_script("<current-script>", &format!("globalThis.__currentScriptNid={};", script.nid));
if let Err(e) = js.execute_script_guarded(&url, &code) {
@@ -963,13 +973,14 @@ impl Page {
PageError::NetworkError(e.to_string())
})?;
self.record_network_event(
self.record_network_event_with_body(
url.as_str(),
"GET",
"Document",
response.status,
&response.headers,
response.body.len(),
&response.body,
false,
);
if !response.redirected_from.is_empty() {
@@ -1063,7 +1074,7 @@ impl Page {
// CSS bodies: honor the Content-Type charset; CSS @charset is
// out of scope for the current scrape-focused pipeline.
let css = obscura_net::decode_non_html(&resp.body, resp.content_type());
self.record_network_event(&url_str, "GET", "Stylesheet", resp.status, &resp.headers, resp.body.len());
self.record_network_event_with_body(&url_str, "GET", "Stylesheet", resp.status, &resp.headers, &resp.body, false);
css_sources.push(css);
}
}
@@ -1356,13 +1367,47 @@ impl Page {
response_headers: &std::collections::HashMap<String, String>,
body_size: usize,
) {
self.record_network_event_inner(url, method, resource_type, status, response_headers, body_size);
}
fn record_network_event_with_body(
&mut self,
url: &str,
method: &str,
resource_type: &str,
status: u16,
response_headers: &std::collections::HashMap<String, String>,
body: &[u8],
base64_encoded: bool,
) {
let request_id = self.record_network_event_inner(
url,
method,
resource_type,
status,
response_headers,
body.len(),
);
self.store_response_body(request_id, body, base64_encoded);
}
fn record_network_event_inner(
&mut self,
url: &str,
method: &str,
resource_type: &str,
status: u16,
response_headers: &std::collections::HashMap<String, String>,
body_size: usize,
) -> String {
self.network_event_counter += 1;
let request_id = format!("{}.{}", self.id, self.network_event_counter);
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs_f64();
self.network_events.push(NetworkEvent {
request_id: format!("{}.{}", self.id, self.network_event_counter),
request_id: request_id.clone(),
url: url.to_string(),
method: method.to_string(),
resource_type: resource_type.to_string(),
@@ -1372,6 +1417,46 @@ impl Page {
body_size,
timestamp,
});
request_id
}
fn store_response_body(&mut self, request_id: String, body: &[u8], base64_encoded: bool) {
let max_entries = response_body_entry_limit();
let max_bytes = response_body_byte_limit();
if max_entries == 0 || max_bytes == 0 || body.len() > max_bytes {
return;
}
let body = if base64_encoded {
BASE64.encode(body)
} else {
String::from_utf8_lossy(body).to_string()
};
self.response_bodies.insert(request_id.clone(), StoredResponseBody { body, base64_encoded });
self.response_body_order.push_back(request_id);
while self.response_body_order.len() > max_entries {
if let Some(oldest) = self.response_body_order.pop_front() {
self.response_bodies.remove(&oldest);
}
}
}
pub fn get_response_body(&self, request_id: &str) -> Option<StoredResponseBody> {
self.response_bodies.get(request_id).cloned().or_else(|| {
self.js.as_ref()?.get_network_response_body(request_id).map(|body| {
StoredResponseBody {
body: body.body,
base64_encoded: body.base64_encoded,
}
})
})
}
pub fn clear_response_bodies(&mut self) {
self.response_bodies.clear();
self.response_body_order.clear();
if let Some(js) = &self.js {
js.clear_network_response_bodies();
}
}
pub fn execute_preload_script(&mut self, source: &str) -> Result<(), String> {
@@ -1478,3 +1563,17 @@ impl From<ObscuraNetError> for PageError {
PageError::NetworkError(e.to_string())
}
}
fn response_body_entry_limit() -> usize {
std::env::var("OBSCURA_NETWORK_BODY_BUFFER_ENTRIES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(128)
}
fn response_body_byte_limit() -> usize {
std::env::var("OBSCURA_NETWORK_BODY_BUFFER_BYTES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(2 * 1024 * 1024)
}
crates/obscura-cdp/src/dispatch.rs
+1
View File
@@ -238,6 +238,7 @@ fn is_v8_free_method(method: &str) -> bool {
| "Network.getCookies" | "Network.getAllCookies"
| "Network.setCookie" | "Network.setCookies"
| "Network.deleteCookies" | "Network.clearBrowserCookies"
| "Network.getResponseBody"
| "Fetch.continueRequest" | "Fetch.fulfillRequest"
| "Fetch.failRequest" | "Fetch.getResponseBody"
| "Storage.getCookies" | "Storage.setCookies" | "Storage.deleteCookies"
crates/obscura-cdp/src/domains/network.rs
+99
View File
@@ -32,6 +32,16 @@ pub async fn handle(
) -> Result<Value, String> {
match method {
"enable" => Ok(json!({})),
"disable" => {
if let Some(page) = ctx.get_session_page_mut(session_id) {
page.clear_response_bodies();
} else {
for page in &mut ctx.pages {
page.clear_response_bodies();
}
}
Ok(json!({}))
}
"setExtraHTTPHeaders" => {
let headers = params.get("headers").and_then(|v| v.as_object());
if let Some(page) = ctx.get_session_page(session_id) {
@@ -86,6 +96,26 @@ pub async fn handle(
}
"setCacheDisabled" => Ok(json!({})),
"setRequestInterception" => Ok(json!({})),
"getResponseBody" => {
let request_id = params
.get("requestId")
.and_then(|v| v.as_str())
.ok_or("Network.getResponseBody requires requestId")?;
let body = if let Some(page) = ctx.get_session_page(session_id) {
page.get_response_body(request_id)
} else {
ctx.pages.iter().find_map(|page| page.get_response_body(request_id))
};
match body {
Some(body) => Ok(json!({
"body": body.body,
"base64Encoded": body.base64_encoded,
})),
None => Err(format!("No response body found for requestId {}", request_id)),
}
}
_ => Err(format!("Unknown Network method: {}", method)),
}
}
@@ -193,6 +223,75 @@ mod tests {
.expect("clearBrowserCookies must succeed");
assert!(ctx.default_context.cookie_jar.get_all_cookies().is_empty());
}
#[tokio::test]
async fn get_response_body_returns_stored_document_body() {
let mut ctx = CdpContext::new();
let page_id = ctx.create_page();
let session_id = Some("session-1".to_string());
ctx.sessions.insert(session_id.clone().unwrap(), page_id.clone());
let page = ctx.get_page_mut(&page_id).unwrap();
page.navigate("data:text/html,<html><body>hello body</body></html>")
.await
.unwrap();
let request_id = page.network_events[0].request_id.clone();
let result = handle(
"getResponseBody",
&json!({ "requestId": request_id }),
&mut ctx,
&session_id,
)
.await
.unwrap();
assert_eq!(result["body"], "<html><body>hello body</body></html>");
assert_eq!(result["base64Encoded"], false);
}
#[tokio::test]
async fn get_response_body_errors_for_unknown_request_id() {
let mut ctx = CdpContext::new();
let err = handle(
"getResponseBody",
&json!({ "requestId": "missing" }),
&mut ctx,
&None,
)
.await
.unwrap_err();
assert!(err.contains("missing"));
}
#[tokio::test]
async fn network_disable_clears_stored_response_bodies() {
let mut ctx = CdpContext::new();
let page_id = ctx.create_page();
let session_id = Some("session-1".to_string());
ctx.sessions.insert(session_id.clone().unwrap(), page_id.clone());
let page = ctx.get_page_mut(&page_id).unwrap();
page.navigate("data:text/html,<html><body>temporary body</body></html>")
.await
.unwrap();
let request_id = page.network_events[0].request_id.clone();
handle("disable", &json!({}), &mut ctx, &session_id)
.await
.unwrap();
let err = handle(
"getResponseBody",
&json!({ "requestId": request_id }),
&mut ctx,
&session_id,
)
.await
.unwrap_err();
assert!(err.contains("No response body found"));
}
}
pub(crate) fn cookie_info_to_cdp_json(c: &obscura_net::CookieInfo) -> Value {
crates/obscura-js/js/bootstrap.js
Vendored
+8 -1
View File
@@ -2897,7 +2897,7 @@ globalThis.fetch = async (input, init = {}) => {
}
const respType = parsed.status === 0 ? "opaque" : (fetchMode === "no-cors" ? "opaque" : "basic");
const responseBody = parsed.bodyBase64 ? _base64ToUint8Array(parsed.bodyBase64) : (parsed.body || "");
return new Response(responseBody, {
const response = new Response(responseBody, {
status: parsed.status,
statusText: "",
headers: parsed.headers || {},
@@ -2905,6 +2905,13 @@ globalThis.fetch = async (input, init = {}) => {
url: parsed.url || url,
redirected: false,
});
if (parsed.requestId) {
Object.defineProperty(response, "__obscuraRequestId", {
value: parsed.requestId,
configurable: true,
});
}
return response;
};
if (typeof Headers === "undefined") {
crates/obscura-js/src/ops.rs
+53 -1
View File
@@ -1,5 +1,5 @@
use std::cell::RefCell;
use std::collections::HashMap;
use std::collections::{HashMap, VecDeque};
use std::rc::Rc;
use std::sync::Arc;
@@ -38,6 +38,12 @@ pub struct InterceptedRequest {
pub resolver: tokio::sync::oneshot::Sender<InterceptResolution>,
}
#[derive(Debug, Clone)]
pub struct StoredNetworkResponseBody {
pub body: String,
pub base64_encoded: bool,
}
pub struct ObscuraState {
pub dom: Option<DomTree>,
pub url: String,
@@ -57,6 +63,9 @@ pub struct ObscuraState {
// `op_binding_called` op. Drained by the CDP layer after each dispatch
// and emitted as `Runtime.bindingCalled` events.
pub pending_binding_calls: Vec<(String, String)>,
pub network_response_bodies: HashMap<String, StoredNetworkResponseBody>,
pub network_response_body_order: VecDeque<String>,
pub network_response_body_counter: u64,
}
impl ObscuraState {
@@ -74,10 +83,27 @@ impl ObscuraState {
intercept_counter: 0,
intercept_enabled: false,
pending_binding_calls: Vec::new(),
network_response_bodies: HashMap::new(),
network_response_body_order: VecDeque::new(),
network_response_body_counter: 0,
}
}
}
fn response_body_entry_limit() -> usize {
std::env::var("OBSCURA_NETWORK_BODY_BUFFER_ENTRIES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(128)
}
fn response_body_byte_limit() -> usize {
std::env::var("OBSCURA_NETWORK_BODY_BUFFER_BYTES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(2 * 1024 * 1024)
}
pub type SharedState = Rc<RefCell<ObscuraState>>;
#[op2]
@@ -872,6 +898,31 @@ async fn op_fetch_url(
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?;
let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
let resp_body_base64 = BASE64.encode(&resp_bytes);
let response_request_id = {
let state_borrow = state.borrow();
let gs = state_borrow.borrow::<SharedState>().clone();
let mut gs = gs.borrow_mut();
gs.network_response_body_counter += 1;
let request_id = format!("fetch-{}", gs.network_response_body_counter);
let max_entries = response_body_entry_limit();
let max_bytes = response_body_byte_limit();
if max_entries > 0 && max_bytes > 0 && resp_bytes.len() <= max_bytes {
gs.network_response_bodies.insert(
request_id.clone(),
StoredNetworkResponseBody {
body: resp_body.clone(),
base64_encoded: false,
},
);
gs.network_response_body_order.push_back(request_id.clone());
while gs.network_response_body_order.len() > max_entries {
if let Some(oldest) = gs.network_response_body_order.pop_front() {
gs.network_response_bodies.remove(&oldest);
}
}
}
request_id
};
tracing::debug!("op_fetch_url completed: {} {} ({} bytes)", method, url, resp_body.len());
@@ -879,6 +930,7 @@ async fn op_fetch_url(
"status": status,
"body": resp_body,
"bodyBase64": resp_body_base64,
"requestId": response_request_id,
"url": url,
"headers": resp_headers,
})
crates/obscura-js/src/runtime.rs
+11 -1
View File
@@ -10,7 +10,7 @@ use obscura_dom::DomTree;
pub use deno_core::v8::IsolateHandle;
use crate::module_loader::ObscuraModuleLoader;
use crate::ops::{build_extension, ObscuraState};
use crate::ops::{build_extension, ObscuraState, StoredNetworkResponseBody};
static SNAPSHOT: &[u8] = include_bytes!(env!("OBSCURA_SNAPSHOT_PATH"));
@@ -184,6 +184,16 @@ impl ObscuraJsRuntime {
std::mem::take(&mut self.state.borrow_mut().pending_binding_calls)
}
pub fn get_network_response_body(&self, request_id: &str) -> Option<StoredNetworkResponseBody> {
self.state.borrow().network_response_bodies.get(request_id).cloned()
}
pub fn clear_network_response_bodies(&self) {
let mut state = self.state.borrow_mut();
state.network_response_bodies.clear();
state.network_response_body_order.clear();
}
/// Wire up the interception channel without enabling interception.
/// Use set_intercept_enabled separately. The two were entangled before
/// and every navigation auto-enabled interception, which made