feat(cdp): stream large response bodies via takeResponseBodyAsStream + IO (#381)

Network.getResponseBody returns the whole body in one base64 blob and the
cache drops anything over 2 MiB, so large downloads (images, PDFs,
archives) either failed or forced the long-running server to hold the
full body in memory (issue #360).

Add Fetch.takeResponseBodyAsStream + IO.read + IO.close. The body is moved
out of the page cache into a stream handle and delivered in bounded chunks
via IO.read, then freed on IO.close, so it is held once and released after
the download. Binary bodies are byte-exact (stored base64). To stream a
large body, raise OBSCURA_NETWORK_BODY_BUFFER_BYTES so it is cached.

This is entirely opt-in: nothing runs unless a client calls the new
methods, so the default fetch/render/scrape path is unchanged (verified:
obstacle course 33/33, obscura-vs-Chrome head-to-head within noise of the
pre-change baseline).

Fixes #360.
This commit is contained in:
SGavrl authored and GitHub committed 2026-07-10 20:38:28 +00:00
1 parent d3a8b9a482
commit 1ca6047351
8 files changed
+123 -1

No files matched your search

+8
View File
@@ -45,6 +45,11 @@ pub struct CdpContext {
pub next_isolated_context_id: i64,
pub fetch_intercept: FetchInterceptState,
pub intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<InterceptedRequest>>,
// Open IO streams for Fetch.takeResponseBodyAsStream: handle -> (bytes, cursor).
// Each holds a response body taken out of the page cache so a large download
// is streamed chunk-by-chunk via IO.read and freed on IO.close (issue #360).
pub io_streams: HashMap<String, (Vec<u8>, usize)>,
pub io_stream_counter: u64,
}
impl CdpContext {
@@ -141,6 +146,8 @@ impl CdpContext {
isolated_worlds: Vec::new(),
valid_context_ids,
next_isolated_context_id: 100,
io_streams: HashMap::new(),
io_stream_counter: 0,
}
}
@@ -334,6 +341,7 @@ pub async fn dispatch(req: &CdpRequest, ctx: &mut CdpContext) -> CdpResponse {
"Runtime" => domains::runtime::handle(method, &req.params, ctx, &req.session_id).await,
"Network" => domains::network::handle(method, &req.params, ctx, &req.session_id).await,
"Fetch" => domains::fetch::handle(method, &req.params, ctx, &req.session_id).await,
"IO" => domains::io::handle(method, &req.params, ctx).await,
"Input" => domains::input::handle(method, &req.params, ctx, &req.session_id).await,
"Storage" => domains::storage::handle(method, &req.params, ctx, &req.session_id).await,
"LP" => domains::lp::handle(method, &req.params, ctx, &req.session_id).await,