feat(cdp): stream large response bodies via takeResponseBodyAsStream + IO (#381)
Network.getResponseBody returns the whole body in one base64 blob and the cache drops anything over 2 MiB, so large downloads (images, PDFs, archives) either failed or forced the long-running server to hold the full body in memory (issue #360). Add Fetch.takeResponseBodyAsStream + IO.read + IO.close. The body is moved out of the page cache into a stream handle and delivered in bounded chunks via IO.read, then freed on IO.close, so it is held once and released after the download. Binary bodies are byte-exact (stored base64). To stream a large body, raise OBSCURA_NETWORK_BODY_BUFFER_BYTES so it is cached. This is entirely opt-in: nothing runs unless a client calls the new methods, so the default fetch/render/scrape path is unchanged (verified: obstacle course 33/33, obscura-vs-Chrome head-to-head within noise of the pre-change baseline). Fixes #360.
This commit is contained in:
1 parent
d3a8b9a482
commit
1ca6047351
8 files changed
+123
-1
No files matched your search
@@ -45,6 +45,11 @@ pub struct CdpContext {
|
||||
pub next_isolated_context_id: i64,
|
||||
pub fetch_intercept: FetchInterceptState,
|
||||
pub intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<InterceptedRequest>>,
|
||||
// Open IO streams for Fetch.takeResponseBodyAsStream: handle -> (bytes, cursor).
|
||||
// Each holds a response body taken out of the page cache so a large download
|
||||
// is streamed chunk-by-chunk via IO.read and freed on IO.close (issue #360).
|
||||
pub io_streams: HashMap<String, (Vec<u8>, usize)>,
|
||||
pub io_stream_counter: u64,
|
||||
}
|
||||
|
||||
impl CdpContext {
|
||||
@@ -141,6 +146,8 @@ impl CdpContext {
|
||||
isolated_worlds: Vec::new(),
|
||||
valid_context_ids,
|
||||
next_isolated_context_id: 100,
|
||||
io_streams: HashMap::new(),
|
||||
io_stream_counter: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -334,6 +341,7 @@ pub async fn dispatch(req: &CdpRequest, ctx: &mut CdpContext) -> CdpResponse {
|
||||
"Runtime" => domains::runtime::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Network" => domains::network::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Fetch" => domains::fetch::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"IO" => domains::io::handle(method, &req.params, ctx).await,
|
||||
"Input" => domains::input::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Storage" => domains::storage::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"LP" => domains::lp::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
|
||||
Reference in new issue
Block a user