diff --git a/Cargo.lock b/Cargo.lock index 0a85eb0..6df302d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1683,6 +1683,8 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha1", + "sha2", "thiserror", "tokio", "tracing", @@ -2364,6 +2366,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" diff --git a/crates/obscura-browser/src/lib.rs b/crates/obscura-browser/src/lib.rs index 8439a9b..fd38ebf 100644 --- a/crates/obscura-browser/src/lib.rs +++ b/crates/obscura-browser/src/lib.rs @@ -2,7 +2,7 @@ pub mod page; pub mod context; pub mod lifecycle; -pub use page::{Page, PageError}; +pub use page::{NetworkEvent, Page, PageError}; pub use context::BrowserContext; pub use lifecycle::{LifecycleState, WaitUntil}; pub use obscura_js::HTML_TO_MARKDOWN_JS; diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs index dd72551..5d77876 100644 --- a/crates/obscura-browser/src/page.rs +++ b/crates/obscura-browser/src/page.rs @@ -501,9 +501,21 @@ impl Page { } if let Some(js) = &mut self.js { + // Bound the post-script settle loop by wall clock, not just by the + // 10ms-tick branch. The old code only consulted `deadline` inside + // the `Err(_)` arm (when the inner tick timed out), so a steady + // stream of inflight XHR/fetch (active_requests() > 0) kept the + // loop running indefinitely because it took the `Ok(Ok(()))` arm + // and slept 1ms each iteration without ever checking the clock. + // On busy sites this could keep the V8 lock held for tens of + // seconds, wedging the entire CDP dispatcher (see triage for + // issue series around the 40-site compat sweep). let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_millis(500); let mut idle_count = 0u32; loop { + if tokio::time::Instant::now() >= deadline { + break; + } let result = tokio::time::timeout( tokio::time::Duration::from_millis(10), js.run_event_loop(), @@ -525,9 +537,6 @@ impl Page { Ok(Err(_)) => break, Err(_) => { idle_count = 0; - if tokio::time::Instant::now() >= deadline { - break; - } } } } @@ -552,6 +561,43 @@ impl Page { wait_until: crate::lifecycle::WaitUntil, method: &str, body: &str, + ) -> Result<(), PageError> { + // Hard ceiling on a single end-to-end navigation. Without this a slow + // primary fetch or a runaway settle loop can hold the V8 lock for + // arbitrarily long (we've measured 60+ seconds on JS-heavy news + // sites), wedging every other in-flight CDP request because the + // dispatcher holds the lock across the entire handler. 30 seconds + // matches reqwest's default per-request timeout — the worst case is + // one slow primary GET plus one slow JS-redirect chain step. Override + // with `OBSCURA_NAV_TIMEOUT_MS=NN`. + let nav_timeout_ms: u64 = std::env::var("OBSCURA_NAV_TIMEOUT_MS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(30_000); + let nav_timeout = tokio::time::Duration::from_millis(nav_timeout_ms); + + match tokio::time::timeout( + nav_timeout, + self.navigate_with_wait_post_inner(url_str, wait_until, method, body), + ) + .await + { + Ok(r) => r, + Err(_) => { + self.lifecycle = crate::lifecycle::LifecycleState::Failed; + Err(PageError::NetworkError(format!( + "navigation exceeded {nav_timeout_ms}ms deadline" + ))) + } + } + } + + async fn navigate_with_wait_post_inner( + &mut self, + url_str: &str, + wait_until: crate::lifecycle::WaitUntil, + method: &str, + body: &str, ) -> Result<(), PageError> { let mut current_url = url_str.to_string(); let mut current_method = method.to_string(); diff --git a/crates/obscura-cdp/src/domains/page.rs b/crates/obscura-cdp/src/domains/page.rs index 4c1ab3c..f1d0bb8 100644 --- a/crates/obscura-cdp/src/domains/page.rs +++ b/crates/obscura-cdp/src/domains/page.rs @@ -5,13 +5,138 @@ use crate::dispatch::CdpContext; use crate::types::CdpEvent; use crate::util::url_is_file_scheme; -async fn do_navigate( - url: &str, - params: &Value, +/// Emit the post-navigation event stream into `ctx.pending_events`. Shared +/// by both the in-process `do_navigate` path and the spawned path in +/// `server::process_navigation`, so the recent goto-returns-Response / +/// per-isolated-world fixes don't have to be duplicated. +pub fn emit_navigation_events( ctx: &mut CdpContext, session_id: &Option, -) -> Result { - let wait_until = params.get("waitUntil") + frame_id: &str, + loader_id: &str, + page_url: &str, + page_id: &str, + network_events: &[obscura_browser::NetworkEvent], + wait_until: WaitUntil, + reached_network_idle: bool, +) { + let es = session_id.clone(); + let ts = timestamp(); + + // Real Chrome uses the navigation's loaderId as the main document's + // request id, and Puppeteer/Playwright identify the navigation response + // via `requestId === loaderId && type === "Document"` (issue #189). + let nav_request_ids: Vec = { + let mut nav_seen = false; + network_events.iter().map(|ev| { + if !nav_seen && ev.resource_type == "Document" && ev.url == page_url { + nav_seen = true; + loader_id.to_string() + } else { + ev.request_id.clone() + } + }).collect() + }; + let nav_idx: Option = network_events + .iter() + .position(|ev| ev.resource_type == "Document" && ev.url == page_url); + + // Playwright needs `Network.requestWillBeSent` for the main document to + // arrive BEFORE `Page.frameNavigated` (issue #190). + if let Some(idx) = nav_idx { + let net_event = &network_events[idx]; + let rid = &nav_request_ids[idx]; + ctx.pending_events.push(CdpEvent { + method: "Network.requestWillBeSent".into(), + params: json!({"requestId": rid, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}), + session_id: es.clone(), + }); + } + + let mut phase1 = vec![ + CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "init", "timestamp": ts}), session_id: es.clone() }, + CdpEvent { method: "Runtime.executionContextsCleared".into(), params: json!({}), session_id: es.clone() }, + CdpEvent { method: "Page.frameNavigated".into(), params: json!({"frame": {"id": frame_id, "loaderId": loader_id, "url": page_url, "domainAndRegistry": "", "securityOrigin": page_url, "mimeType": "text/html", "adFrameStatus": {"adFrameType": "none"}}, "type": "Navigation"}), session_id: es.clone() }, + CdpEvent { method: "Runtime.executionContextCreated".into(), params: json!({"context": {"id": 2, "origin": page_url, "name": "", "uniqueId": format!("ctx-nav-{}", page_id), "auxData": {"isDefault": true, "type": "default", "frameId": frame_id}}}), session_id: es.clone() }, + ]; + let world_names: Vec = if ctx.isolated_worlds.is_empty() { + vec!["__puppeteer_utility_world__24.40.0".to_string()] + } else { + ctx.isolated_worlds.clone() + }; + // Issue #192: fresh, monotonically increasing executionContextId per re-create. + for world_name in &world_names { + let world_ctx_id = ctx.next_isolated_context(); + phase1.push(CdpEvent { + method: "Runtime.executionContextCreated".into(), + params: json!({"context": {"id": world_ctx_id, "origin": page_url, "name": world_name, "uniqueId": format!("ctx-isolated-nav-{}-{}", page_id, world_ctx_id), "auxData": {"isDefault": false, "type": "isolated", "frameId": frame_id}}}), + session_id: es.clone(), + }); + } + phase1.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "commit", "timestamp": ts}), session_id: es.clone() }); + ctx.pending_events.extend(phase1); + + if ctx.fetch_intercept.enabled { + for (i, net_event) in network_events.iter().enumerate() { + let rid = &nav_request_ids[i]; + ctx.pending_events.push(CdpEvent { + method: "Fetch.requestPaused".into(), + params: json!({ + "requestId": rid, + "request": { + "url": net_event.url, + "method": net_event.method, + "headers": net_event.headers, + }, + "frameId": frame_id, + "resourceType": net_event.resource_type, + "networkId": rid, + }), + session_id: es.clone(), + }); + } + } + + for (i, net_event) in network_events.iter().enumerate() { + let rid = &nav_request_ids[i]; + if Some(i) != nav_idx { + ctx.pending_events.push(CdpEvent { + method: "Network.requestWillBeSent".into(), + params: json!({"requestId": rid, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}), + session_id: es.clone(), + }); + } + ctx.pending_events.push(CdpEvent { + method: "Network.responseReceived".into(), + params: json!({"requestId": rid, "loaderId": loader_id, "timestamp": net_event.timestamp, "type": net_event.resource_type, "response": {"url": net_event.url, "status": net_event.status, "statusText": "", "headers": &*net_event.response_headers, "mimeType": net_event.response_headers.get("content-type").cloned().unwrap_or_default()}, "frameId": frame_id}), + session_id: es.clone(), + }); + ctx.pending_events.push(CdpEvent { + method: "Network.loadingFinished".into(), + params: json!({"requestId": rid, "timestamp": net_event.timestamp, "encodedDataLength": net_event.body_size}), + session_id: es.clone(), + }); + } + + let mut phase3 = vec![ + CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "DOMContentLoaded", "timestamp": ts}), session_id: es.clone() }, + CdpEvent { method: "Page.domContentEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() }, + CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "load", "timestamp": ts}), session_id: es.clone() }, + CdpEvent { method: "Page.loadEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() }, + ]; + if reached_network_idle || matches!(wait_until, WaitUntil::Load | WaitUntil::DomContentLoaded) { + let idle_ts = timestamp(); + phase3.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "networkIdle", "timestamp": idle_ts}), session_id: es.clone() }); + } + phase3.push(CdpEvent { method: "Page.frameStoppedLoading".into(), params: json!({"frameId": frame_id}), session_id: es }); + ctx.pending_events.extend(phase3); +} + +/// Parse the `waitUntil` argument that Puppeteer/Playwright pass on +/// `Page.navigate`. +pub fn parse_wait_until(params: &Value) -> WaitUntil { + params + .get("waitUntil") .and_then(|v| { if let Some(s) = v.as_str() { Some(WaitUntil::from_str(s)) @@ -29,7 +154,27 @@ async fn do_navigate( None } }) - .unwrap_or(WaitUntil::Load); + // Puppeteer and Playwright drive navigation via `Page.navigate` + // without a server-side waitUntil — they wait for `Page.lifecycleEvent` + // on the client side. Defaulting the server to `Load` means we run + // every parser/deferred/async script on JS-heavy pages before + // emitting `load`, which on sites like github.com / reddit.com + // pushes nav past 25s and clients time out at 15s. Real Chrome + // streams `DOMContentLoaded` as soon as the parser is done; we + // batch our event emission at the end of navigation, so the + // closest we can get is to default to `DomContentLoaded` and skip + // the full-load wait. CLI callers that pass `--wait-until load` + // (or `networkidle*`) are unaffected; they get the old behaviour. + .unwrap_or(WaitUntil::DomContentLoaded) +} + +async fn do_navigate( + url: &str, + params: &Value, + ctx: &mut CdpContext, + session_id: &Option, +) -> Result { + let wait_until = parse_wait_until(params); // Block CDP-initiated file:// navigation by default. // Anyone who can reach the CDP port (default localhost, @@ -70,131 +215,17 @@ async fn do_navigate( (frame_id, loader_id, network_events, page_url, page_id, reached_network_idle) }; - let es = session_id.clone(); - let ts = timestamp(); - - // Real Chrome uses the navigation's loaderId as the main document's - // request id, and Puppeteer/Playwright identify the navigation response - // via `requestId === loaderId && type === "Document"` (issue #189). - // Override the first Document event's request id with loaderId so - // `page.goto()` resolves to the navigation Response instead of null. - let nav_request_ids: Vec = { - let mut nav_seen = false; - network_events.iter().map(|ev| { - if !nav_seen && ev.resource_type == "Document" && ev.url == page_url { - nav_seen = true; - loader_id.clone() - } else { - ev.request_id.clone() - } - }).collect() - }; - let nav_idx: Option = network_events - .iter() - .position(|ev| ev.resource_type == "Document" && ev.url == page_url); - - // Playwright needs `Network.requestWillBeSent` for the main document to - // arrive BEFORE `Page.frameNavigated`, otherwise its FrameManager commits - // the navigation with `currentDocument.request = void 0` and never - // attaches the response. Mirrors real Chrome's event order. We emit only - // the navigation request here; the response and subresource requests - // come after `frameNavigated` (still before `Page.lifecycleEvent: load`). - if let Some(idx) = nav_idx { - let net_event = &network_events[idx]; - let rid = &nav_request_ids[idx]; - ctx.pending_events.push(CdpEvent { - method: "Network.requestWillBeSent".into(), - params: json!({"requestId": rid, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}), - session_id: es.clone(), - }); - } - - let mut phase1 = vec![ - CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "init", "timestamp": ts}), session_id: es.clone() }, - CdpEvent { method: "Runtime.executionContextsCleared".into(), params: json!({}), session_id: es.clone() }, - CdpEvent { method: "Page.frameNavigated".into(), params: json!({"frame": {"id": frame_id, "loaderId": loader_id, "url": page_url, "domainAndRegistry": "", "securityOrigin": page_url, "mimeType": "text/html", "adFrameStatus": {"adFrameType": "none"}}, "type": "Navigation"}), session_id: es.clone() }, - CdpEvent { method: "Runtime.executionContextCreated".into(), params: json!({"context": {"id": 2, "origin": page_url, "name": "", "uniqueId": format!("ctx-nav-{}", page_id), "auxData": {"isDefault": true, "type": "default", "frameId": frame_id}}}), session_id: es.clone() }, - ]; - let world_names: Vec = if ctx.isolated_worlds.is_empty() { - vec!["__puppeteer_utility_world__24.40.0".to_string()] - } else { - ctx.isolated_worlds.clone() - }; - // Issue #192: real Chrome emits a fresh, monotonically increasing - // executionContextId on every re-creation. Reusing ids 100, 101, … - // across navigations made Playwright's client-side counter and our - // server-side `valid_context_ids` diverge after the first nav, so - // Runtime.evaluate failed with "Cannot find context with specified - // id: 101". Each post-nav isolated world now claims a fresh id from - // the same counter that backs Page.createIsolatedWorld. - for world_name in &world_names { - let world_ctx_id = ctx.next_isolated_context(); - phase1.push(CdpEvent { - method: "Runtime.executionContextCreated".into(), - params: json!({"context": {"id": world_ctx_id, "origin": page_url, "name": world_name, "uniqueId": format!("ctx-isolated-nav-{}-{}", page_id, world_ctx_id), "auxData": {"isDefault": false, "type": "isolated", "frameId": frame_id}}}), - session_id: es.clone(), - }); - } - phase1.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "commit", "timestamp": ts}), session_id: es.clone() }); - ctx.pending_events.extend(phase1); - - if ctx.fetch_intercept.enabled { - for (i, net_event) in network_events.iter().enumerate() { - let rid = &nav_request_ids[i]; - ctx.pending_events.push(CdpEvent { - method: "Fetch.requestPaused".into(), - params: json!({ - "requestId": rid, - "request": { - "url": net_event.url, - "method": net_event.method, - "headers": net_event.headers, - }, - "frameId": frame_id, - "resourceType": net_event.resource_type, - "networkId": rid, - }), - session_id: es.clone(), - }); - } - } - - for (i, net_event) in network_events.iter().enumerate() { - let rid = &nav_request_ids[i]; - // Document's requestWillBeSent was already emitted above (before - // Page.frameNavigated) so Playwright can attach the request to the - // pending document. Skip re-emitting it here. - if Some(i) != nav_idx { - ctx.pending_events.push(CdpEvent { - method: "Network.requestWillBeSent".into(), - params: json!({"requestId": rid, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}), - session_id: es.clone(), - }); - } - ctx.pending_events.push(CdpEvent { - method: "Network.responseReceived".into(), - params: json!({"requestId": rid, "loaderId": loader_id, "timestamp": net_event.timestamp, "type": net_event.resource_type, "response": {"url": net_event.url, "status": net_event.status, "statusText": "", "headers": &*net_event.response_headers, "mimeType": net_event.response_headers.get("content-type").cloned().unwrap_or_default()}, "frameId": frame_id}), - session_id: es.clone(), - }); - ctx.pending_events.push(CdpEvent { - method: "Network.loadingFinished".into(), - params: json!({"requestId": rid, "timestamp": net_event.timestamp, "encodedDataLength": net_event.body_size}), - session_id: es.clone(), - }); - } - - let mut phase3 = vec![ - CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "DOMContentLoaded", "timestamp": ts}), session_id: es.clone() }, - CdpEvent { method: "Page.domContentEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() }, - CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "load", "timestamp": ts}), session_id: es.clone() }, - CdpEvent { method: "Page.loadEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() }, - ]; - if reached_network_idle || matches!(wait_until, WaitUntil::Load | WaitUntil::DomContentLoaded) { - let idle_ts = timestamp(); - phase3.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "networkIdle", "timestamp": idle_ts}), session_id: es.clone() }); - } - phase3.push(CdpEvent { method: "Page.frameStoppedLoading".into(), params: json!({"frameId": frame_id}), session_id: es }); - ctx.pending_events.extend(phase3); + emit_navigation_events( + ctx, + session_id, + &frame_id, + &loader_id, + &page_url, + &page_id, + &network_events, + wait_until, + reached_network_idle, + ); Ok(json!({ "frameId": frame_id, diff --git a/crates/obscura-cdp/src/server.rs b/crates/obscura-cdp/src/server.rs index a0501f0..6eecb4c 100644 --- a/crates/obscura-cdp/src/server.rs +++ b/crates/obscura-cdp/src/server.rs @@ -381,10 +381,17 @@ async fn cdp_processor( ); } ServerMessage::Cdp(cdp_msg) => { + // Route every Page.navigate through the spawn-and-defer path, + // not just intercepted ones. Holding the V8 lock across a + // multi-second navigate inside the regular dispatch wedges the + // entire processor (40-site sweep: 39/40 timeouts). Spawning + // navigation lets `cdp_processor` keep multiplexing other CDP + // messages via the `process_with_interception` select loop; + // unrelated requests get deferred only briefly and are drained + // as soon as the nav settles. let is_navigation = cdp_msg.text.contains("Page.navigate"); - let has_interception = ctx.fetch_intercept.enabled; - if is_navigation && has_interception { + if is_navigation { process_with_interception( &cdp_msg.text, &mut ctx, &cdp_msg.reply_tx, &mut rx, &mut intercept_rx, &mut intercepted_paused, @@ -509,25 +516,9 @@ async fn process_with_interception( } let url = req.params.get("url").and_then(|v| v.as_str()).unwrap_or(""); - let wait_until = req.params.get("waitUntil") - .and_then(|v| { - if let Some(s) = v.as_str() { - Some(obscura_browser::WaitUntil::from_str(s)) - } else if let Some(arr) = v.as_array() { - arr.iter() - .filter_map(|item| item.as_str()) - .map(obscura_browser::WaitUntil::from_str) - .max_by_key(|w| match w { - obscura_browser::WaitUntil::DomContentLoaded => 0, - obscura_browser::WaitUntil::Load => 1, - obscura_browser::WaitUntil::NetworkIdle2 => 2, - obscura_browser::WaitUntil::NetworkIdle0 => 3, - }) - } else { - None - } - }) - .unwrap_or(obscura_browser::WaitUntil::Load); + let wait_until = crate::domains::page::parse_wait_until(&req.params); + let nav_method = req.params.get("__method").and_then(|v| v.as_str()).unwrap_or("GET").to_string(); + let nav_body = req.params.get("__body").and_then(|v| v.as_str()).unwrap_or("").to_string(); let preload_scripts: Vec = ctx.preload_scripts.iter().map(|(_, s)| s.clone()).collect(); @@ -552,7 +543,12 @@ async fn process_with_interception( // must run BEFORE the page's own scripts (CDP contract). Hand them // to the page so navigate_single can inject them at the right point. page.set_preload_scripts(preload_scripts); - let result = page.navigate_with_wait(&url_owned, wait_until).await.map_err(|e| e.to_string()); + let result = if nav_method == "POST" && !nav_body.is_empty() { + page.navigate_with_wait_post(&url_owned, wait_until, &nav_method, &nav_body).await + } else { + page.navigate_with_wait(&url_owned, wait_until).await + } + .map_err(|e| e.to_string()); drop(_v8_guard); let _ = nav_done_tx.send((page, result)).await; }); @@ -722,43 +718,27 @@ async fn process_with_interception( let _ = reply_tx.send(json); } - let ts = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_secs_f64(); - let es = session_for_events; - - for event in [ - crate::types::CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "init", "timestamp": ts}), session_id: es.clone() }, - crate::types::CdpEvent { method: "Runtime.executionContextsCleared".into(), params: json!({}), session_id: es.clone() }, - crate::types::CdpEvent { method: "Page.frameNavigated".into(), params: json!({"frame": {"id": frame_id, "loaderId": loader_id, "url": page_url, "domainAndRegistry": "", "securityOrigin": page_url, "mimeType": "text/html", "adFrameStatus": {"adFrameType": "none"}}, "type": "Navigation"}), session_id: es.clone() }, - crate::types::CdpEvent { method: "Runtime.executionContextCreated".into(), params: json!({"context": {"id": 2, "origin": page_url, "name": "", "uniqueId": format!("ctx-nav-{}", page_id_for_events), "auxData": {"isDefault": true, "type": "default", "frameId": frame_id}}}), session_id: es.clone() }, - crate::types::CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "commit", "timestamp": ts}), session_id: es.clone() }, - ] { - if let Ok(json) = serde_json::to_string(&event) { let _ = reply_tx.send(json); } - } - - for net_event in &network_events { - for event in [ - crate::types::CdpEvent { method: "Network.requestWillBeSent".into(), params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}), session_id: es.clone() }, - crate::types::CdpEvent { method: "Network.responseReceived".into(), params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "timestamp": net_event.timestamp, "type": net_event.resource_type, "response": {"url": net_event.url, "status": net_event.status, "statusText": "", "headers": &*net_event.response_headers, "mimeType": ""}, "frameId": frame_id}), session_id: es.clone() }, - crate::types::CdpEvent { method: "Network.loadingFinished".into(), params: json!({"requestId": net_event.request_id, "timestamp": net_event.timestamp, "encodedDataLength": net_event.body_size}), session_id: es.clone() }, - ] { - if let Ok(json) = serde_json::to_string(&event) { let _ = reply_tx.send(json); } + // Shared event emission: includes the post-#190 Network.requestWillBeSent + // -before-frameNavigated ordering, the #189 requestId=loaderId trick that + // makes `page.goto()` resolve to a Response, and the #192 per-isolated- + // world fresh context ids. Pushes to `ctx.pending_events`; we then drain + // to the WS reply channel. + crate::domains::page::emit_navigation_events( + ctx, + &session_for_events, + &frame_id, + &loader_id, + &page_url, + &page_id_for_events, + &network_events, + wait_until, + reached_network_idle, + ); + for event in ctx.pending_events.drain(..) { + if let Ok(json) = serde_json::to_string(&event) { + let _ = reply_tx.send(json); } } - - for event in [ - crate::types::CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "DOMContentLoaded", "timestamp": ts}), session_id: es.clone() }, - crate::types::CdpEvent { method: "Page.domContentEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() }, - crate::types::CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "load", "timestamp": ts}), session_id: es.clone() }, - crate::types::CdpEvent { method: "Page.loadEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() }, - ] { - if let Ok(json) = serde_json::to_string(&event) { let _ = reply_tx.send(json); } - } - if reached_network_idle { - let idle_event = crate::types::CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "networkIdle", "timestamp": ts}), session_id: es.clone() }; - if let Ok(json) = serde_json::to_string(&idle_event) { let _ = reply_tx.send(json); } - } - let stop_event = crate::types::CdpEvent { method: "Page.frameStoppedLoading".into(), params: json!({"frameId": frame_id}), session_id: es }; - if let Ok(json) = serde_json::to_string(&stop_event) { let _ = reply_tx.send(json); } } async fn process_cdp_message( diff --git a/crates/obscura-cdp/tests/cdp_click_submit_parity.rs b/crates/obscura-cdp/tests/cdp_click_submit_parity.rs index b511222..1516bed 100644 --- a/crates/obscura-cdp/tests/cdp_click_submit_parity.rs +++ b/crates/obscura-cdp/tests/cdp_click_submit_parity.rs @@ -93,7 +93,12 @@ async fn runtime_click_submit_prevent_default_navigation_updates_page() { &mut ctx, 1, "Page.navigate", - json!({"url": url}), + // Explicit `waitUntil: 'load'` so the inline `