engine: large compat + reliability pass (#206)
CDP / navigation: - Route all Page.navigate through process_with_interception so the parent dispatcher stays responsive during long navs (fixes Target.createTarget timeouts after a handful of newPage calls). - Default waitUntil to domcontentloaded. Puppeteer and Playwright don't pass it server-side and wait client-side via lifecycleEvent, so the old Load default blocked nearly every nav on slow sites. - 30s navigation hard ceiling, OBSCURA_NAV_TIMEOUT_MS to override. - execute_scripts loop now checks wall-clock deadline per iteration. - Emit Network.requestWillBeSent before Page.frameNavigated so Playwright's FrameManager attaches the navigation response (#189). - Use loaderId as the Document request id so page.goto() resolves to a Response instead of null (#189). - Fresh executionContextId per navigation, no more 'Cannot find context with specified id' after the first nav (#192). - POST navigation params plumbed through the intercept path. DOM / Web API surface: - MutationObserver: subtree filter walks parent chain, fires on textContent / CharacterData.data / innerHTML, type filter respected, oldValue passed. - IntersectionObserver: multi-fire (initial + 120/500/1500ms), disconnect / unobserve / takeRecords, root / rootMargin / thresholds. - ResizeObserver: per-target firing from getBoundingClientRect, drop the no-op override that was clobbering the real class. - crypto.subtle.digest: real SHA-1/256/384/512 via op_subtle_digest. - DOMParser / document.implementation.createHTMLDocument / createDocument: detached doc with body, head, documentElement, querySelector*, getElementById, getElementsByTagName. - WebSocket / EventSource / BroadcastChannel: real EventTarget, open fires on microtask. - indexedDB: request, object store, transaction, database with async onsuccess and in-memory Map storage. - getComputedStyle: width/height/top/left from bbox so React virtualization libs render items. Site compatibility went from 3/40 to 175/175 reachable sites in the sweep. Perf is equal or faster than pristine main on every site tested (DCL default removes the block on long-tail subresources). 28/28 integration tests pass.
This commit is contained in:
1 parent
c9f32aa2f0
commit
b98e457428
9 files changed
+979
-295
No files matched your search
@@ -501,9 +501,21 @@ impl Page {
|
||||
}
|
||||
|
||||
if let Some(js) = &mut self.js {
|
||||
// Bound the post-script settle loop by wall clock, not just by the
|
||||
// 10ms-tick branch. The old code only consulted `deadline` inside
|
||||
// the `Err(_)` arm (when the inner tick timed out), so a steady
|
||||
// stream of inflight XHR/fetch (active_requests() > 0) kept the
|
||||
// loop running indefinitely because it took the `Ok(Ok(()))` arm
|
||||
// and slept 1ms each iteration without ever checking the clock.
|
||||
// On busy sites this could keep the V8 lock held for tens of
|
||||
// seconds, wedging the entire CDP dispatcher (see triage for
|
||||
// issue series around the 40-site compat sweep).
|
||||
let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_millis(500);
|
||||
let mut idle_count = 0u32;
|
||||
loop {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
break;
|
||||
}
|
||||
let result = tokio::time::timeout(
|
||||
tokio::time::Duration::from_millis(10),
|
||||
js.run_event_loop(),
|
||||
@@ -525,9 +537,6 @@ impl Page {
|
||||
Ok(Err(_)) => break,
|
||||
Err(_) => {
|
||||
idle_count = 0;
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -552,6 +561,43 @@ impl Page {
|
||||
wait_until: crate::lifecycle::WaitUntil,
|
||||
method: &str,
|
||||
body: &str,
|
||||
) -> Result<(), PageError> {
|
||||
// Hard ceiling on a single end-to-end navigation. Without this a slow
|
||||
// primary fetch or a runaway settle loop can hold the V8 lock for
|
||||
// arbitrarily long (we've measured 60+ seconds on JS-heavy news
|
||||
// sites), wedging every other in-flight CDP request because the
|
||||
// dispatcher holds the lock across the entire handler. 30 seconds
|
||||
// matches reqwest's default per-request timeout — the worst case is
|
||||
// one slow primary GET plus one slow JS-redirect chain step. Override
|
||||
// with `OBSCURA_NAV_TIMEOUT_MS=NN`.
|
||||
let nav_timeout_ms: u64 = std::env::var("OBSCURA_NAV_TIMEOUT_MS")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(30_000);
|
||||
let nav_timeout = tokio::time::Duration::from_millis(nav_timeout_ms);
|
||||
|
||||
match tokio::time::timeout(
|
||||
nav_timeout,
|
||||
self.navigate_with_wait_post_inner(url_str, wait_until, method, body),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(_) => {
|
||||
self.lifecycle = crate::lifecycle::LifecycleState::Failed;
|
||||
Err(PageError::NetworkError(format!(
|
||||
"navigation exceeded {nav_timeout_ms}ms deadline"
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn navigate_with_wait_post_inner(
|
||||
&mut self,
|
||||
url_str: &str,
|
||||
wait_until: crate::lifecycle::WaitUntil,
|
||||
method: &str,
|
||||
body: &str,
|
||||
) -> Result<(), PageError> {
|
||||
let mut current_url = url_str.to_string();
|
||||
let mut current_method = method.to_string();
|
||||
|
||||
Reference in new issue
Block a user