fix(browser,cdp): correct document lifecycle ordering and add missing DOM methods

document.readyState was hardcoded to "complete" so scripts checking
readyState === 'loading' always took the wrong branch. Fixed by using a
JS global (__documentReadyState__) that Rust sets to "loading" before
any scripts execute, "interactive" before DOMContentLoaded fires, and
"complete" after load.

document.addEventListener and document.dispatchEvent on Document were
no-ops, so DOMContentLoaded listeners registered by inline scripts were
never called. Added proper listener storage and dispatch on Document.

Adds document.getElementsByName() (was missing entirely).

Page.reload is now implemented (re-navigates to the current URL),
resolving "Unknown Page method: reload" errors from Playwright.

data: URL navigation now works, fixing Playwright's goto("data:text/html,...")
pattern that previously errored with "invalid format".

Page.navigate logic extracted into do_navigate() so reload can reuse it.

Fixes #137, #146 (SPA bootstrap lifecycle ordering).
This commit is contained in:
SGavrl committed 2026-05-18 09:50:23 +00:00
1 parent e371e942fc
commit 913e6a4865
5 files changed
+249 -154

No files matched your search

+1
View File
@@ -18,3 +18,4 @@ tracing = { workspace = true }
thiserror = { workspace = true }
anyhow = { workspace = true }
serde_json = { workspace = true }
base64 = { workspace = true }
+70 -4
View File
@@ -1,5 +1,6 @@
use std::sync::Arc;
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
use obscura_dom::{parse_html, DomTree};
use obscura_js::runtime::ObscuraJsRuntime;
use obscura_net::{ObscuraHttpClient, ObscuraNetError, Response};
@@ -8,6 +9,48 @@ use url::Url;
use crate::context::BrowserContext;
use crate::lifecycle::LifecycleState;
fn decode_data_uri(uri: &str) -> Option<Vec<u8>> {
let rest = uri.strip_prefix("data:")?;
let comma = rest.find(',')?;
let meta = &rest[..comma];
let payload = &rest[comma + 1..];
if meta.split(';').any(|t| t.eq_ignore_ascii_case("base64")) {
let cleaned: String = payload.chars().filter(|c| !c.is_whitespace()).collect();
BASE64.decode(cleaned).ok()
} else {
Some(percent_decode(payload))
}
}
fn percent_decode(s: &str) -> Vec<u8> {
let mut out = Vec::with_capacity(s.len());
let b = s.as_bytes();
let mut i = 0;
while i < b.len() {
if b[i] == b'%' && i + 2 < b.len() {
let hi = hex_val(b[i + 1]);
let lo = hex_val(b[i + 2]);
if let (Some(h), Some(l)) = (hi, lo) {
out.push((h << 4) | l);
i += 3;
continue;
}
}
out.push(b[i]);
i += 1;
}
out
}
fn hex_val(b: u8) -> Option<u8> {
match b {
b'0'..=b'9' => Some(b - b'0'),
b'a'..=b'f' => Some(b - b'a' + 10),
b'A'..=b'F' => Some(b - b'A' + 10),
_ => None,
}
}
#[cfg(feature = "stealth")]
use obscura_net::StealthHttpClient;
@@ -364,6 +407,13 @@ impl Page {
}
}
// Spec: readyState is "loading" while parser-discovered scripts execute.
// Scripts that check readyState === 'loading' will register DOMContentLoaded
// listeners instead of calling their callback immediately.
if let Some(js) = &mut self.js {
let _ = js.execute_script("<ready-state>", "globalThis.__documentReadyState__ = 'loading';");
}
for (i, script) in all_to_execute.iter().enumerate() {
if script.src.is_some() {
if let Some((url, code, resp)) = fetched.remove(&i) {
@@ -417,10 +467,15 @@ impl Page {
}
if let Some(js) = &mut self.js {
// Spec order: readyState -> interactive, fire DOMContentLoaded on both
// document and window, then readyState -> complete, fire load.
let _ = js.execute_script("<load-events>",
"if (typeof window.onload === 'function') { try { window.onload(); } catch(e) {} }\n\
try { document.dispatchEvent(new Event('DOMContentLoaded')); } catch(e) {}\n\
try { window.dispatchEvent(new Event('load')); } catch(e) {}");
"globalThis.__documentReadyState__ = 'interactive';\n\
try { document.dispatchEvent(new Event('DOMContentLoaded', {bubbles:false,cancelable:false})); } catch(e) {}\n\
try { window.dispatchEvent(new Event('DOMContentLoaded', {bubbles:false,cancelable:false})); } catch(e) {}\n\
if (typeof window.onload === 'function') { try { window.onload(); } catch(e) {} }\n\
globalThis.__documentReadyState__ = 'complete';\n\
try { window.dispatchEvent(new Event('load', {bubbles:false,cancelable:false})); } catch(e) {}");
}
if let Some(js) = &mut self.js {
@@ -556,7 +611,18 @@ impl Page {
}
}
let response = if method == "POST" {
let response = if url.scheme() == "data" {
let content_type = url_str.strip_prefix("data:")
.and_then(|s| s.split(',').next())
.unwrap_or("text/html")
.split(';').next()
.unwrap_or("text/html")
.to_string();
let body_bytes = decode_data_uri(url_str).unwrap_or_default();
let mut headers = std::collections::HashMap::new();
headers.insert("content-type".to_string(), content_type);
Ok(obscura_net::Response { url: url.clone(), status: 200, headers, body: body_bytes, redirected_from: Vec::new() })
} else if method == "POST" {
self.http_client.post_form(&url, body).await
} else {
self.do_fetch(&url).await
+158 -148
View File
@@ -5,6 +5,154 @@ use crate::dispatch::CdpContext;
use crate::types::CdpEvent;
use crate::util::url_is_file_scheme;
async fn do_navigate(
url: &str,
params: &Value,
ctx: &mut CdpContext,
session_id: &Option<String>,
) -> Result<Value, String> {
let wait_until = params.get("waitUntil")
.and_then(|v| {
if let Some(s) = v.as_str() {
Some(WaitUntil::from_str(s))
} else if let Some(arr) = v.as_array() {
arr.iter()
.filter_map(|item| item.as_str())
.map(WaitUntil::from_str)
.max_by_key(|w| match w {
WaitUntil::DomContentLoaded => 0,
WaitUntil::Load => 1,
WaitUntil::NetworkIdle2 => 2,
WaitUntil::NetworkIdle0 => 3,
})
} else {
None
}
})
.unwrap_or(WaitUntil::Load);
// Block CDP-initiated file:// navigation by default.
// Anyone who can reach the CDP port (default localhost,
// but Docker images bind 0.0.0.0) could otherwise read
// any file the obscura process can read. Opt in via
// `obscura serve --allow-file-access` when local-HTML
// testing is the intended workflow.
if url_is_file_scheme(url) && !ctx.default_context.allow_file_access {
return Err(
"Page.navigate to file:// is disabled. Restart with `obscura serve --allow-file-access` to enable.".to_string()
);
}
let preload_scripts: Vec<String> = ctx.preload_scripts.iter().map(|(_, s)| s.clone()).collect();
let (frame_id, loader_id, network_events, page_url, page_id, reached_network_idle) = {
let page = ctx.get_session_page_mut(session_id).ok_or("No page for session")?;
let frame_id = page.frame_id.clone();
let loader_id = format!("loader-{}", uuid::Uuid::new_v4());
let nav_method = params.get("__method").and_then(|v| v.as_str()).unwrap_or("GET");
let nav_body = params.get("__body").and_then(|v| v.as_str()).unwrap_or("");
if nav_method == "POST" && !nav_body.is_empty() {
page.navigate_with_wait_post(url, wait_until, nav_method, nav_body).await.map_err(|e| e.to_string())?;
} else {
page.navigate_with_wait(url, wait_until).await.map_err(|e| e.to_string())?;
}
for source in &preload_scripts {
if let Err(e) = page.execute_preload_script(source) {
tracing::debug!("Preload script error: {}", e);
}
}
let reached_network_idle = page.lifecycle.is_network_idle();
let network_events: Vec<_> = page.network_events.drain(..).collect();
let page_url = page.url_string();
let page_id = page.id.clone();
(frame_id, loader_id, network_events, page_url, page_id, reached_network_idle)
};
let es = session_id.clone();
let ts = timestamp();
let mut phase1 = vec![
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "init", "timestamp": ts}), session_id: es.clone() },
CdpEvent { method: "Runtime.executionContextsCleared".into(), params: json!({}), session_id: es.clone() },
CdpEvent { method: "Page.frameNavigated".into(), params: json!({"frame": {"id": frame_id, "loaderId": loader_id, "url": page_url, "domainAndRegistry": "", "securityOrigin": page_url, "mimeType": "text/html", "adFrameStatus": {"adFrameType": "none"}}, "type": "Navigation"}), session_id: es.clone() },
CdpEvent { method: "Runtime.executionContextCreated".into(), params: json!({"context": {"id": 2, "origin": page_url, "name": "", "uniqueId": format!("ctx-nav-{}", page_id), "auxData": {"isDefault": true, "type": "default", "frameId": frame_id}}}), session_id: es.clone() },
];
let world_names: Vec<String> = if ctx.isolated_worlds.is_empty() {
vec!["__puppeteer_utility_world__24.40.0".to_string()]
} else {
ctx.isolated_worlds.clone()
};
for (idx, world_name) in world_names.iter().enumerate() {
let world_ctx_id = 100 + idx as u32;
phase1.push(CdpEvent {
method: "Runtime.executionContextCreated".into(),
params: json!({"context": {"id": world_ctx_id, "origin": page_url, "name": world_name, "uniqueId": format!("ctx-isolated-nav-{}-{}", page_id, idx), "auxData": {"isDefault": false, "type": "isolated", "frameId": frame_id}}}),
session_id: es.clone(),
});
}
phase1.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "commit", "timestamp": ts}), session_id: es.clone() });
ctx.pending_events.extend(phase1);
if ctx.fetch_intercept.enabled {
for net_event in &network_events {
ctx.pending_events.push(CdpEvent {
method: "Fetch.requestPaused".into(),
params: json!({
"requestId": net_event.request_id,
"request": {
"url": net_event.url,
"method": net_event.method,
"headers": net_event.headers,
},
"frameId": frame_id,
"resourceType": net_event.resource_type,
"networkId": net_event.request_id,
}),
session_id: es.clone(),
});
}
}
for net_event in &network_events {
ctx.pending_events.push(CdpEvent {
method: "Network.requestWillBeSent".into(),
params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}),
session_id: es.clone(),
});
ctx.pending_events.push(CdpEvent {
method: "Network.responseReceived".into(),
params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "timestamp": net_event.timestamp, "type": net_event.resource_type, "response": {"url": net_event.url, "status": net_event.status, "statusText": "", "headers": &*net_event.response_headers, "mimeType": net_event.response_headers.get("content-type").cloned().unwrap_or_default()}, "frameId": frame_id}),
session_id: es.clone(),
});
ctx.pending_events.push(CdpEvent {
method: "Network.loadingFinished".into(),
params: json!({"requestId": net_event.request_id, "timestamp": net_event.timestamp, "encodedDataLength": net_event.body_size}),
session_id: es.clone(),
});
}
let mut phase3 = vec![
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "DOMContentLoaded", "timestamp": ts}), session_id: es.clone() },
CdpEvent { method: "Page.domContentEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() },
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "load", "timestamp": ts}), session_id: es.clone() },
CdpEvent { method: "Page.loadEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() },
];
if reached_network_idle || matches!(wait_until, WaitUntil::Load | WaitUntil::DomContentLoaded) {
let idle_ts = timestamp();
phase3.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "networkIdle", "timestamp": idle_ts}), session_id: es.clone() });
}
phase3.push(CdpEvent { method: "Page.frameStoppedLoading".into(), params: json!({"frameId": frame_id}), session_id: es });
ctx.pending_events.extend(phase3);
Ok(json!({
"frameId": frame_id,
"loaderId": loader_id,
}))
}
pub async fn handle(
method: &str,
params: &Value,
@@ -16,154 +164,16 @@ pub async fn handle(
"navigate" => {
let url = params.get("url").and_then(|v| v.as_str())
.ok_or("url required")?;
// Block CDP-initiated file:// navigation by default.
// Anyone who can reach the CDP port (default localhost,
// but Docker images bind 0.0.0.0) could otherwise read
// any file the obscura process can read. Opt in via
// `obscura serve --allow-file-access` when local-HTML
// testing is the intended workflow.
if url_is_file_scheme(url) && !ctx.default_context.allow_file_access {
return Err(
"Page.navigate to file:// is disabled. Restart with `obscura serve --allow-file-access` to enable.".to_string()
);
}
let wait_until = params.get("waitUntil")
.and_then(|v| {
if let Some(s) = v.as_str() {
Some(WaitUntil::from_str(s))
} else if let Some(arr) = v.as_array() {
arr.iter()
.filter_map(|item| item.as_str())
.map(WaitUntil::from_str)
.max_by_key(|w| match w {
WaitUntil::DomContentLoaded => 0,
WaitUntil::Load => 1,
WaitUntil::NetworkIdle2 => 2,
WaitUntil::NetworkIdle0 => 3,
})
} else {
None
}
})
.unwrap_or(WaitUntil::Load);
let preload_scripts: Vec<String> = ctx.preload_scripts.iter().map(|(_, s)| s.clone()).collect();
let (frame_id, loader_id, network_events, page_url, page_id, reached_network_idle) = {
let page = ctx.get_session_page_mut(session_id).ok_or("No page for session")?;
let frame_id = page.frame_id.clone();
let loader_id = format!("loader-{}", uuid::Uuid::new_v4());
let nav_method = params.get("__method").and_then(|v| v.as_str()).unwrap_or("GET");
let nav_body = params.get("__body").and_then(|v| v.as_str()).unwrap_or("");
if nav_method == "POST" && !nav_body.is_empty() {
page.navigate_with_wait_post(url, wait_until, nav_method, nav_body).await.map_err(|e| e.to_string())?;
} else {
page.navigate_with_wait(url, wait_until).await.map_err(|e| e.to_string())?;
}
for source in &preload_scripts {
if let Err(e) = page.execute_preload_script(source) {
tracing::debug!("Preload script error: {}", e);
}
}
let reached_network_idle = page.lifecycle.is_network_idle();
let network_events: Vec<_> = page.network_events.drain(..).collect();
let page_url = page.url_string();
let page_id = page.id.clone();
(frame_id, loader_id, network_events, page_url, page_id, reached_network_idle)
};
let es = session_id.clone();
let ts = timestamp();
let mut phase1 = vec![
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "init", "timestamp": ts}), session_id: es.clone() },
CdpEvent { method: "Runtime.executionContextsCleared".into(), params: json!({}), session_id: es.clone() },
CdpEvent { method: "Page.frameNavigated".into(), params: json!({"frame": {"id": frame_id, "loaderId": loader_id, "url": page_url, "domainAndRegistry": "", "securityOrigin": page_url, "mimeType": "text/html", "adFrameStatus": {"adFrameType": "none"}}, "type": "Navigation"}), session_id: es.clone() },
CdpEvent { method: "Runtime.executionContextCreated".into(), params: json!({"context": {"id": 2, "origin": page_url, "name": "", "uniqueId": format!("ctx-nav-{}", page_id), "auxData": {"isDefault": true, "type": "default", "frameId": frame_id}}}), session_id: es.clone() },
];
// Re-emit each isolated world the client previously registered
// via Page.createIsolatedWorld. Without this, Playwright's
// utility-world handle becomes stale after navigation and
// every subsequent evaluate() (including page.title()) hangs.
// Fallback to the legacy hardcoded Puppeteer name so older
// Puppeteer clients that don't call createIsolatedWorld
// continue to work.
let world_names: Vec<String> = if ctx.isolated_worlds.is_empty() {
vec!["__puppeteer_utility_world__24.40.0".to_string()]
} else {
ctx.isolated_worlds.clone()
};
for (idx, world_name) in world_names.iter().enumerate() {
let world_ctx_id = 100 + idx as u32;
phase1.push(CdpEvent {
method: "Runtime.executionContextCreated".into(),
params: json!({"context": {"id": world_ctx_id, "origin": page_url, "name": world_name, "uniqueId": format!("ctx-isolated-nav-{}-{}", page_id, idx), "auxData": {"isDefault": false, "type": "isolated", "frameId": frame_id}}}),
session_id: es.clone(),
});
}
phase1.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "commit", "timestamp": ts}), session_id: es.clone() });
ctx.pending_events.extend(phase1);
if ctx.fetch_intercept.enabled {
for net_event in &network_events {
ctx.pending_events.push(CdpEvent {
method: "Fetch.requestPaused".into(),
params: json!({
"requestId": net_event.request_id,
"request": {
"url": net_event.url,
"method": net_event.method,
"headers": net_event.headers,
},
"frameId": frame_id,
"resourceType": net_event.resource_type,
"networkId": net_event.request_id,
}),
session_id: es.clone(),
});
}
}
for net_event in &network_events {
ctx.pending_events.push(CdpEvent {
method: "Network.requestWillBeSent".into(),
params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}),
session_id: es.clone(),
});
ctx.pending_events.push(CdpEvent {
method: "Network.responseReceived".into(),
params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "timestamp": net_event.timestamp, "type": net_event.resource_type, "response": {"url": net_event.url, "status": net_event.status, "statusText": "", "headers": &*net_event.response_headers, "mimeType": net_event.response_headers.get("content-type").cloned().unwrap_or_default()}, "frameId": frame_id}),
session_id: es.clone(),
});
ctx.pending_events.push(CdpEvent {
method: "Network.loadingFinished".into(),
params: json!({"requestId": net_event.request_id, "timestamp": net_event.timestamp, "encodedDataLength": net_event.body_size}),
session_id: es.clone(),
});
}
let mut phase3 = vec![
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "DOMContentLoaded", "timestamp": ts}), session_id: es.clone() },
CdpEvent { method: "Page.domContentEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() },
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "load", "timestamp": ts}), session_id: es.clone() },
CdpEvent { method: "Page.loadEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() },
];
if reached_network_idle || matches!(wait_until, WaitUntil::Load | WaitUntil::DomContentLoaded) {
let idle_ts = timestamp();
phase3.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "networkIdle", "timestamp": idle_ts}), session_id: es.clone() });
}
phase3.push(CdpEvent { method: "Page.frameStoppedLoading".into(), params: json!({"frameId": frame_id}), session_id: es });
ctx.pending_events.extend(phase3);
Ok(json!({
"frameId": frame_id,
"loaderId": loader_id,
}))
do_navigate(url, params, ctx, session_id).await
}
"reload" => {
let current_url = ctx.get_session_page(session_id)
.map(|p| p.url_string())
.unwrap_or_else(|| "about:blank".to_string());
let reload_params = json!({
"waitUntil": params.get("waitUntil").cloned().unwrap_or(json!("load"))
});
do_navigate(&current_url, &reload_params, ctx, session_id).await
}
"getFrameTree" => {
let page = ctx.get_session_page(session_id).ok_or("No page for session")?;
+19 -2
View File
@@ -791,7 +791,7 @@ class Document extends Node {
get compatMode() { return "CSS1Compat"; }
get characterSet() { return "UTF-8"; }
get contentType() { return "text/html"; }
get readyState() { return "complete"; }
get readyState() { return globalThis.__documentReadyState__ || 'complete'; }
get hidden() { return false; }
get visibilityState() { return "visible"; }
getElementById(id) { return _wrapEl(+_dom("get_element_by_id", id)); }
@@ -805,6 +805,7 @@ class Document extends Node {
}
getElementsByTagName(t) { return this.querySelectorAll(t); }
getElementsByClassName(c) { return this.querySelectorAll("." + c); }
getElementsByName(name) { return this.querySelectorAll('[name="' + String(name).replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '"]'); }
createElement(t) {
const el = _wrapEl(+_dom("create_element", t.toLowerCase()));
if (el && t.toLowerCase() === 'template') {
@@ -853,7 +854,23 @@ class Document extends Node {
return new Cls('');
}
createRange() { return { setStart(){}, setEnd(){}, collapse(){}, selectNodeContents(){}, cloneContents(){ return document.createDocumentFragment(); } }; }
addEventListener(type, fn, opts) {} removeEventListener() {} dispatchEvent() { return true; }
addEventListener(type, fn, opts) {
if (typeof fn !== 'function') return;
if (!this._listeners) this._listeners = {};
if (!this._listeners[type]) this._listeners[type] = [];
if (!this._listeners[type].includes(fn)) this._listeners[type].push(fn);
}
removeEventListener(type, fn) {
if (this._listeners?.[type]) {
this._listeners[type] = this._listeners[type].filter(h => h !== fn);
}
}
dispatchEvent(event) {
if (!event) return true;
const handlers = (this._listeners?.[event.type] || []).slice();
for (const h of handlers) { try { h.call(this, event); } catch(e) { console.error('document event error:', e); } }
return !event.defaultPrevented;
}
createTreeWalker(root, whatToShow, filter) {
whatToShow = whatToShow || 0xFFFFFFFF; // NodeFilter.SHOW_ALL
const walker = {