diff --git a/Cargo.lock b/Cargo.lock index 9d539b1..95093c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1594,6 +1594,7 @@ name = "obscura-browser" version = "0.1.0" dependencies = [ "anyhow", + "base64", "futures", "obscura-dom", "obscura-js", diff --git a/crates/obscura-browser/Cargo.toml b/crates/obscura-browser/Cargo.toml index bb88efe..0179e94 100644 --- a/crates/obscura-browser/Cargo.toml +++ b/crates/obscura-browser/Cargo.toml @@ -18,3 +18,4 @@ tracing = { workspace = true } thiserror = { workspace = true } anyhow = { workspace = true } serde_json = { workspace = true } +base64 = { workspace = true } diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs index 69b52b8..0b7d8c3 100644 --- a/crates/obscura-browser/src/page.rs +++ b/crates/obscura-browser/src/page.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use obscura_dom::{parse_html, DomTree}; use obscura_js::runtime::ObscuraJsRuntime; use obscura_net::{ObscuraHttpClient, ObscuraNetError, Response}; @@ -8,6 +9,48 @@ use url::Url; use crate::context::BrowserContext; use crate::lifecycle::LifecycleState; +fn decode_data_uri(uri: &str) -> Option> { + let rest = uri.strip_prefix("data:")?; + let comma = rest.find(',')?; + let meta = &rest[..comma]; + let payload = &rest[comma + 1..]; + if meta.split(';').any(|t| t.eq_ignore_ascii_case("base64")) { + let cleaned: String = payload.chars().filter(|c| !c.is_whitespace()).collect(); + BASE64.decode(cleaned).ok() + } else { + Some(percent_decode(payload)) + } +} + +fn percent_decode(s: &str) -> Vec { + let mut out = Vec::with_capacity(s.len()); + let b = s.as_bytes(); + let mut i = 0; + while i < b.len() { + if b[i] == b'%' && i + 2 < b.len() { + let hi = hex_val(b[i + 1]); + let lo = hex_val(b[i + 2]); + if let (Some(h), Some(l)) = (hi, lo) { + out.push((h << 4) | l); + i += 3; + continue; + } + } + out.push(b[i]); + i += 1; + } + out +} + +fn hex_val(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + #[cfg(feature = "stealth")] use obscura_net::StealthHttpClient; @@ -364,6 +407,13 @@ impl Page { } } + // Spec: readyState is "loading" while parser-discovered scripts execute. + // Scripts that check readyState === 'loading' will register DOMContentLoaded + // listeners instead of calling their callback immediately. + if let Some(js) = &mut self.js { + let _ = js.execute_script("", "globalThis.__documentReadyState__ = 'loading';"); + } + for (i, script) in all_to_execute.iter().enumerate() { if script.src.is_some() { if let Some((url, code, resp)) = fetched.remove(&i) { @@ -417,10 +467,15 @@ impl Page { } if let Some(js) = &mut self.js { + // Spec order: readyState -> interactive, fire DOMContentLoaded on both + // document and window, then readyState -> complete, fire load. let _ = js.execute_script("", - "if (typeof window.onload === 'function') { try { window.onload(); } catch(e) {} }\n\ - try { document.dispatchEvent(new Event('DOMContentLoaded')); } catch(e) {}\n\ - try { window.dispatchEvent(new Event('load')); } catch(e) {}"); + "globalThis.__documentReadyState__ = 'interactive';\n\ + try { document.dispatchEvent(new Event('DOMContentLoaded', {bubbles:false,cancelable:false})); } catch(e) {}\n\ + try { window.dispatchEvent(new Event('DOMContentLoaded', {bubbles:false,cancelable:false})); } catch(e) {}\n\ + if (typeof window.onload === 'function') { try { window.onload(); } catch(e) {} }\n\ + globalThis.__documentReadyState__ = 'complete';\n\ + try { window.dispatchEvent(new Event('load', {bubbles:false,cancelable:false})); } catch(e) {}"); } if let Some(js) = &mut self.js { @@ -556,7 +611,18 @@ impl Page { } } - let response = if method == "POST" { + let response = if url.scheme() == "data" { + let content_type = url_str.strip_prefix("data:") + .and_then(|s| s.split(',').next()) + .unwrap_or("text/html") + .split(';').next() + .unwrap_or("text/html") + .to_string(); + let body_bytes = decode_data_uri(url_str).unwrap_or_default(); + let mut headers = std::collections::HashMap::new(); + headers.insert("content-type".to_string(), content_type); + Ok(obscura_net::Response { url: url.clone(), status: 200, headers, body: body_bytes, redirected_from: Vec::new() }) + } else if method == "POST" { self.http_client.post_form(&url, body).await } else { self.do_fetch(&url).await diff --git a/crates/obscura-cdp/src/domains/page.rs b/crates/obscura-cdp/src/domains/page.rs index 7b85199..a3d9145 100644 --- a/crates/obscura-cdp/src/domains/page.rs +++ b/crates/obscura-cdp/src/domains/page.rs @@ -5,6 +5,154 @@ use crate::dispatch::CdpContext; use crate::types::CdpEvent; use crate::util::url_is_file_scheme; +async fn do_navigate( + url: &str, + params: &Value, + ctx: &mut CdpContext, + session_id: &Option, +) -> Result { + let wait_until = params.get("waitUntil") + .and_then(|v| { + if let Some(s) = v.as_str() { + Some(WaitUntil::from_str(s)) + } else if let Some(arr) = v.as_array() { + arr.iter() + .filter_map(|item| item.as_str()) + .map(WaitUntil::from_str) + .max_by_key(|w| match w { + WaitUntil::DomContentLoaded => 0, + WaitUntil::Load => 1, + WaitUntil::NetworkIdle2 => 2, + WaitUntil::NetworkIdle0 => 3, + }) + } else { + None + } + }) + .unwrap_or(WaitUntil::Load); + + // Block CDP-initiated file:// navigation by default. + // Anyone who can reach the CDP port (default localhost, + // but Docker images bind 0.0.0.0) could otherwise read + // any file the obscura process can read. Opt in via + // `obscura serve --allow-file-access` when local-HTML + // testing is the intended workflow. + if url_is_file_scheme(url) && !ctx.default_context.allow_file_access { + return Err( + "Page.navigate to file:// is disabled. Restart with `obscura serve --allow-file-access` to enable.".to_string() + ); + } + + let preload_scripts: Vec = ctx.preload_scripts.iter().map(|(_, s)| s.clone()).collect(); + + let (frame_id, loader_id, network_events, page_url, page_id, reached_network_idle) = { + let page = ctx.get_session_page_mut(session_id).ok_or("No page for session")?; + let frame_id = page.frame_id.clone(); + let loader_id = format!("loader-{}", uuid::Uuid::new_v4()); + + let nav_method = params.get("__method").and_then(|v| v.as_str()).unwrap_or("GET"); + let nav_body = params.get("__body").and_then(|v| v.as_str()).unwrap_or(""); + if nav_method == "POST" && !nav_body.is_empty() { + page.navigate_with_wait_post(url, wait_until, nav_method, nav_body).await.map_err(|e| e.to_string())?; + } else { + page.navigate_with_wait(url, wait_until).await.map_err(|e| e.to_string())?; + } + + for source in &preload_scripts { + if let Err(e) = page.execute_preload_script(source) { + tracing::debug!("Preload script error: {}", e); + } + } + + let reached_network_idle = page.lifecycle.is_network_idle(); + let network_events: Vec<_> = page.network_events.drain(..).collect(); + let page_url = page.url_string(); + let page_id = page.id.clone(); + (frame_id, loader_id, network_events, page_url, page_id, reached_network_idle) + }; + + let es = session_id.clone(); + let ts = timestamp(); + + let mut phase1 = vec![ + CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "init", "timestamp": ts}), session_id: es.clone() }, + CdpEvent { method: "Runtime.executionContextsCleared".into(), params: json!({}), session_id: es.clone() }, + CdpEvent { method: "Page.frameNavigated".into(), params: json!({"frame": {"id": frame_id, "loaderId": loader_id, "url": page_url, "domainAndRegistry": "", "securityOrigin": page_url, "mimeType": "text/html", "adFrameStatus": {"adFrameType": "none"}}, "type": "Navigation"}), session_id: es.clone() }, + CdpEvent { method: "Runtime.executionContextCreated".into(), params: json!({"context": {"id": 2, "origin": page_url, "name": "", "uniqueId": format!("ctx-nav-{}", page_id), "auxData": {"isDefault": true, "type": "default", "frameId": frame_id}}}), session_id: es.clone() }, + ]; + let world_names: Vec = if ctx.isolated_worlds.is_empty() { + vec!["__puppeteer_utility_world__24.40.0".to_string()] + } else { + ctx.isolated_worlds.clone() + }; + for (idx, world_name) in world_names.iter().enumerate() { + let world_ctx_id = 100 + idx as u32; + phase1.push(CdpEvent { + method: "Runtime.executionContextCreated".into(), + params: json!({"context": {"id": world_ctx_id, "origin": page_url, "name": world_name, "uniqueId": format!("ctx-isolated-nav-{}-{}", page_id, idx), "auxData": {"isDefault": false, "type": "isolated", "frameId": frame_id}}}), + session_id: es.clone(), + }); + } + phase1.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "commit", "timestamp": ts}), session_id: es.clone() }); + ctx.pending_events.extend(phase1); + + if ctx.fetch_intercept.enabled { + for net_event in &network_events { + ctx.pending_events.push(CdpEvent { + method: "Fetch.requestPaused".into(), + params: json!({ + "requestId": net_event.request_id, + "request": { + "url": net_event.url, + "method": net_event.method, + "headers": net_event.headers, + }, + "frameId": frame_id, + "resourceType": net_event.resource_type, + "networkId": net_event.request_id, + }), + session_id: es.clone(), + }); + } + } + + for net_event in &network_events { + ctx.pending_events.push(CdpEvent { + method: "Network.requestWillBeSent".into(), + params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}), + session_id: es.clone(), + }); + ctx.pending_events.push(CdpEvent { + method: "Network.responseReceived".into(), + params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "timestamp": net_event.timestamp, "type": net_event.resource_type, "response": {"url": net_event.url, "status": net_event.status, "statusText": "", "headers": &*net_event.response_headers, "mimeType": net_event.response_headers.get("content-type").cloned().unwrap_or_default()}, "frameId": frame_id}), + session_id: es.clone(), + }); + ctx.pending_events.push(CdpEvent { + method: "Network.loadingFinished".into(), + params: json!({"requestId": net_event.request_id, "timestamp": net_event.timestamp, "encodedDataLength": net_event.body_size}), + session_id: es.clone(), + }); + } + + let mut phase3 = vec![ + CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "DOMContentLoaded", "timestamp": ts}), session_id: es.clone() }, + CdpEvent { method: "Page.domContentEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() }, + CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "load", "timestamp": ts}), session_id: es.clone() }, + CdpEvent { method: "Page.loadEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() }, + ]; + if reached_network_idle || matches!(wait_until, WaitUntil::Load | WaitUntil::DomContentLoaded) { + let idle_ts = timestamp(); + phase3.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "networkIdle", "timestamp": idle_ts}), session_id: es.clone() }); + } + phase3.push(CdpEvent { method: "Page.frameStoppedLoading".into(), params: json!({"frameId": frame_id}), session_id: es }); + ctx.pending_events.extend(phase3); + + Ok(json!({ + "frameId": frame_id, + "loaderId": loader_id, + })) +} + pub async fn handle( method: &str, params: &Value, @@ -16,154 +164,16 @@ pub async fn handle( "navigate" => { let url = params.get("url").and_then(|v| v.as_str()) .ok_or("url required")?; - - // Block CDP-initiated file:// navigation by default. - // Anyone who can reach the CDP port (default localhost, - // but Docker images bind 0.0.0.0) could otherwise read - // any file the obscura process can read. Opt in via - // `obscura serve --allow-file-access` when local-HTML - // testing is the intended workflow. - if url_is_file_scheme(url) && !ctx.default_context.allow_file_access { - return Err( - "Page.navigate to file:// is disabled. Restart with `obscura serve --allow-file-access` to enable.".to_string() - ); - } - - let wait_until = params.get("waitUntil") - .and_then(|v| { - if let Some(s) = v.as_str() { - Some(WaitUntil::from_str(s)) - } else if let Some(arr) = v.as_array() { - arr.iter() - .filter_map(|item| item.as_str()) - .map(WaitUntil::from_str) - .max_by_key(|w| match w { - WaitUntil::DomContentLoaded => 0, - WaitUntil::Load => 1, - WaitUntil::NetworkIdle2 => 2, - WaitUntil::NetworkIdle0 => 3, - }) - } else { - None - } - }) - .unwrap_or(WaitUntil::Load); - - let preload_scripts: Vec = ctx.preload_scripts.iter().map(|(_, s)| s.clone()).collect(); - - let (frame_id, loader_id, network_events, page_url, page_id, reached_network_idle) = { - let page = ctx.get_session_page_mut(session_id).ok_or("No page for session")?; - let frame_id = page.frame_id.clone(); - let loader_id = format!("loader-{}", uuid::Uuid::new_v4()); - - let nav_method = params.get("__method").and_then(|v| v.as_str()).unwrap_or("GET"); - let nav_body = params.get("__body").and_then(|v| v.as_str()).unwrap_or(""); - if nav_method == "POST" && !nav_body.is_empty() { - page.navigate_with_wait_post(url, wait_until, nav_method, nav_body).await.map_err(|e| e.to_string())?; - } else { - page.navigate_with_wait(url, wait_until).await.map_err(|e| e.to_string())?; - } - - for source in &preload_scripts { - if let Err(e) = page.execute_preload_script(source) { - tracing::debug!("Preload script error: {}", e); - } - } - - let reached_network_idle = page.lifecycle.is_network_idle(); - let network_events: Vec<_> = page.network_events.drain(..).collect(); - let page_url = page.url_string(); - let page_id = page.id.clone(); - (frame_id, loader_id, network_events, page_url, page_id, reached_network_idle) - }; - - let es = session_id.clone(); - let ts = timestamp(); - - let mut phase1 = vec![ - CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "init", "timestamp": ts}), session_id: es.clone() }, - CdpEvent { method: "Runtime.executionContextsCleared".into(), params: json!({}), session_id: es.clone() }, - CdpEvent { method: "Page.frameNavigated".into(), params: json!({"frame": {"id": frame_id, "loaderId": loader_id, "url": page_url, "domainAndRegistry": "", "securityOrigin": page_url, "mimeType": "text/html", "adFrameStatus": {"adFrameType": "none"}}, "type": "Navigation"}), session_id: es.clone() }, - CdpEvent { method: "Runtime.executionContextCreated".into(), params: json!({"context": {"id": 2, "origin": page_url, "name": "", "uniqueId": format!("ctx-nav-{}", page_id), "auxData": {"isDefault": true, "type": "default", "frameId": frame_id}}}), session_id: es.clone() }, - ]; - // Re-emit each isolated world the client previously registered - // via Page.createIsolatedWorld. Without this, Playwright's - // utility-world handle becomes stale after navigation and - // every subsequent evaluate() (including page.title()) hangs. - // Fallback to the legacy hardcoded Puppeteer name so older - // Puppeteer clients that don't call createIsolatedWorld - // continue to work. - let world_names: Vec = if ctx.isolated_worlds.is_empty() { - vec!["__puppeteer_utility_world__24.40.0".to_string()] - } else { - ctx.isolated_worlds.clone() - }; - for (idx, world_name) in world_names.iter().enumerate() { - let world_ctx_id = 100 + idx as u32; - phase1.push(CdpEvent { - method: "Runtime.executionContextCreated".into(), - params: json!({"context": {"id": world_ctx_id, "origin": page_url, "name": world_name, "uniqueId": format!("ctx-isolated-nav-{}-{}", page_id, idx), "auxData": {"isDefault": false, "type": "isolated", "frameId": frame_id}}}), - session_id: es.clone(), - }); - } - phase1.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "commit", "timestamp": ts}), session_id: es.clone() }); - ctx.pending_events.extend(phase1); - - if ctx.fetch_intercept.enabled { - for net_event in &network_events { - ctx.pending_events.push(CdpEvent { - method: "Fetch.requestPaused".into(), - params: json!({ - "requestId": net_event.request_id, - "request": { - "url": net_event.url, - "method": net_event.method, - "headers": net_event.headers, - }, - "frameId": frame_id, - "resourceType": net_event.resource_type, - "networkId": net_event.request_id, - }), - session_id: es.clone(), - }); - } - } - - for net_event in &network_events { - ctx.pending_events.push(CdpEvent { - method: "Network.requestWillBeSent".into(), - params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}), - session_id: es.clone(), - }); - ctx.pending_events.push(CdpEvent { - method: "Network.responseReceived".into(), - params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "timestamp": net_event.timestamp, "type": net_event.resource_type, "response": {"url": net_event.url, "status": net_event.status, "statusText": "", "headers": &*net_event.response_headers, "mimeType": net_event.response_headers.get("content-type").cloned().unwrap_or_default()}, "frameId": frame_id}), - session_id: es.clone(), - }); - ctx.pending_events.push(CdpEvent { - method: "Network.loadingFinished".into(), - params: json!({"requestId": net_event.request_id, "timestamp": net_event.timestamp, "encodedDataLength": net_event.body_size}), - session_id: es.clone(), - }); - } - - let mut phase3 = vec![ - CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "DOMContentLoaded", "timestamp": ts}), session_id: es.clone() }, - CdpEvent { method: "Page.domContentEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() }, - CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "load", "timestamp": ts}), session_id: es.clone() }, - CdpEvent { method: "Page.loadEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() }, - ]; - if reached_network_idle || matches!(wait_until, WaitUntil::Load | WaitUntil::DomContentLoaded) { - let idle_ts = timestamp(); - phase3.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "networkIdle", "timestamp": idle_ts}), session_id: es.clone() }); - } - phase3.push(CdpEvent { method: "Page.frameStoppedLoading".into(), params: json!({"frameId": frame_id}), session_id: es }); - ctx.pending_events.extend(phase3); - - Ok(json!({ - "frameId": frame_id, - "loaderId": loader_id, - })) + do_navigate(url, params, ctx, session_id).await + } + "reload" => { + let current_url = ctx.get_session_page(session_id) + .map(|p| p.url_string()) + .unwrap_or_else(|| "about:blank".to_string()); + let reload_params = json!({ + "waitUntil": params.get("waitUntil").cloned().unwrap_or(json!("load")) + }); + do_navigate(¤t_url, &reload_params, ctx, session_id).await } "getFrameTree" => { let page = ctx.get_session_page(session_id).ok_or("No page for session")?; diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js index 4332875..2aa1016 100644 --- a/crates/obscura-js/js/bootstrap.js +++ b/crates/obscura-js/js/bootstrap.js @@ -791,7 +791,7 @@ class Document extends Node { get compatMode() { return "CSS1Compat"; } get characterSet() { return "UTF-8"; } get contentType() { return "text/html"; } - get readyState() { return "complete"; } + get readyState() { return globalThis.__documentReadyState__ || 'complete'; } get hidden() { return false; } get visibilityState() { return "visible"; } getElementById(id) { return _wrapEl(+_dom("get_element_by_id", id)); } @@ -805,6 +805,7 @@ class Document extends Node { } getElementsByTagName(t) { return this.querySelectorAll(t); } getElementsByClassName(c) { return this.querySelectorAll("." + c); } + getElementsByName(name) { return this.querySelectorAll('[name="' + String(name).replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '"]'); } createElement(t) { const el = _wrapEl(+_dom("create_element", t.toLowerCase())); if (el && t.toLowerCase() === 'template') { @@ -853,7 +854,23 @@ class Document extends Node { return new Cls(''); } createRange() { return { setStart(){}, setEnd(){}, collapse(){}, selectNodeContents(){}, cloneContents(){ return document.createDocumentFragment(); } }; } - addEventListener(type, fn, opts) {} removeEventListener() {} dispatchEvent() { return true; } + addEventListener(type, fn, opts) { + if (typeof fn !== 'function') return; + if (!this._listeners) this._listeners = {}; + if (!this._listeners[type]) this._listeners[type] = []; + if (!this._listeners[type].includes(fn)) this._listeners[type].push(fn); + } + removeEventListener(type, fn) { + if (this._listeners?.[type]) { + this._listeners[type] = this._listeners[type].filter(h => h !== fn); + } + } + dispatchEvent(event) { + if (!event) return true; + const handlers = (this._listeners?.[event.type] || []).slice(); + for (const h of handlers) { try { h.call(this, event); } catch(e) { console.error('document event error:', e); } } + return !event.defaultPrevented; + } createTreeWalker(root, whatToShow, filter) { whatToShow = whatToShow || 0xFFFFFFFF; // NodeFilter.SHOW_ALL const walker = {