Brings Obscura's CDP server in line with what Playwright's `chromium.connectOverCDP()` expects. Each fix is small and surgical; together they make `connect → newContext → newPage → goto → page.title()` work end-to-end against `https://example.com`. 1. **server.rs — emit pending events before command response.** Chromium CDP semantics require side-effect events (e.g. `Target.targetCreated` + `Target.attachedToTarget` from `Target.createTarget`) to arrive *before* the originating command's response. Playwright's wire reader assumes this ordering — it resolves the response promise on the response and then synchronously looks up the freshly-created session. Reversing the order makes that lookup miss and the page hang at `newPage()`. 2. **domains/browser.rs — accept Browser.setWindowBounds as a no-op.** Playwright calls `Browser.setWindowBounds` during teardown. Returning "Unknown Browser method" causes Playwright to treat the target as broken and tear it down mid-test, surfacing as "Frame has been detached" on the next `page.title()`. 3. **obscura-browser/page.rs — frame_id == target_id for main frame.** Was `format!("{id}.1", id)`. Playwright's frame manager indexes frames by frame id and looks up the main frame by target id, so the `.1` suffix made every main-frame lookup miss. 4. **domains/page.rs + dispatch.rs — track isolated worlds and re-emit them post-navigation.** Playwright registers a utility world via `Page.createIsolatedWorld` and expects `Runtime.executionContextCreated` for each registered world after every navigation. Without re-emission, the utility-world handle goes stale post-`goto` and `page.title()` hangs waiting for the world's execution context. Falls back to the legacy `__puppeteer_utility_world__24.40.0` name when no worlds have been registered, preserving prior Puppeteer support. 5. **obscura-js/runtime.rs — strip trailing semicolons before wrapping.** Playwright's utility-script expression is an IIFE ending in `})();`. Both `wrap_expression` (CDP `Runtime.evaluate`) and `evaluate_for_cdp` (the `meta_code` IIFE) embed the user expression inside `({EXPR})` / `return ({EXPR});`. With the trailing `;` left in, this produces `({...;})` — a SyntaxError. Parse errors are not catchable, so the surrounding try/catch never fires and the call silently returns `undefined`, surfacing as `page.title() === ""`. Verified end-to-end against `https://example.com`: [probe] connect OK [probe] newContext OK [probe] newPage OK [probe] goto OK [probe] title = Example Domain
865 lines
31 KiB
Rust
865 lines
31 KiB
Rust
use std::sync::Arc;
|
|
|
|
use obscura_dom::{parse_html, DomTree};
|
|
use obscura_js::runtime::ObscuraJsRuntime;
|
|
use obscura_net::{ObscuraHttpClient, ObscuraNetError, Response};
|
|
use url::Url;
|
|
|
|
use crate::context::BrowserContext;
|
|
use crate::lifecycle::LifecycleState;
|
|
|
|
#[cfg(feature = "stealth")]
|
|
use obscura_net::StealthHttpClient;
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct NetworkEvent {
|
|
pub request_id: String,
|
|
pub url: String,
|
|
pub method: String,
|
|
pub resource_type: String,
|
|
pub status: u16,
|
|
pub headers: std::collections::HashMap<String, String>,
|
|
pub response_headers: Arc<std::collections::HashMap<String, String>>,
|
|
pub body_size: usize,
|
|
pub timestamp: f64,
|
|
}
|
|
|
|
pub struct Page {
|
|
pub id: String,
|
|
pub frame_id: String,
|
|
pub url: Option<Url>,
|
|
pub dom: Option<DomTree>,
|
|
pub js: Option<ObscuraJsRuntime>,
|
|
pub lifecycle: LifecycleState,
|
|
pub http_client: Arc<ObscuraHttpClient>,
|
|
pub context: Arc<BrowserContext>,
|
|
pub title: String,
|
|
pub network_events: Vec<NetworkEvent>,
|
|
network_event_counter: u32,
|
|
pub intercept_enabled: bool,
|
|
pub intercept_block_patterns: Vec<String>,
|
|
intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<obscura_js::ops::InterceptedRequest>>,
|
|
#[cfg(feature = "stealth")]
|
|
pub stealth_client: Option<Arc<StealthHttpClient>>,
|
|
}
|
|
|
|
impl Page {
|
|
pub fn new(id: String, context: Arc<BrowserContext>) -> Self {
|
|
let http_client = context.http_client.clone();
|
|
// Chromium convention: the main frame's frameId == the targetId.
|
|
// Playwright's frame manager looks up the main frame by targetId
|
|
// (via target._targetInfo.targetId), so any divergence here makes
|
|
// Page.getFrameTree return a frame the client cannot match,
|
|
// triggering a Target.closeTarget and "Frame has been detached".
|
|
let frame_id = id.clone();
|
|
#[cfg(feature = "stealth")]
|
|
let stealth_client = if context.stealth {
|
|
Some(Arc::new(StealthHttpClient::new(context.cookie_jar.clone())))
|
|
} else {
|
|
None
|
|
};
|
|
|
|
Page {
|
|
id,
|
|
frame_id,
|
|
url: None,
|
|
dom: None,
|
|
js: None,
|
|
lifecycle: LifecycleState::Idle,
|
|
http_client,
|
|
context,
|
|
title: String::new(),
|
|
network_events: Vec::new(),
|
|
network_event_counter: 0,
|
|
intercept_enabled: false,
|
|
intercept_block_patterns: Vec::new(),
|
|
intercept_tx: None,
|
|
#[cfg(feature = "stealth")]
|
|
stealth_client,
|
|
}
|
|
}
|
|
|
|
fn should_block_url(&self, url: &str) -> bool {
|
|
if !self.intercept_enabled || self.intercept_block_patterns.is_empty() {
|
|
return false;
|
|
}
|
|
for pattern in &self.intercept_block_patterns {
|
|
if pattern == "*" { return true; }
|
|
if pattern.starts_with('*') && pattern.ends_with('*') {
|
|
if url.contains(&pattern[1..pattern.len()-1]) { return true; }
|
|
} else if pattern.starts_with('*') {
|
|
if url.ends_with(&pattern[1..]) { return true; }
|
|
} else if pattern.ends_with('*') {
|
|
if url.starts_with(&pattern[..pattern.len()-1]) { return true; }
|
|
} else if url.contains(pattern) {
|
|
return true;
|
|
}
|
|
}
|
|
false
|
|
}
|
|
|
|
async fn do_fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
|
#[cfg(feature = "stealth")]
|
|
if let Some(ref stealth) = self.stealth_client {
|
|
return stealth.fetch(url).await;
|
|
}
|
|
self.http_client.fetch(url).await
|
|
}
|
|
fn init_js(&mut self) {
|
|
if self.js.is_some() {
|
|
let url_str = self.url_string();
|
|
let title = self.title.clone();
|
|
let dom = self.dom.take();
|
|
|
|
let js = self.js.as_mut().unwrap();
|
|
js.set_url(&url_str);
|
|
js.set_title(&title);
|
|
|
|
if let Some(d) = dom {
|
|
js.set_dom(d);
|
|
}
|
|
|
|
let _ = js.execute_script("<reset>",
|
|
"_cache.clear(); globalThis.__obscura_objects = {}; globalThis.__obscura_oid = 0; \
|
|
_iframeRegistry.length = 0; globalThis.length = 0; \
|
|
globalThis._formValues = {}; globalThis._formChecked = {}; \
|
|
globalThis._eventRegistry = {}; \
|
|
globalThis.document = new Document(+_dom('document_node_id'));");
|
|
|
|
return;
|
|
}
|
|
|
|
let mut rt = ObscuraJsRuntime::with_base_url(&self.url_string());
|
|
rt.set_url(&self.url_string());
|
|
rt.set_title(&self.title);
|
|
|
|
#[cfg(feature = "stealth")]
|
|
if self.stealth_client.is_some() {
|
|
rt.set_user_agent(obscura_net::STEALTH_USER_AGENT);
|
|
} else if let Ok(ua) = self.http_client.user_agent.try_read() {
|
|
rt.set_user_agent(&ua);
|
|
}
|
|
#[cfg(not(feature = "stealth"))]
|
|
if let Ok(ua) = self.http_client.user_agent.try_read() {
|
|
rt.set_user_agent(&ua);
|
|
}
|
|
|
|
rt.set_cookie_jar(self.context.cookie_jar.clone());
|
|
rt.set_http_client(self.http_client.clone());
|
|
|
|
if let Some(tx) = &self.intercept_tx {
|
|
rt.set_intercept_tx(tx.clone());
|
|
}
|
|
|
|
if let Some(dom) = self.dom.take() {
|
|
rt.set_dom(dom);
|
|
}
|
|
|
|
self.js = Some(rt);
|
|
}
|
|
|
|
async fn execute_scripts(&mut self) {
|
|
tracing::info!("execute_scripts called, js runtime exists: {}", self.js.is_some());
|
|
|
|
#[derive(Debug)]
|
|
struct ScriptInfo {
|
|
src: Option<String>,
|
|
inline: String,
|
|
is_defer: bool,
|
|
is_async: bool,
|
|
is_module: bool,
|
|
}
|
|
|
|
let all_scripts = match &self.js {
|
|
Some(js) => {
|
|
js.with_dom(|dom| {
|
|
let script_ids = dom.query_selector_all("script").unwrap_or_default();
|
|
let mut scripts = Vec::new();
|
|
|
|
for sid in script_ids {
|
|
if let Some(node) = dom.get_node(sid) {
|
|
let src = node.get_attribute("src").map(|s| s.to_string());
|
|
let script_type = node.get_attribute("type").unwrap_or("").to_string();
|
|
let is_defer = node.get_attribute("defer").is_some();
|
|
let is_async = node.get_attribute("async").is_some();
|
|
let is_module = script_type == "module";
|
|
|
|
if !script_type.is_empty()
|
|
&& script_type != "text/javascript"
|
|
&& script_type != "application/javascript"
|
|
&& script_type != "module"
|
|
{
|
|
continue;
|
|
}
|
|
|
|
let inline_code = if src.is_none() {
|
|
dom.text_content(sid)
|
|
} else {
|
|
String::new()
|
|
};
|
|
|
|
if src.is_some() || !inline_code.trim().is_empty() {
|
|
scripts.push(ScriptInfo {
|
|
src,
|
|
inline: inline_code,
|
|
is_defer,
|
|
is_async,
|
|
is_module,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
scripts
|
|
}).unwrap_or_default()
|
|
}
|
|
None => return,
|
|
};
|
|
|
|
let mut regular = Vec::new();
|
|
let mut deferred = Vec::new();
|
|
let mut async_scripts = Vec::new();
|
|
|
|
let mut module_scripts = Vec::new();
|
|
|
|
for script in all_scripts {
|
|
if script.is_module {
|
|
module_scripts.push(script);
|
|
continue;
|
|
}
|
|
if script.is_defer {
|
|
deferred.push(script);
|
|
} else if script.is_async {
|
|
async_scripts.push(script);
|
|
} else {
|
|
regular.push(script);
|
|
}
|
|
}
|
|
|
|
let scripts = regular;
|
|
|
|
tracing::info!("Found {} regular + {} deferred + {} async scripts", scripts.len(), deferred.len(), async_scripts.len());
|
|
let all_to_execute: Vec<ScriptInfo> = scripts.into_iter()
|
|
.chain(deferred.into_iter())
|
|
.chain(async_scripts.into_iter())
|
|
.collect();
|
|
|
|
let mut resolved: Vec<(usize, String)> = Vec::new();
|
|
let mut fetch_tasks: Vec<(usize, String)> = Vec::new();
|
|
|
|
for (i, script) in all_to_execute.iter().enumerate() {
|
|
if let Some(src_url) = &script.src {
|
|
let full_url = if src_url.starts_with("http://") || src_url.starts_with("https://") {
|
|
src_url.clone()
|
|
} else if let Some(base) = &self.url {
|
|
base.join(src_url).map(|u| u.to_string()).unwrap_or_else(|_| src_url.clone())
|
|
} else {
|
|
src_url.clone()
|
|
};
|
|
|
|
if self.should_block_url(&full_url) {
|
|
tracing::info!("Blocked script by interception: {}", full_url);
|
|
continue;
|
|
}
|
|
resolved.push((i, full_url.clone()));
|
|
fetch_tasks.push((i, full_url));
|
|
}
|
|
}
|
|
|
|
let client = self.http_client.clone();
|
|
let fetch_futures: Vec<_> = fetch_tasks.iter().map(|(idx, url)| {
|
|
let client = client.clone();
|
|
let url = url.clone();
|
|
let idx = *idx;
|
|
async move {
|
|
let parsed = Url::parse(&url).unwrap_or_else(|_| Url::parse("about:blank").unwrap());
|
|
match client.fetch(&parsed).await {
|
|
Ok(resp) => Some((idx, url, resp)),
|
|
Err(e) => {
|
|
tracing::warn!("Failed to fetch script {}: {}", url, e);
|
|
None
|
|
}
|
|
}
|
|
}
|
|
}).collect();
|
|
|
|
let fetch_results = futures::future::join_all(fetch_futures).await;
|
|
|
|
let mut fetched: std::collections::HashMap<usize, (String, String, obscura_net::Response)> = std::collections::HashMap::new();
|
|
for result in fetch_results {
|
|
if let Some((idx, url, resp)) = result {
|
|
let code = String::from_utf8_lossy(&resp.body).to_string();
|
|
fetched.insert(idx, (url, code, resp));
|
|
}
|
|
}
|
|
|
|
for (i, script) in all_to_execute.iter().enumerate() {
|
|
if script.src.is_some() {
|
|
if let Some((url, code, resp)) = fetched.remove(&i) {
|
|
tracing::info!("Executing script ({} bytes): {}", code.len(), url);
|
|
self.record_network_event(&url, "GET", "Script", resp.status, &resp.headers, resp.body.len());
|
|
if let Some(js) = &mut self.js {
|
|
if let Err(e) = js.execute_script_guarded(&url, &code) {
|
|
tracing::warn!("Script error ({}): {}", url, e);
|
|
}
|
|
}
|
|
}
|
|
} else if !script.inline.is_empty() {
|
|
if let Some(js) = &mut self.js {
|
|
if let Err(e) = js.execute_script_guarded("<inline>", &script.inline) {
|
|
tracing::warn!("Inline script error: {}", e);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
for module_script in &module_scripts {
|
|
if let Some(ref src) = module_script.src {
|
|
let full_url = if src.starts_with("http://") || src.starts_with("https://") {
|
|
src.clone()
|
|
} else if let Some(base) = &self.url {
|
|
base.join(src).map(|u| u.to_string()).unwrap_or_else(|_| src.clone())
|
|
} else {
|
|
src.clone()
|
|
};
|
|
|
|
tracing::info!("Loading ES module: {}", full_url);
|
|
if let Some(js) = &mut self.js {
|
|
match js.load_module(&full_url).await {
|
|
Ok(()) => {
|
|
tracing::info!("ES module loaded: {}", full_url);
|
|
self.record_network_event(&full_url, "GET", "Script", 200, &std::collections::HashMap::new(), 0);
|
|
}
|
|
Err(e) => {
|
|
tracing::warn!("ES module error ({}): {}", full_url, e);
|
|
}
|
|
}
|
|
}
|
|
} else if !module_script.inline.is_empty() {
|
|
let base = self.url_string();
|
|
if let Some(js) = &mut self.js {
|
|
if let Err(e) = js.load_inline_module(&module_script.inline, &base).await {
|
|
tracing::warn!("Inline ES module error: {}", e);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if let Some(js) = &mut self.js {
|
|
let _ = js.execute_script("<load-events>",
|
|
"if (typeof window.onload === 'function') { try { window.onload(); } catch(e) {} }\n\
|
|
try { document.dispatchEvent(new Event('DOMContentLoaded')); } catch(e) {}\n\
|
|
try { window.dispatchEvent(new Event('load')); } catch(e) {}");
|
|
}
|
|
|
|
if let Some(js) = &mut self.js {
|
|
let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_millis(500);
|
|
let mut idle_count = 0u32;
|
|
loop {
|
|
let result = tokio::time::timeout(
|
|
tokio::time::Duration::from_millis(10),
|
|
js.run_event_loop(),
|
|
).await;
|
|
|
|
match result {
|
|
Ok(Ok(())) => {
|
|
if self.http_client.active_requests() == 0 {
|
|
idle_count += 1;
|
|
if idle_count >= 2 {
|
|
break;
|
|
}
|
|
tokio::task::yield_now().await;
|
|
} else {
|
|
idle_count = 0;
|
|
tokio::time::sleep(tokio::time::Duration::from_millis(1)).await;
|
|
}
|
|
}
|
|
Ok(Err(_)) => break,
|
|
Err(_) => {
|
|
idle_count = 0;
|
|
if tokio::time::Instant::now() >= deadline {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pub async fn navigate(&mut self, url_str: &str) -> Result<(), PageError> {
|
|
self.navigate_with_wait(url_str, crate::lifecycle::WaitUntil::Load).await
|
|
}
|
|
|
|
pub async fn navigate_with_wait(
|
|
&mut self,
|
|
url_str: &str,
|
|
wait_until: crate::lifecycle::WaitUntil,
|
|
) -> Result<(), PageError> {
|
|
self.navigate_with_wait_post(url_str, wait_until, "GET", "").await
|
|
}
|
|
|
|
pub async fn navigate_with_wait_post(
|
|
&mut self,
|
|
url_str: &str,
|
|
wait_until: crate::lifecycle::WaitUntil,
|
|
method: &str,
|
|
body: &str,
|
|
) -> Result<(), PageError> {
|
|
let mut current_url = url_str.to_string();
|
|
let mut current_method = method.to_string();
|
|
let mut current_body = body.to_string();
|
|
for _chain in 0..10 {
|
|
self.navigate_single(¤t_url, wait_until, ¤t_method, ¤t_body).await?;
|
|
if let Some((next_url, next_method, next_body)) = self.take_pending_navigation() {
|
|
tracing::info!("JS-triggered navigation chain: {} {} -> {}", current_method, current_url, next_url);
|
|
current_url = next_url;
|
|
current_method = next_method;
|
|
current_body = next_body;
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
async fn navigate_single(
|
|
&mut self,
|
|
url_str: &str,
|
|
wait_until: crate::lifecycle::WaitUntil,
|
|
method: &str,
|
|
body: &str,
|
|
) -> Result<(), PageError> {
|
|
let url = Url::parse(url_str).map_err(|e| PageError::InvalidUrl(e.to_string()))?;
|
|
|
|
self.lifecycle = LifecycleState::Loading;
|
|
self.url = Some(url.clone());
|
|
self.network_events.clear();
|
|
|
|
if self.context.obey_robots {
|
|
if let Some(domain) = url.host_str() {
|
|
if self.context.robots_cache.is_allowed(domain, "/robots.txt") {
|
|
let robots_url = format!("{}://{}/robots.txt", url.scheme(), domain);
|
|
if let Ok(robots_url) = Url::parse(&robots_url) {
|
|
if let Ok(resp) = self.http_client.fetch(&robots_url).await {
|
|
if resp.status == 200 {
|
|
let body = String::from_utf8_lossy(&resp.body);
|
|
self.context.robots_cache.parse_and_store(
|
|
domain,
|
|
&body,
|
|
&self.context.user_agent,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if !self.context.robots_cache.is_allowed(domain, url.path()) {
|
|
self.lifecycle = LifecycleState::Failed;
|
|
return Err(PageError::NetworkError(format!(
|
|
"Blocked by robots.txt: {}",
|
|
url
|
|
)));
|
|
}
|
|
}
|
|
}
|
|
|
|
let response = if method == "POST" {
|
|
self.http_client.post_form(&url, body).await
|
|
} else {
|
|
self.do_fetch(&url).await
|
|
}.map_err(|e| {
|
|
self.lifecycle = LifecycleState::Failed;
|
|
PageError::NetworkError(e.to_string())
|
|
})?;
|
|
|
|
self.record_network_event(
|
|
url.as_str(),
|
|
"GET",
|
|
"Document",
|
|
response.status,
|
|
&response.headers,
|
|
response.body.len(),
|
|
);
|
|
|
|
if !response.redirected_from.is_empty() {
|
|
self.url = Some(response.url.clone());
|
|
}
|
|
|
|
let body_text = String::from_utf8_lossy(&response.body).to_string();
|
|
let dom = parse_html(&body_text);
|
|
|
|
self.title = dom
|
|
.query_selector("title")
|
|
.ok()
|
|
.flatten()
|
|
.map(|title_id| dom.text_content(title_id))
|
|
.unwrap_or_default();
|
|
|
|
let stylesheet_urls: Vec<String> = dom
|
|
.query_selector_all("link")
|
|
.unwrap_or_default()
|
|
.iter()
|
|
.filter_map(|&nid| {
|
|
let node = dom.get_node(nid)?;
|
|
let rel = node.get_attribute("rel")?;
|
|
if rel.to_lowercase() != "stylesheet" {
|
|
return None;
|
|
}
|
|
node.get_attribute("href").map(|s| s.to_string())
|
|
})
|
|
.collect();
|
|
|
|
let mut css_fetch_urls: Vec<String> = Vec::new();
|
|
for href in &stylesheet_urls {
|
|
let full_url = if href.starts_with("http://") || href.starts_with("https://") {
|
|
href.clone()
|
|
} else if let Some(base) = &self.url {
|
|
base.join(href).map(|u| u.to_string()).unwrap_or_else(|_| href.clone())
|
|
} else {
|
|
href.clone()
|
|
};
|
|
if self.should_block_url(&full_url) {
|
|
tracing::info!("Blocked stylesheet by interception: {}", full_url);
|
|
continue;
|
|
}
|
|
css_fetch_urls.push(full_url);
|
|
}
|
|
|
|
let client = self.http_client.clone();
|
|
let css_futures: Vec<_> = css_fetch_urls.iter().map(|full_url| {
|
|
let client = client.clone();
|
|
let url_str = full_url.clone();
|
|
async move {
|
|
let parsed = Url::parse(&url_str).unwrap_or_else(|_| Url::parse("about:blank").unwrap());
|
|
match client.fetch(&parsed).await {
|
|
Ok(resp) => Some((url_str, resp)),
|
|
Err(e) => {
|
|
tracing::debug!("Failed to fetch stylesheet {}: {}", url_str, e);
|
|
None
|
|
}
|
|
}
|
|
}
|
|
}).collect();
|
|
|
|
let css_results = futures::future::join_all(css_futures).await;
|
|
let mut css_sources = Vec::new();
|
|
for result in css_results {
|
|
if let Some((url_str, resp)) = result {
|
|
let css = String::from_utf8_lossy(&resp.body).to_string();
|
|
self.record_network_event(&url_str, "GET", "Stylesheet", resp.status, &resp.headers, resp.body.len());
|
|
css_sources.push(css);
|
|
}
|
|
}
|
|
|
|
self.dom = Some(dom);
|
|
self.lifecycle = LifecycleState::DomContentLoaded;
|
|
|
|
if wait_until == crate::lifecycle::WaitUntil::DomContentLoaded {
|
|
self.init_js();
|
|
return Ok(());
|
|
}
|
|
|
|
self.init_js();
|
|
|
|
if !css_sources.is_empty() {
|
|
if let Some(js) = &mut self.js {
|
|
let combined_css = css_sources.join("\n");
|
|
let escaped = combined_css
|
|
.replace('\\', "\\\\")
|
|
.replace('`', "\\`")
|
|
.replace("${", "\\${");
|
|
let code = format!("globalThis.__obscura_css = `{}`;", escaped);
|
|
let _ = js.execute_script("<css>", &code);
|
|
}
|
|
}
|
|
if let Some(js) = &mut self.js {
|
|
let _ = js.execute_script("<iframe-load>",
|
|
"(function() { var iframes = document.querySelectorAll('iframe[src]'); for (var i = 0; i < iframes.length; i++) { var src = iframes[i].getAttribute('src'); if (src && src !== 'about:blank') iframes[i]._loadIframeSrc(src); } })()");
|
|
}
|
|
|
|
self.execute_scripts().await;
|
|
|
|
if let Some(js) = &mut self.js {
|
|
if let Ok(new_title) = js.evaluate("document.title") {
|
|
if let Some(t) = new_title.as_str() {
|
|
self.title = t.to_string();
|
|
}
|
|
}
|
|
}
|
|
|
|
self.lifecycle = LifecycleState::Loaded;
|
|
|
|
if matches!(
|
|
wait_until,
|
|
crate::lifecycle::WaitUntil::NetworkIdle0 | crate::lifecycle::WaitUntil::NetworkIdle2
|
|
) {
|
|
let threshold = match wait_until {
|
|
crate::lifecycle::WaitUntil::NetworkIdle0 => 0,
|
|
crate::lifecycle::WaitUntil::NetworkIdle2 => 2,
|
|
_ => 0,
|
|
};
|
|
|
|
let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_secs(5);
|
|
let mut idle_since: Option<tokio::time::Instant> = None;
|
|
|
|
loop {
|
|
let active = self.http_client.active_requests();
|
|
let now = tokio::time::Instant::now();
|
|
|
|
if active <= threshold {
|
|
if idle_since.is_none() {
|
|
idle_since = Some(now);
|
|
}
|
|
if now.duration_since(idle_since.unwrap()) >= tokio::time::Duration::from_millis(500) {
|
|
break;
|
|
}
|
|
} else {
|
|
idle_since = None;
|
|
}
|
|
|
|
if now >= deadline {
|
|
tracing::debug!("Network idle timeout reached with {} active requests", active);
|
|
break;
|
|
}
|
|
|
|
if let Some(js) = &mut self.js {
|
|
let _ = tokio::time::timeout(
|
|
tokio::time::Duration::from_millis(50),
|
|
js.run_event_loop(),
|
|
).await;
|
|
} else {
|
|
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
|
|
}
|
|
}
|
|
|
|
self.lifecycle = LifecycleState::NetworkIdle;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
pub fn navigate_blank(&mut self) {
|
|
self.js = None;
|
|
self.url = Some(Url::parse("about:blank").unwrap());
|
|
self.dom = Some(parse_html("<!DOCTYPE html><html><head></head><body></body></html>"));
|
|
self.title = String::new();
|
|
self.lifecycle = LifecycleState::Loaded;
|
|
}
|
|
|
|
pub fn url_string(&self) -> String {
|
|
self.url
|
|
.as_ref()
|
|
.map(|u| u.to_string())
|
|
.unwrap_or_else(|| "about:blank".to_string())
|
|
}
|
|
|
|
pub fn with_dom<R>(&self, f: impl FnOnce(&DomTree) -> R) -> Option<R> {
|
|
if let Some(js) = &self.js {
|
|
return js.with_dom(f);
|
|
}
|
|
self.dom.as_ref().map(f)
|
|
}
|
|
|
|
pub fn dom(&self) -> Option<&DomTree> {
|
|
self.dom.as_ref()
|
|
}
|
|
|
|
pub fn evaluate(&mut self, expression: &str) -> serde_json::Value {
|
|
if let Some(js) = &mut self.js {
|
|
match js.evaluate(expression) {
|
|
Ok(val) => val,
|
|
Err(e) => {
|
|
tracing::debug!("JS eval error for '{}': {}", &expression[..expression.len().min(80)], e);
|
|
serde_json::Value::Null
|
|
}
|
|
}
|
|
} else {
|
|
match expression.trim() {
|
|
"document.title" => serde_json::Value::String(self.title.clone()),
|
|
"document.URL" | "document.location.href" | "window.location.href" => {
|
|
serde_json::Value::String(self.url_string())
|
|
}
|
|
_ => serde_json::Value::Null,
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn evaluate_for_cdp(
|
|
&mut self,
|
|
expression: &str,
|
|
return_by_value: bool,
|
|
) -> obscura_js::runtime::RemoteObjectInfo {
|
|
if let Some(js) = &mut self.js {
|
|
match js.evaluate_for_cdp(expression, return_by_value) {
|
|
Ok(info) => info,
|
|
Err(e) => {
|
|
tracing::debug!("evaluate_for_cdp error: {}", e);
|
|
obscura_js::runtime::RemoteObjectInfo {
|
|
js_type: "undefined".into(),
|
|
subtype: None,
|
|
class_name: String::new(),
|
|
description: String::new(),
|
|
object_id: None,
|
|
value: None,
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
let val = self.evaluate(expression);
|
|
obscura_js::runtime::RemoteObjectInfo {
|
|
js_type: match &val {
|
|
serde_json::Value::String(_) => "string".into(),
|
|
serde_json::Value::Number(_) => "number".into(),
|
|
serde_json::Value::Bool(_) => "boolean".into(),
|
|
_ => "undefined".into(),
|
|
},
|
|
subtype: None,
|
|
class_name: String::new(),
|
|
description: String::new(),
|
|
object_id: None,
|
|
value: Some(val),
|
|
}
|
|
}
|
|
}
|
|
|
|
pub async fn call_function_on_for_cdp(
|
|
&mut self,
|
|
function_declaration: &str,
|
|
object_id: Option<&str>,
|
|
args: &[serde_json::Value],
|
|
return_by_value: bool,
|
|
await_promise: bool,
|
|
) -> obscura_js::runtime::RemoteObjectInfo {
|
|
if let Some(js) = &mut self.js {
|
|
match js.call_function_on_for_cdp(function_declaration, object_id, args, return_by_value, await_promise).await {
|
|
Ok(info) => info,
|
|
Err(e) => {
|
|
tracing::debug!("callFunctionOn error: {}", e);
|
|
obscura_js::runtime::RemoteObjectInfo {
|
|
js_type: "undefined".into(),
|
|
subtype: None,
|
|
class_name: String::new(),
|
|
description: String::new(),
|
|
object_id: None,
|
|
value: None,
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
obscura_js::runtime::RemoteObjectInfo {
|
|
js_type: "undefined".into(),
|
|
subtype: None,
|
|
class_name: String::new(),
|
|
description: String::new(),
|
|
object_id: None,
|
|
value: None,
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn set_blocked_urls(&mut self, patterns: Vec<String>) {
|
|
if let Some(js) = &self.js {
|
|
js.set_blocked_urls(patterns);
|
|
}
|
|
}
|
|
|
|
pub fn release_object(&mut self, object_id: &str) {
|
|
if let Some(js) = &mut self.js {
|
|
js.release_object(object_id);
|
|
}
|
|
}
|
|
|
|
fn record_network_event(
|
|
&mut self,
|
|
url: &str,
|
|
method: &str,
|
|
resource_type: &str,
|
|
status: u16,
|
|
response_headers: &std::collections::HashMap<String, String>,
|
|
body_size: usize,
|
|
) {
|
|
self.network_event_counter += 1;
|
|
let timestamp = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_secs_f64();
|
|
self.network_events.push(NetworkEvent {
|
|
request_id: format!("{}.{}", self.id, self.network_event_counter),
|
|
url: url.to_string(),
|
|
method: method.to_string(),
|
|
resource_type: resource_type.to_string(),
|
|
status,
|
|
headers: std::collections::HashMap::new(),
|
|
response_headers: Arc::new(response_headers.clone()),
|
|
body_size,
|
|
timestamp,
|
|
});
|
|
}
|
|
|
|
pub fn execute_preload_script(&mut self, source: &str) -> Result<(), String> {
|
|
if let Some(js) = &mut self.js {
|
|
js.execute_script("<preload>", source)
|
|
} else {
|
|
Err("No JS runtime".to_string())
|
|
}
|
|
}
|
|
|
|
pub fn suspend_js(&mut self) {
|
|
if let Some(js) = &self.js {
|
|
if let Some(dom) = js.take_dom() {
|
|
self.dom = Some(dom);
|
|
}
|
|
}
|
|
self.js = None;
|
|
}
|
|
|
|
pub fn resume_js(&mut self) {
|
|
if self.js.is_some() {
|
|
return;
|
|
}
|
|
self.init_js();
|
|
}
|
|
|
|
pub fn has_js(&self) -> bool {
|
|
self.js.is_some()
|
|
}
|
|
|
|
pub fn release_object_group(&mut self) {
|
|
if let Some(js) = &mut self.js {
|
|
js.release_object_group();
|
|
}
|
|
}
|
|
|
|
pub fn take_pending_navigation(&self) -> Option<(String, String, String)> {
|
|
if let Some(js) = &self.js {
|
|
js.take_pending_navigation()
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
pub fn set_intercept_tx(&mut self, tx: tokio::sync::mpsc::UnboundedSender<obscura_js::ops::InterceptedRequest>) {
|
|
self.intercept_tx = Some(tx.clone());
|
|
if let Some(js) = &self.js {
|
|
js.set_intercept_tx(tx);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum PageError {
|
|
#[error("Invalid URL: {0}")]
|
|
InvalidUrl(String),
|
|
|
|
#[error("Network error: {0}")]
|
|
NetworkError(String),
|
|
|
|
#[error("Parse error: {0}")]
|
|
ParseError(String),
|
|
}
|
|
|
|
impl From<ObscuraNetError> for PageError {
|
|
fn from(e: ObscuraNetError) -> Self {
|
|
PageError::NetworkError(e.to_string())
|
|
}
|
|
}
|