Add Target.targetInfoChanged and DOMSnapshot.captureSnapshot for DOM agents (#282)
CDP DOM-agent frameworks (browser-use and similar) could connect but saw a blank page: navigation never refreshed their cached target url/title, and their DOM build aborted with "Unknown domain: DOMSnapshot". - Emit Target.targetInfoChanged after navigation (both CDP Page.navigate and JS-initiated) carrying the post-nav url/title. Strict clients refresh a target only on this event; it also fixes Puppeteer/Playwright page.url() tracking. - Implement DOMSnapshot.captureSnapshot, built from the DOM with synthetic geometry (no paint engine) and plausible computed styles, keyed by the same backendNodeId as DOM.getDocument. Enough for tag/ARIA/accessibility element detection; it runs only when a client requests it, so the fetch path is unaffected. Coordinate clicking still requires a real layout engine.
This commit is contained in:
SGavrl authored and GitHub committed
1 parent
5225f6d985
commit
f4cbb78f0b
4 files changed
+481
No files matched your search
@@ -326,6 +326,7 @@ pub async fn dispatch(req: &CdpRequest, ctx: &mut CdpContext) -> CdpResponse {
|
||||
"Browser" => domains::browser::handle(method, &req.params).await,
|
||||
"Page" => domains::page::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"DOM" => domains::dom::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"DOMSnapshot" => domains::domsnapshot::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Runtime" => domains::runtime::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Network" => domains::network::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Fetch" => domains::fetch::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
//! `DOMSnapshot.captureSnapshot` for layout-free engines.
|
||||
//!
|
||||
//! browser-use (and other CDP DOM-agent frameworks) build their interactive
|
||||
//! element index from `DOMSnapshot.captureSnapshot`: the per-node bounds,
|
||||
//! computed styles, and `isClickable` flag, correlated with `DOM.getDocument`
|
||||
//! by `backendNodeId`. Without this domain their DOM build aborts and the agent
|
||||
//! sees zero elements.
|
||||
//!
|
||||
//! Obscura has no layout/paint engine, so there is no real geometry to report.
|
||||
//! We synthesize it: every node gets a distinct, on-screen, non-icon-sized box
|
||||
//! (a simple vertical stack) plus plausible computed styles (visible, opaque,
|
||||
//! pointer cursor on interactive tags). That is enough for the element
|
||||
//! detection path, which keys off tag name / ARIA / accessibility role and does
|
||||
//! not need true geometry. Clicking still falls back to JS `.click()` since the
|
||||
//! coordinates are synthetic. `backendNodeId == nid`, matching `DOM.getDocument`.
|
||||
|
||||
use obscura_dom::{DomTree, NodeData, NodeId};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
/// Computed-style names browser-use requests, in the exact order it expects to
|
||||
/// read them back out of each layout node's `styles` index array.
|
||||
const REQUIRED_STYLES: &[&str] = &[
|
||||
"display",
|
||||
"visibility",
|
||||
"opacity",
|
||||
"overflow",
|
||||
"overflow-x",
|
||||
"overflow-y",
|
||||
"cursor",
|
||||
"pointer-events",
|
||||
"position",
|
||||
"background-color",
|
||||
];
|
||||
|
||||
/// Cap the synthesized snapshot so a pathologically large DOM cannot produce a
|
||||
/// runaway payload. Matches the spirit of the descendants() length cap.
|
||||
const MAX_NODES: usize = 20_000;
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
_params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" | "disable" => Ok(json!({})),
|
||||
"captureSnapshot" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let url = page.url_string();
|
||||
let title = page.title.clone();
|
||||
page.with_dom(|dom| build_capture_snapshot(dom, &url, &title))
|
||||
.ok_or_else(|| "No DOM loaded".to_string())
|
||||
}
|
||||
// Permissive no-op for the rest of the domain (e.g. getSnapshot) so a
|
||||
// client that probes it does not abort on an Unknown-method error.
|
||||
_ => Ok(json!({})),
|
||||
}
|
||||
}
|
||||
|
||||
/// String table with de-duplication. Every string in a DOMSnapshot response is
|
||||
/// referenced by its index into the top-level `strings` array.
|
||||
struct Interner {
|
||||
map: std::collections::HashMap<String, i64>,
|
||||
list: Vec<String>,
|
||||
}
|
||||
|
||||
impl Interner {
|
||||
fn new() -> Self {
|
||||
let mut s = Interner { map: std::collections::HashMap::new(), list: Vec::new() };
|
||||
s.intern(""); // index 0 is the empty string by convention
|
||||
s
|
||||
}
|
||||
fn intern(&mut self, s: &str) -> i64 {
|
||||
if let Some(&i) = self.map.get(s) {
|
||||
return i;
|
||||
}
|
||||
let i = self.list.len() as i64;
|
||||
self.list.push(s.to_string());
|
||||
self.map.insert(s.to_string(), i);
|
||||
i
|
||||
}
|
||||
}
|
||||
|
||||
/// Pre-order DFS from the document, recording each node and its parent's index.
|
||||
fn walk(
|
||||
dom: &DomTree,
|
||||
id: NodeId,
|
||||
parent: i64,
|
||||
order: &mut Vec<NodeId>,
|
||||
parent_idx: &mut Vec<i64>,
|
||||
) {
|
||||
if order.len() >= MAX_NODES {
|
||||
return;
|
||||
}
|
||||
let my = order.len() as i64;
|
||||
order.push(id);
|
||||
parent_idx.push(parent);
|
||||
for child in dom.children(id) {
|
||||
walk(dom, child, my, order, parent_idx);
|
||||
}
|
||||
}
|
||||
|
||||
fn build_capture_snapshot(dom: &DomTree, url: &str, title: &str) -> Value {
|
||||
let mut order: Vec<NodeId> = Vec::new();
|
||||
let mut parent_idx: Vec<i64> = Vec::new();
|
||||
walk(dom, dom.document(), -1, &mut order, &mut parent_idx);
|
||||
|
||||
let mut strings = Interner::new();
|
||||
let doc_url_idx = strings.intern(url);
|
||||
let title_idx = strings.intern(title);
|
||||
|
||||
let n = order.len();
|
||||
let mut node_type: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut node_name: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut node_value: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut backend_ids: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut attributes: Vec<Value> = Vec::with_capacity(n);
|
||||
let mut clickable: Vec<i64> = Vec::new();
|
||||
|
||||
// Layout arrays are 1:1 with nodes (nodeIndex[i] == i).
|
||||
let mut layout_node_index: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut bounds: Vec<Value> = Vec::with_capacity(n);
|
||||
let mut styles: Vec<Value> = Vec::with_capacity(n);
|
||||
let mut paint_orders: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut client_rects: Vec<Value> = Vec::with_capacity(n);
|
||||
let mut layout_text: Vec<i64> = Vec::with_capacity(n);
|
||||
|
||||
for (i, &nid) in order.iter().enumerate() {
|
||||
let node = match dom.get_node(nid) {
|
||||
Some(node) => node,
|
||||
None => {
|
||||
// Keep arrays aligned even for a vanished node.
|
||||
node_type.push(0);
|
||||
node_name.push(0);
|
||||
node_value.push(0);
|
||||
backend_ids.push(nid.index() as i64);
|
||||
attributes.push(json!([]));
|
||||
layout_node_index.push(i as i64);
|
||||
bounds.push(json!([0.0, 0.0, 0.0, 0.0]));
|
||||
styles.push(json!([]));
|
||||
paint_orders.push(i as i64);
|
||||
client_rects.push(json!([0.0, 0.0, 0.0, 0.0]));
|
||||
layout_text.push(-1);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let (ntype, nname, nval, attrs, tag): (i64, String, String, Vec<(String, String)>, String) =
|
||||
match &node.data {
|
||||
NodeData::Document => (9, "#document".into(), String::new(), vec![], String::new()),
|
||||
NodeData::Doctype { name, .. } => {
|
||||
(10, name.to_string(), String::new(), vec![], String::new())
|
||||
}
|
||||
NodeData::Element { name, attrs, .. } => {
|
||||
let tag = name.local.as_ref().to_string();
|
||||
let flat = attrs
|
||||
.iter()
|
||||
.map(|a| (a.name.local.to_string(), a.value.clone()))
|
||||
.collect();
|
||||
(1, tag.to_ascii_uppercase(), String::new(), flat, tag.to_ascii_lowercase())
|
||||
}
|
||||
NodeData::Text { contents } => {
|
||||
(3, "#text".into(), contents.clone(), vec![], String::new())
|
||||
}
|
||||
NodeData::Comment { contents } => {
|
||||
(8, "#comment".into(), contents.clone(), vec![], String::new())
|
||||
}
|
||||
NodeData::ProcessingInstruction { target, data } => {
|
||||
(7, target.to_string(), data.clone(), vec![], String::new())
|
||||
}
|
||||
};
|
||||
|
||||
node_type.push(ntype);
|
||||
node_name.push(strings.intern(&nname));
|
||||
node_value.push(strings.intern(&nval));
|
||||
backend_ids.push(nid.index() as i64);
|
||||
|
||||
let mut attr_idx: Vec<Value> = Vec::with_capacity(attrs.len() * 2);
|
||||
for (k, v) in &attrs {
|
||||
attr_idx.push(json!(strings.intern(k)));
|
||||
attr_idx.push(json!(strings.intern(v)));
|
||||
}
|
||||
attributes.push(json!(attr_idx));
|
||||
|
||||
let interactive = matches!(
|
||||
tag.as_str(),
|
||||
"a" | "button" | "input" | "select" | "textarea" | "summary" | "details" | "option" | "label"
|
||||
);
|
||||
let has_onclick = attrs.iter().any(|(k, _)| k.eq_ignore_ascii_case("onclick"));
|
||||
if interactive || has_onclick {
|
||||
clickable.push(i as i64);
|
||||
}
|
||||
|
||||
// Tags that never render box content; report display:none so the agent
|
||||
// does not treat them as visible.
|
||||
let hidden = matches!(
|
||||
tag.as_str(),
|
||||
"head" | "meta" | "title" | "script" | "style" | "link" | "noscript" | "base"
|
||||
);
|
||||
let display = if ntype == 1 && hidden { "none" } else { "block" };
|
||||
let cursor = if interactive { "pointer" } else { "auto" };
|
||||
let style_vals = [
|
||||
display,
|
||||
"visible",
|
||||
"1",
|
||||
"visible",
|
||||
"visible",
|
||||
"visible",
|
||||
cursor,
|
||||
"auto",
|
||||
"static",
|
||||
"rgba(0, 0, 0, 0)",
|
||||
];
|
||||
debug_assert_eq!(style_vals.len(), REQUIRED_STYLES.len());
|
||||
let style_idx: Vec<Value> = style_vals.iter().map(|s| json!(strings.intern(s))).collect();
|
||||
styles.push(json!(style_idx));
|
||||
|
||||
// Synthetic geometry: a vertical stack, full-width, 18px tall. Distinct
|
||||
// and non-icon-sized so visibility/size heuristics include the element;
|
||||
// the coordinates are not real (no layout engine).
|
||||
let y = (i as f64) * 18.0;
|
||||
bounds.push(json!([0.0, y, 1280.0, 18.0]));
|
||||
client_rects.push(json!([0.0, y, 1280.0, 18.0]));
|
||||
paint_orders.push(i as i64);
|
||||
layout_node_index.push(i as i64);
|
||||
layout_text.push(-1);
|
||||
}
|
||||
|
||||
let content_height = (n as i64) * 18;
|
||||
json!({
|
||||
"documents": [{
|
||||
"documentURL": doc_url_idx,
|
||||
"title": title_idx,
|
||||
"baseURL": doc_url_idx,
|
||||
"contentLanguage": 0,
|
||||
"encodingName": 0,
|
||||
"publicId": 0,
|
||||
"systemId": 0,
|
||||
"frameId": 0,
|
||||
"nodes": {
|
||||
"parentIndex": parent_idx,
|
||||
"nodeType": node_type,
|
||||
"nodeName": node_name,
|
||||
"nodeValue": node_value,
|
||||
"backendNodeId": backend_ids,
|
||||
"attributes": attributes,
|
||||
"isClickable": { "index": clickable },
|
||||
},
|
||||
"layout": {
|
||||
"nodeIndex": layout_node_index,
|
||||
"styles": styles,
|
||||
"bounds": bounds,
|
||||
"text": layout_text,
|
||||
"paintOrders": paint_orders,
|
||||
"clientRects": client_rects,
|
||||
},
|
||||
"textBoxes": { "layoutIndex": [], "bounds": [], "start": [], "length": [] },
|
||||
"scrollOffsetX": 0.0,
|
||||
"scrollOffsetY": 0.0,
|
||||
"contentWidth": 1280,
|
||||
"contentHeight": content_height,
|
||||
}],
|
||||
"strings": strings.list,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
fn collect_backend_ids(node: &Value, out: &mut Vec<i64>) {
|
||||
if let Some(id) = node.get("backendNodeId").and_then(|v| v.as_i64()) {
|
||||
out.push(id);
|
||||
}
|
||||
if let Some(children) = node.get("children").and_then(|v| v.as_array()) {
|
||||
for c in children {
|
||||
collect_backend_ids(c, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn find_backend_id_by_name(node: &Value, name: &str) -> Option<i64> {
|
||||
if node.get("nodeName").and_then(|v| v.as_str()) == Some(name) {
|
||||
return node.get("backendNodeId").and_then(|v| v.as_i64());
|
||||
}
|
||||
node.get("children")
|
||||
.and_then(|v| v.as_array())
|
||||
.and_then(|children| children.iter().find_map(|c| find_backend_id_by_name(c, name)))
|
||||
}
|
||||
|
||||
async fn navigate(ctx: &mut CdpContext, body: &str) -> String {
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = format!("{}-session", page_id);
|
||||
ctx.sessions.insert(session_id.clone(), page_id.clone());
|
||||
let url = format!("data:text/html,{body}");
|
||||
crate::domains::page::handle(
|
||||
"navigate",
|
||||
&json!({ "url": url, "waitUntil": "load" }),
|
||||
ctx,
|
||||
&Some(session_id.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("navigate should succeed");
|
||||
session_id
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capture_snapshot_matches_getdocument_and_flags_clickable() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let session = navigate(&mut ctx, "<button id=go>Go</button><a href=/x>L</a>").await;
|
||||
|
||||
// DOM.getDocument is the structural source; the snapshot must use the
|
||||
// same backendNodeId scheme so a client can correlate the two.
|
||||
let doc = crate::domains::dom::handle(
|
||||
"getDocument",
|
||||
&json!({ "depth": -1 }),
|
||||
&mut ctx,
|
||||
&Some(session.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("getDocument should succeed");
|
||||
let mut doc_ids = Vec::new();
|
||||
collect_backend_ids(&doc["root"], &mut doc_ids);
|
||||
assert!(!doc_ids.is_empty(), "getDocument returned no nodes");
|
||||
|
||||
let snap = handle("captureSnapshot", &json!({}), &mut ctx, &Some(session.clone()))
|
||||
.await
|
||||
.expect("captureSnapshot should succeed");
|
||||
|
||||
let documents = snap["documents"].as_array().expect("documents array");
|
||||
assert_eq!(documents.len(), 1);
|
||||
let nodes = &documents[0]["nodes"];
|
||||
let layout = &documents[0]["layout"];
|
||||
|
||||
let snap_ids: Vec<i64> = nodes["backendNodeId"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_i64().unwrap())
|
||||
.collect();
|
||||
|
||||
// Every node from getDocument must appear in the snapshot under the
|
||||
// identical backendNodeId.
|
||||
for id in &doc_ids {
|
||||
assert!(
|
||||
snap_ids.contains(id),
|
||||
"snapshot is missing backendNodeId {id} present in getDocument"
|
||||
);
|
||||
}
|
||||
|
||||
// String table is populated (everything is referenced by index).
|
||||
assert!(
|
||||
snap["strings"].as_array().unwrap().len() > 1,
|
||||
"string table should be populated"
|
||||
);
|
||||
|
||||
// Layout arrays are aligned 1:1 with nodes and carry bounds + the 10
|
||||
// computed styles browser-use reads back positionally.
|
||||
let n = snap_ids.len();
|
||||
assert_eq!(layout["nodeIndex"].as_array().unwrap().len(), n);
|
||||
assert_eq!(layout["bounds"].as_array().unwrap().len(), n);
|
||||
assert_eq!(layout["styles"].as_array().unwrap().len(), n);
|
||||
assert_eq!(
|
||||
layout["bounds"][0].as_array().unwrap().len(),
|
||||
4,
|
||||
"bounds entries are [x, y, w, h]"
|
||||
);
|
||||
assert_eq!(
|
||||
layout["styles"][0].as_array().unwrap().len(),
|
||||
REQUIRED_STYLES.len(),
|
||||
"each layout node carries all required computed styles"
|
||||
);
|
||||
|
||||
// Interactive elements are flagged isClickable (by node index).
|
||||
let clickable: Vec<i64> = nodes["isClickable"]["index"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_i64().unwrap())
|
||||
.collect();
|
||||
for tag in ["BUTTON", "A"] {
|
||||
let bid = find_backend_id_by_name(&doc["root"], tag)
|
||||
.unwrap_or_else(|| panic!("{tag} should be in the document"));
|
||||
let idx = snap_ids
|
||||
.iter()
|
||||
.position(|&id| id == bid)
|
||||
.unwrap_or_else(|| panic!("{tag} should be in the snapshot")) as i64;
|
||||
assert!(clickable.contains(&idx), "{tag} must be flagged isClickable");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_domsnapshot_method_is_permissive_noop() {
|
||||
// Probing the domain (e.g. getSnapshot) must not abort with an
|
||||
// Unknown-method error the way an unhandled domain would.
|
||||
let mut ctx = CdpContext::new();
|
||||
let r = handle("getSnapshot", &json!({}), &mut ctx, &None)
|
||||
.await
|
||||
.expect("unknown DOMSnapshot methods are a permissive no-op");
|
||||
assert!(r.is_object());
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ pub mod target;
|
||||
pub mod browser;
|
||||
pub mod page;
|
||||
pub mod dom;
|
||||
pub mod domsnapshot;
|
||||
pub mod runtime;
|
||||
pub mod network;
|
||||
pub mod fetch;
|
||||
|
||||
@@ -130,6 +130,30 @@ pub fn emit_navigation_events(
|
||||
}
|
||||
phase3.push(CdpEvent { method: "Page.frameStoppedLoading".into(), params: json!({"frameId": frame_id}), session_id: es });
|
||||
ctx.pending_events.extend(phase3);
|
||||
|
||||
// Target.targetInfoChanged: strict CDP clients (browser-use, and
|
||||
// Puppeteer/Playwright `page.url()` tracking) cache the TargetInfo from
|
||||
// attachedToTarget and only refresh it on this event. Without it they keep
|
||||
// reporting the pre-navigation url/title (about:blank) and never see the
|
||||
// loaded page. Emit it browser-level (no sessionId) with the new url/title.
|
||||
let (tic_title, tic_ctx) = ctx
|
||||
.get_page(page_id)
|
||||
.map(|p| (p.title.clone(), p.context.id.clone()))
|
||||
.unwrap_or_default();
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.targetInfoChanged",
|
||||
json!({
|
||||
"targetInfo": {
|
||||
"targetId": page_id,
|
||||
"type": "page",
|
||||
"title": tic_title,
|
||||
"url": page_url,
|
||||
"attached": true,
|
||||
"canAccessOpener": false,
|
||||
"browserContextId": tic_ctx,
|
||||
}
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
/// Parse the `waitUntil` argument that Puppeteer/Playwright pass on
|
||||
@@ -584,4 +608,54 @@ mod tests {
|
||||
"captureSnapshot must NOT fall through: {err2}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn navigation_emits_target_info_changed_with_url_and_title() {
|
||||
// Strict CDP clients (browser-use, Puppeteer/Playwright `page.url()`)
|
||||
// refresh a target's url/title only on Target.targetInfoChanged. A
|
||||
// navigation must emit it with the post-nav url/title, otherwise those
|
||||
// clients stay stuck on the pre-nav about:blank.
|
||||
let mut ctx = CdpContext::new();
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = format!("{}-session", page_id);
|
||||
ctx.sessions.insert(session_id.clone(), page_id.clone());
|
||||
|
||||
let params = json!({
|
||||
"url": "data:text/html,<title>Hello</title><button>B</button>",
|
||||
"waitUntil": "load",
|
||||
});
|
||||
handle("navigate", ¶ms, &mut ctx, &Some(session_id.clone()))
|
||||
.await
|
||||
.expect("navigate should succeed");
|
||||
|
||||
let evt = ctx
|
||||
.pending_events
|
||||
.iter()
|
||||
.find(|e| e.method == "Target.targetInfoChanged")
|
||||
.expect("navigation must emit Target.targetInfoChanged");
|
||||
// Browser-level event (no sessionId) so the root connection's
|
||||
// targetInfoChanged handler receives it.
|
||||
assert!(
|
||||
evt.session_id.is_none(),
|
||||
"targetInfoChanged must be browser-level (no sessionId)"
|
||||
);
|
||||
let info = evt.params["targetInfo"].clone();
|
||||
|
||||
// The payload must carry the live post-navigation url/title and the
|
||||
// canAccessOpener field strict clients require on every TargetInfo.
|
||||
let (exp_url, exp_title) = {
|
||||
let page = ctx.get_page(&page_id).expect("page exists");
|
||||
(page.url_string(), page.title.clone())
|
||||
};
|
||||
assert_eq!(info["targetId"], json!(page_id));
|
||||
assert_eq!(info["type"], "page");
|
||||
assert_eq!(info["url"], json!(exp_url));
|
||||
assert_eq!(info["title"], json!(exp_title));
|
||||
assert!(
|
||||
info["url"].as_str().unwrap_or_default().starts_with("data:"),
|
||||
"url should reflect the navigated page, got {}",
|
||||
info["url"]
|
||||
);
|
||||
assert_eq!(info["canAccessOpener"], json!(false));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user