feat(cdp): implement Accessibility domain (getFullAXTree)
Implements the CDP Accessibility domain with getFullAXTree support,
enabling AI agent tools (agent-browser, Playwright MCP) that rely on
Accessibility.getFullAXTree to work with Obscura.
Changes:
- Add crates/obscura-cdp/src/domains/accessibility.rs (~380 lines)
- Accessibility.enable: returns {}
- Accessibility.getFullAXTree: walks DOM tree, produces AXNode array
- Role mapping: 50+ HTML tags → ARIA roles, respects explicit role attr
- Name computation: aria-label → aria-labelledby → alt → title → placeholder
- Properties: focusable, editable, checked, disabled, level, required, multiline
- Update mod.rs: register accessibility module
- Update dispatch.rs: route Accessibility domain to real handler
Tested with example.com (18 nodes), httpbin.org/html (16 nodes),
news.ycombinator.com (1308 nodes). Zero structure errors.
Closes #12
This commit is contained in:
1 parent
99e75f1e62
commit
a4591707e3
3 files changed
+381
-1
No files matched your search
@@ -135,8 +135,9 @@ pub async fn dispatch(req: &CdpRequest, ctx: &mut CdpContext) -> CdpResponse {
|
|||||||
"Input" => domains::input::handle(method, &req.params, ctx, &req.session_id).await,
|
"Input" => domains::input::handle(method, &req.params, ctx, &req.session_id).await,
|
||||||
"Storage" => domains::storage::handle(method, &req.params, ctx, &req.session_id).await,
|
"Storage" => domains::storage::handle(method, &req.params, ctx, &req.session_id).await,
|
||||||
"LP" => domains::lp::handle(method, &req.params, ctx, &req.session_id).await,
|
"LP" => domains::lp::handle(method, &req.params, ctx, &req.session_id).await,
|
||||||
|
"Accessibility" => domains::accessibility::handle(method, &req.params, ctx, &req.session_id).await,
|
||||||
"Emulation" | "Log" | "Performance" | "Security" | "CSS"
|
"Emulation" | "Log" | "Performance" | "Security" | "CSS"
|
||||||
| "Accessibility" | "ServiceWorker" | "Inspector"
|
| "ServiceWorker" | "Inspector"
|
||||||
| "Debugger" | "Profiler" | "HeapProfiler" | "Overlay" => {
|
| "Debugger" | "Profiler" | "HeapProfiler" | "Overlay" => {
|
||||||
Ok(json!({}))
|
Ok(json!({}))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,378 @@
|
|||||||
|
use obscura_dom::{DomTree, NodeData, NodeId};
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
|
use crate::dispatch::CdpContext;
|
||||||
|
|
||||||
|
pub async fn handle(
|
||||||
|
method: &str,
|
||||||
|
_params: &Value,
|
||||||
|
ctx: &mut CdpContext,
|
||||||
|
session_id: &Option<String>,
|
||||||
|
) -> Result<Value, String> {
|
||||||
|
match method {
|
||||||
|
"enable" => Ok(json!({})),
|
||||||
|
"getFullAXTree" => {
|
||||||
|
let page = ctx
|
||||||
|
.get_session_page(session_id)
|
||||||
|
.ok_or("No page")?;
|
||||||
|
let nodes = page
|
||||||
|
.with_dom(|dom| build_ax_nodes(dom))
|
||||||
|
.unwrap_or_default();
|
||||||
|
Ok(json!({ "nodes": nodes }))
|
||||||
|
}
|
||||||
|
_ => Ok(json!({})),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Walk the full DOM tree and produce CDP Accessibility AXNode array.
|
||||||
|
fn build_ax_nodes(dom: &DomTree) -> Vec<Value> {
|
||||||
|
let mut nodes: Vec<Value> = Vec::new();
|
||||||
|
let mut id_counter: u32 = 0;
|
||||||
|
// Map DOM NodeId → AX string id, populated only for nodes actually in the AX tree
|
||||||
|
let mut dom_to_ax: std::collections::HashMap<u32, String> = std::collections::HashMap::new();
|
||||||
|
|
||||||
|
let document = dom.document();
|
||||||
|
|
||||||
|
// Collect all DOM nodes in tree order (root + descendants)
|
||||||
|
let mut all_dom_ids: Vec<NodeId> = vec![document];
|
||||||
|
all_dom_ids.extend(dom.descendants(document));
|
||||||
|
|
||||||
|
// First pass: assign AX IDs only to nodes that will produce an AX node
|
||||||
|
let mut eligible: Vec<NodeId> = Vec::new();
|
||||||
|
for dom_id in &all_dom_ids {
|
||||||
|
// Quick check without full build_ax_node (just role check to avoid borrow issues)
|
||||||
|
if let Some(node) = dom.get_node(*dom_id) {
|
||||||
|
let role = map_role(&node.data);
|
||||||
|
if !role.is_empty() {
|
||||||
|
id_counter += 1;
|
||||||
|
dom_to_ax.insert(dom_id.raw(), id_counter.to_string());
|
||||||
|
eligible.push(*dom_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Second pass: build AXNode for eligible nodes
|
||||||
|
for dom_id in &eligible {
|
||||||
|
if let Some(ax) = build_ax_node(dom, *dom_id, &dom_to_ax) {
|
||||||
|
nodes.push(ax);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_ax_node(
|
||||||
|
dom: &DomTree,
|
||||||
|
node_id: NodeId,
|
||||||
|
dom_to_ax: &std::collections::HashMap<u32, String>,
|
||||||
|
) -> Option<Value> {
|
||||||
|
let node = dom.get_node(node_id)?;
|
||||||
|
let ax_id = dom_to_ax.get(&node_id.raw())?.clone();
|
||||||
|
|
||||||
|
let role = map_role(&node.data);
|
||||||
|
// Skip non-relevant nodes (Document, Doctype, Comment, PI)
|
||||||
|
if role.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let name = compute_name(dom, &node);
|
||||||
|
let value = compute_value(dom, &node);
|
||||||
|
let properties = compute_properties(dom, &node);
|
||||||
|
|
||||||
|
let child_ids: Vec<String> = dom
|
||||||
|
.children(node_id)
|
||||||
|
.iter()
|
||||||
|
.filter_map(|child_id| dom_to_ax.get(&child_id.raw()).cloned())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Some(json!({
|
||||||
|
"nodeId": ax_id,
|
||||||
|
"ignored": false,
|
||||||
|
"role": role,
|
||||||
|
"name": name,
|
||||||
|
"value": value,
|
||||||
|
"properties": properties,
|
||||||
|
"childIds": child_ids,
|
||||||
|
"backendDOMNodeId": node_id.raw(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Map HTML element tag to ARIA role value.
|
||||||
|
fn map_role(data: &NodeData) -> &'static str {
|
||||||
|
match data {
|
||||||
|
NodeData::Document => "RootWebArea",
|
||||||
|
NodeData::Element { name, attrs, .. } => {
|
||||||
|
let tag = name.local.as_ref();
|
||||||
|
|
||||||
|
// Check explicit role attribute first
|
||||||
|
if let Some(role_attr) = attrs.iter().find(|a| a.name.local.as_ref() == "role") {
|
||||||
|
return match role_attr.value.as_str() {
|
||||||
|
"button" => "button",
|
||||||
|
"link" => "link",
|
||||||
|
"heading" => "heading",
|
||||||
|
"textbox" | "searchbox" => "textbox",
|
||||||
|
"checkbox" => "checkbox",
|
||||||
|
"radio" => "radio",
|
||||||
|
"listbox" => "listbox",
|
||||||
|
"combobox" => "combobox",
|
||||||
|
"list" => "list",
|
||||||
|
"listitem" => "listitem",
|
||||||
|
"navigation" => "navigation",
|
||||||
|
"banner" => "banner",
|
||||||
|
"main" => "main",
|
||||||
|
"complementary" => "complementary",
|
||||||
|
"contentinfo" => "contentinfo",
|
||||||
|
"form" => "form",
|
||||||
|
"table" => "table",
|
||||||
|
"row" => "row",
|
||||||
|
"cell" | "gridcell" => "cell",
|
||||||
|
"img" => "image",
|
||||||
|
"dialog" => "dialog",
|
||||||
|
"alert" => "alert",
|
||||||
|
"tab" => "tab",
|
||||||
|
"tablist" => "tablist",
|
||||||
|
"tabpanel" => "tabpanel",
|
||||||
|
"menu" => "menu",
|
||||||
|
"menuitem" => "menuitem",
|
||||||
|
"toolbar" => "toolbar",
|
||||||
|
"separator" => "separator",
|
||||||
|
"presentation" | "none" => {
|
||||||
|
// presentation/none roles get the role but content is still in tree
|
||||||
|
"presentation"
|
||||||
|
}
|
||||||
|
_ => "generic",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
match tag {
|
||||||
|
"a" => {
|
||||||
|
if attrs.iter().any(|a| a.name.local.as_ref() == "href") {
|
||||||
|
"link"
|
||||||
|
} else {
|
||||||
|
"generic"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"button" | "summary" => "button",
|
||||||
|
"input" => {
|
||||||
|
let type_attr = attrs
|
||||||
|
.iter()
|
||||||
|
.find(|a| a.name.local.as_ref() == "type")
|
||||||
|
.map(|a| a.value.as_str())
|
||||||
|
.unwrap_or("text");
|
||||||
|
match type_attr {
|
||||||
|
"submit" | "reset" | "button" | "image" => "button",
|
||||||
|
"checkbox" => "checkbox",
|
||||||
|
"radio" => "radio",
|
||||||
|
"range" => "slider",
|
||||||
|
"number" => "spinbutton",
|
||||||
|
"search" => "searchbox",
|
||||||
|
_ => "textbox",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"textarea" => "textbox",
|
||||||
|
"select" => {
|
||||||
|
if attrs.iter().any(|a| {
|
||||||
|
a.name.local.as_ref() == "multiple"
|
||||||
|
|| a.name.local.as_ref() == "size"
|
||||||
|
}) {
|
||||||
|
"listbox"
|
||||||
|
} else {
|
||||||
|
"combobox"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"h1" | "h2" | "h3" | "h4" | "h5" | "h6" => "heading",
|
||||||
|
"img" | "svg" => "image",
|
||||||
|
"ul" | "ol" | "menu" => "list",
|
||||||
|
"li" => "listitem",
|
||||||
|
"table" => "table",
|
||||||
|
"tr" => "row",
|
||||||
|
"td" | "th" => "cell",
|
||||||
|
"nav" => "navigation",
|
||||||
|
"header" => "banner",
|
||||||
|
"main" => "main",
|
||||||
|
"footer" => "contentinfo",
|
||||||
|
"form" => "form",
|
||||||
|
"dialog" => "dialog",
|
||||||
|
"hr" => "separator",
|
||||||
|
"label" => "LabelText",
|
||||||
|
"article" => "article",
|
||||||
|
"aside" => "complementary",
|
||||||
|
"section" => "region",
|
||||||
|
"figure" => "figure",
|
||||||
|
"figcaption" => "StaticText",
|
||||||
|
"p" | "div" | "span" | "pre" | "blockquote" | "code"
|
||||||
|
| "em" | "strong" | "b" | "i" | "u" | "s" | "small"
|
||||||
|
| "sub" | "sup" | "mark" | "del" | "ins" => "generic",
|
||||||
|
"iframe" => "Iframe",
|
||||||
|
_ => "generic",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
NodeData::Text { .. } => "StaticText",
|
||||||
|
NodeData::Doctype { .. } | NodeData::Comment { .. } | NodeData::ProcessingInstruction { .. } => {
|
||||||
|
""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compute the accessible name for a node.
|
||||||
|
fn compute_name(dom: &DomTree, node: &obscura_dom::Node) -> Option<Value> {
|
||||||
|
if let NodeData::Element { attrs, .. } = &node.data {
|
||||||
|
// aria-label takes highest priority
|
||||||
|
if let Some(label) = attrs
|
||||||
|
.iter()
|
||||||
|
.find(|a| a.name.local.as_ref() == "aria-label")
|
||||||
|
.map(|a| a.value.clone())
|
||||||
|
{
|
||||||
|
return Some(json!(label));
|
||||||
|
}
|
||||||
|
|
||||||
|
// aria-labelledby
|
||||||
|
if let Some(labelledby) = attrs
|
||||||
|
.iter()
|
||||||
|
.find(|a| a.name.local.as_ref() == "aria-labelledby")
|
||||||
|
{
|
||||||
|
let ids: Vec<&str> = labelledby.value.split_whitespace().collect();
|
||||||
|
let mut name = String::new();
|
||||||
|
for id_str in ids {
|
||||||
|
if let Some(ref_id) = dom.get_element_by_id(id_str) {
|
||||||
|
name.push_str(&dom.text_content(ref_id));
|
||||||
|
name.push(' ');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let trimmed = name.trim().to_string();
|
||||||
|
if !trimmed.is_empty() {
|
||||||
|
return Some(json!(trimmed));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// alt attribute for images
|
||||||
|
if let Some(alt) = attrs
|
||||||
|
.iter()
|
||||||
|
.find(|a| a.name.local.as_ref() == "alt")
|
||||||
|
.map(|a| a.value.clone())
|
||||||
|
{
|
||||||
|
if !alt.is_empty() {
|
||||||
|
return Some(json!(alt));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// title attribute
|
||||||
|
if let Some(title) = attrs
|
||||||
|
.iter()
|
||||||
|
.find(|a| a.name.local.as_ref() == "title")
|
||||||
|
.map(|a| a.value.clone())
|
||||||
|
{
|
||||||
|
if !title.is_empty() {
|
||||||
|
return Some(json!(title));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// placeholder
|
||||||
|
if let Some(placeholder) = attrs
|
||||||
|
.iter()
|
||||||
|
.find(|a| a.name.local.as_ref() == "placeholder")
|
||||||
|
.map(|a| a.value.clone())
|
||||||
|
{
|
||||||
|
if !placeholder.is_empty() {
|
||||||
|
return Some(json!(placeholder));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// For text nodes, the name is the text content
|
||||||
|
if let NodeData::Text { contents } = &node.data {
|
||||||
|
let trimmed = contents.trim().to_string();
|
||||||
|
if !trimmed.is_empty() {
|
||||||
|
return Some(json!(trimmed));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compute the accessible value for a node (e.g., current input value).
|
||||||
|
fn compute_value(_dom: &DomTree, node: &obscura_dom::Node) -> Option<Value> {
|
||||||
|
if let NodeData::Element { name, attrs, .. } = &node.data {
|
||||||
|
let tag = name.local.as_ref();
|
||||||
|
// For input elements, return the value attribute
|
||||||
|
if tag == "input" || tag == "textarea" || tag == "select" {
|
||||||
|
if let Some(val) = attrs
|
||||||
|
.iter()
|
||||||
|
.find(|a| a.name.local.as_ref() == "value")
|
||||||
|
.map(|a| json!(a.value.clone()))
|
||||||
|
{
|
||||||
|
return Some(val);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compute accessibility properties for a node.
|
||||||
|
fn compute_properties(_dom: &DomTree, node: &obscura_dom::Node) -> Vec<Value> {
|
||||||
|
if let NodeData::Element { name, attrs, .. } = &node.data {
|
||||||
|
let tag = name.local.as_ref();
|
||||||
|
let mut props = Vec::new();
|
||||||
|
|
||||||
|
// focusable
|
||||||
|
let focusable = matches!(
|
||||||
|
tag,
|
||||||
|
"a" | "button" | "input" | "select" | "textarea" | "details" | "summary"
|
||||||
|
) || attrs.iter().any(|a| {
|
||||||
|
let an = a.name.local.as_ref();
|
||||||
|
an == "tabindex" || an == "contenteditable"
|
||||||
|
});
|
||||||
|
if focusable {
|
||||||
|
props.push(json!({"name": "focusable", "value": {"type": "boolean", "value": true}}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// editable
|
||||||
|
if tag == "input" || tag == "textarea"
|
||||||
|
|| attrs
|
||||||
|
.iter()
|
||||||
|
.any(|a| a.name.local.as_ref() == "contenteditable" && a.value != "false")
|
||||||
|
{
|
||||||
|
props.push(json!({"name": "editable", "value": {"type": "boolean", "value": true}}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// checked for checkboxes/radios
|
||||||
|
if attrs
|
||||||
|
.iter()
|
||||||
|
.any(|a| a.name.local.as_ref() == "checked")
|
||||||
|
{
|
||||||
|
props.push(json!({"name": "checked", "value": {"type": "boolean", "value": true}}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// disabled
|
||||||
|
if attrs
|
||||||
|
.iter()
|
||||||
|
.any(|a| a.name.local.as_ref() == "disabled")
|
||||||
|
{
|
||||||
|
props.push(json!({"name": "disabled", "value": {"type": "boolean", "value": true}}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// level for headings
|
||||||
|
if let Some(level) = tag.strip_prefix('h').and_then(|s| s.parse::<u32>().ok()) {
|
||||||
|
if level >= 1 && level <= 6 {
|
||||||
|
props.push(json!({"name": "level", "value": {"type": "integer", "value": level}}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// required
|
||||||
|
if attrs
|
||||||
|
.iter()
|
||||||
|
.any(|a| a.name.local.as_ref() == "required" || a.name.local.as_ref() == "aria-required")
|
||||||
|
{
|
||||||
|
props.push(json!({"name": "required", "value": {"type": "boolean", "value": true}}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// multiline for textarea
|
||||||
|
if tag == "textarea" {
|
||||||
|
props.push(json!({"name": "multiline", "value": {"type": "boolean", "value": true}}));
|
||||||
|
}
|
||||||
|
|
||||||
|
props
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,4 +7,5 @@ pub mod network;
|
|||||||
pub mod fetch;
|
pub mod fetch;
|
||||||
pub mod input;
|
pub mod input;
|
||||||
pub mod storage;
|
pub mod storage;
|
||||||
|
pub mod accessibility;
|
||||||
pub mod lp;
|
pub mod lp;
|
||||||
Reference in new issue
Block a user