fix(cli): make --dump text extraction iterative (deep-DOM crash) (#368)
extract_readable_text recursed once per DOM nesting level, so a deeply nested page overflowed the call stack and aborted the process (exit 134) on --dump text. This is the CLI counterpart of the serialize/textContent walks made iterative in obscura-dom; that fix did not cover this function. Rewrite it as an iterative DFS over an explicit work stack, with a Newline work item for a block element's trailing newline so output is byte-for-byte unchanged, plus the same defense-in-depth node cap the DOM walkers use. Relates to #362.
This commit is contained in:
1 parent
b662ff8d72
commit
8f26dfa331
1 file changed
+50
-21
@@ -940,10 +940,42 @@ fn dump_markdown(page: &mut Page) -> String {
|
|||||||
fn extract_readable_text(dom: &obscura_dom::DomTree, node_id: obscura_dom::NodeId) -> String {
|
fn extract_readable_text(dom: &obscura_dom::DomTree, node_id: obscura_dom::NodeId) -> String {
|
||||||
use obscura_dom::NodeData;
|
use obscura_dom::NodeData;
|
||||||
|
|
||||||
|
// Iterative DFS over an explicit work stack. A recursive walk overflowed the
|
||||||
|
// call stack (a hard abort, not a catchable panic) on deeply nested pages,
|
||||||
|
// taking down the process on `--dump text` (issue #362, the CLI counterpart
|
||||||
|
// of the serialize/textContent paths made iterative in obscura-dom). A
|
||||||
|
// `Newline` work item emits a block element's trailing newline after its
|
||||||
|
// children, matching the old pre/post-recursion output exactly.
|
||||||
|
enum Work {
|
||||||
|
Visit(obscura_dom::NodeId),
|
||||||
|
Newline,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Defense-in-depth cap mirroring DomTree::descendants; never reached on a
|
||||||
|
// valid tree since append_child / insert_before reject cycles.
|
||||||
|
const MAX_NODES: usize = 5_000_000;
|
||||||
|
|
||||||
let mut result = String::new();
|
let mut result = String::new();
|
||||||
let node = match dom.get_node(node_id) {
|
let mut stack: Vec<Work> = vec![Work::Visit(node_id)];
|
||||||
|
let mut visited = 0usize;
|
||||||
|
|
||||||
|
while let Some(work) = stack.pop() {
|
||||||
|
let id = match work {
|
||||||
|
Work::Newline => {
|
||||||
|
result.push('\n');
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Work::Visit(id) => id,
|
||||||
|
};
|
||||||
|
|
||||||
|
visited += 1;
|
||||||
|
if visited > MAX_NODES {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
let node = match dom.get_node(id) {
|
||||||
Some(n) => n,
|
Some(n) => n,
|
||||||
None => return result,
|
None => continue,
|
||||||
};
|
};
|
||||||
|
|
||||||
match &node.data {
|
match &node.data {
|
||||||
@@ -955,6 +987,14 @@ fn extract_readable_text(dom: &obscura_dom::DomTree, node_id: obscura_dom::NodeI
|
|||||||
}
|
}
|
||||||
NodeData::Element { name, .. } => {
|
NodeData::Element { name, .. } => {
|
||||||
let tag = name.local.as_ref();
|
let tag = name.local.as_ref();
|
||||||
|
|
||||||
|
// Boilerplate elements rarely contain content the user wants to
|
||||||
|
// scrape — strip them so `--dump text` returns the article body
|
||||||
|
// instead of menus, footers, and cookie banners.
|
||||||
|
if matches!(tag, "script" | "style" | "nav" | "header" | "footer" | "aside") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
let is_block = matches!(
|
let is_block = matches!(
|
||||||
tag,
|
tag,
|
||||||
"div" | "p" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6"
|
"div" | "p" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6"
|
||||||
@@ -965,31 +1005,20 @@ fn extract_readable_text(dom: &obscura_dom::DomTree, node_id: obscura_dom::NodeI
|
|||||||
| "ul" | "ol"
|
| "ul" | "ol"
|
||||||
);
|
);
|
||||||
|
|
||||||
// Boilerplate elements rarely contain content the user wants to
|
|
||||||
// scrape — strip them so `--dump text` returns the article body
|
|
||||||
// instead of menus, footers, and cookie banners.
|
|
||||||
if matches!(
|
|
||||||
tag,
|
|
||||||
"script" | "style" | "nav" | "header" | "footer" | "aside"
|
|
||||||
) {
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
if is_block {
|
if is_block {
|
||||||
result.push('\n');
|
result.push('\n');
|
||||||
|
// Processed after all children (stack is LIFO): the trailing newline.
|
||||||
|
stack.push(Work::Newline);
|
||||||
}
|
}
|
||||||
|
// Push children in reverse so they pop in document order.
|
||||||
for child_id in dom.children(node_id) {
|
for child_id in dom.children(id).into_iter().rev() {
|
||||||
result.push_str(&extract_readable_text(dom, child_id));
|
stack.push(Work::Visit(child_id));
|
||||||
}
|
|
||||||
|
|
||||||
if is_block {
|
|
||||||
result.push('\n');
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
for child_id in dom.children(node_id) {
|
for child_id in dom.children(id).into_iter().rev() {
|
||||||
result.push_str(&extract_readable_text(dom, child_id));
|
stack.push(Work::Visit(child_id));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user