diff --git a/crates/obscura-cli/src/main.rs b/crates/obscura-cli/src/main.rs index 768032e..bacb760 100644 --- a/crates/obscura-cli/src/main.rs +++ b/crates/obscura-cli/src/main.rs @@ -471,7 +471,13 @@ fn extract_readable_text(dom: &obscura_dom::DomTree, node_id: obscura_dom::NodeI | "ul" | "ol" ); - if tag == "script" || tag == "style" { + // Boilerplate elements rarely contain content the user wants to + // scrape — strip them so `--dump text` returns the article body + // instead of menus, footers, and cookie banners. + if matches!( + tag, + "script" | "style" | "nav" | "header" | "footer" | "aside" + ) { return result; } @@ -872,3 +878,54 @@ mod tests { assert!(is_quiet_command(&cmd)); } } + +#[cfg(test)] +mod tests { + use super::extract_readable_text; + use obscura_dom::parse_html; + + fn body_text(html: &str) -> String { + let dom = parse_html(html); + let body = dom + .query_selector("body") + .ok() + .flatten() + .expect("body must exist"); + extract_readable_text(&dom, body).split_whitespace().collect::>().join(" ") + } + + #[test] + fn skips_nav_header_footer_aside() { + let text = body_text( + r#" +
SITE HEADER
+ + +

Article body.

+ + "#, + ); + assert!(text.contains("Article body."), "main content kept: {text}"); + for boilerplate in ["SITE HEADER", "NAV LINKS", "SIDEBAR", "FOOTER"] { + assert!( + !text.contains(boilerplate), + "boilerplate '{boilerplate}' leaked through: {text}" + ); + } + } + + #[test] + fn still_skips_script_and_style() { + // Regression guard for the original skip list. + let text = body_text( + r#" +

Hello.

+ + + "#, + ); + assert!(text.contains("Hello.")); + assert!(!text.contains("console.log")); + assert!(!text.contains("color: red")); + } +}