fix(cli): strip nav/header/footer/aside from --dump text output (#55) (#82)

`obscura fetch --dump text` walked the entire <body> and emitted the
text of navigation menus, sidebars, and footers alongside the article
body. The skip list inside extract_readable_text already excluded
<script>/<style>; extend it with the four standard semantic-boilerplate
tags so the dump is closer to the article content users actually want
to scrape.

Behaviour change for users who relied on the noise being included is
acknowledged but unlikely — these tags are the canonical wrappers
boilerplate goes in. Anyone needing the full body should use --dump
html and post-process.

Generated by Ora Studio
Vibe coded by ousamabenyounes

Co-authored-by: Ora Studio <noreply@oratelecom.net>
This commit is contained in:
Ben YounesandOra Studio authored and GitHub committed 2026-05-09 17:32:59 +02:00
1 parent 81b9cf0f42
commit 7dfc2a9c01
1 file changed
+58 -1
+58 -1
View File
@@ -471,7 +471,13 @@ fn extract_readable_text(dom: &obscura_dom::DomTree, node_id: obscura_dom::NodeI
| "ul" | "ol"
);
if tag == "script" || tag == "style" {
// Boilerplate elements rarely contain content the user wants to
// scrape — strip them so `--dump text` returns the article body
// instead of menus, footers, and cookie banners.
if matches!(
tag,
"script" | "style" | "nav" | "header" | "footer" | "aside"
) {
return result;
}
@@ -872,3 +878,54 @@ mod tests {
assert!(is_quiet_command(&cmd));
}
}
#[cfg(test)]
mod tests {
use super::extract_readable_text;
use obscura_dom::parse_html;
fn body_text(html: &str) -> String {
let dom = parse_html(html);
let body = dom
.query_selector("body")
.ok()
.flatten()
.expect("body must exist");
extract_readable_text(&dom, body).split_whitespace().collect::<Vec<_>>().join(" ")
}
#[test]
fn skips_nav_header_footer_aside() {
let text = body_text(
r#"<html><body>
<header>SITE HEADER</header>
<nav>NAV LINKS</nav>
<aside>SIDEBAR</aside>
<main><p>Article body.</p></main>
<footer>FOOTER</footer>
</body></html>"#,
);
assert!(text.contains("Article body."), "main content kept: {text}");
for boilerplate in ["SITE HEADER", "NAV LINKS", "SIDEBAR", "FOOTER"] {
assert!(
!text.contains(boilerplate),
"boilerplate '{boilerplate}' leaked through: {text}"
);
}
}
#[test]
fn still_skips_script_and_style() {
// Regression guard for the original skip list.
let text = body_text(
r#"<html><body>
<p>Hello.</p>
<script>console.log("nope")</script>
<style>.x { color: red }</style>
</body></html>"#,
);
assert!(text.contains("Hello."));
assert!(!text.contains("console.log"));
assert!(!text.contains("color: red"));
}
}