From 7dfc2a9c01cc89c7b82ee31c1202fbbd83dd7397 Mon Sep 17 00:00:00 2001
From: Ben Younes <benyounes.ousama@gmail.com>
Date: Sat, 9 May 2026 17:32:59 +0200
Subject: [PATCH] fix(cli): strip nav/header/footer/aside from --dump text
 output (#55) (#82)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`obscura fetch --dump text` walked the entire <body> and emitted the
text of navigation menus, sidebars, and footers alongside the article
body. The skip list inside extract_readable_text already excluded
<script>/<style>; extend it with the four standard semantic-boilerplate
tags so the dump is closer to the article content users actually want
to scrape.

Behaviour change for users who relied on the noise being included is
acknowledged but unlikely — these tags are the canonical wrappers
boilerplate goes in. Anyone needing the full body should use --dump
html and post-process.

Generated by Ora Studio
Vibe coded by ousamabenyounes

Co-authored-by: Ora Studio <noreply@oratelecom.net>
---
 crates/obscura-cli/src/main.rs | 59 +++++++++++++++++++++++++++++++++-
 1 file changed, 58 insertions(+), 1 deletion(-)

diff --git a/crates/obscura-cli/src/main.rs b/crates/obscura-cli/src/main.rs
index 768032e..bacb760 100644
--- a/crates/obscura-cli/src/main.rs
+++ b/crates/obscura-cli/src/main.rs
@@ -471,7 +471,13 @@ fn extract_readable_text(dom: &obscura_dom::DomTree, node_id: obscura_dom::NodeI
                     | "ul" | "ol"
             );
 
-            if tag == "script" || tag == "style" {
+            // Boilerplate elements rarely contain content the user wants to
+            // scrape — strip them so `--dump text` returns the article body
+            // instead of menus, footers, and cookie banners.
+            if matches!(
+                tag,
+                "script" | "style" | "nav" | "header" | "footer" | "aside"
+            ) {
                 return result;
             }
 
@@ -872,3 +878,54 @@ mod tests {
         assert!(is_quiet_command(&cmd));
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::extract_readable_text;
+    use obscura_dom::parse_html;
+
+    fn body_text(html: &str) -> String {
+        let dom = parse_html(html);
+        let body = dom
+            .query_selector("body")
+            .ok()
+            .flatten()
+            .expect("body must exist");
+        extract_readable_text(&dom, body).split_whitespace().collect::<Vec<_>>().join(" ")
+    }
+
+    #[test]
+    fn skips_nav_header_footer_aside() {
+        let text = body_text(
+            r#"<html><body>
+                <header>SITE HEADER</header>
+                <nav>NAV LINKS</nav>
+                <aside>SIDEBAR</aside>
+                <main><p>Article body.</p></main>
+                <footer>FOOTER</footer>
+            </body></html>"#,
+        );
+        assert!(text.contains("Article body."), "main content kept: {text}");
+        for boilerplate in ["SITE HEADER", "NAV LINKS", "SIDEBAR", "FOOTER"] {
+            assert!(
+                !text.contains(boilerplate),
+                "boilerplate '{boilerplate}' leaked through: {text}"
+            );
+        }
+    }
+
+    #[test]
+    fn still_skips_script_and_style() {
+        // Regression guard for the original skip list.
+        let text = body_text(
+            r#"<html><body>
+                <p>Hello.</p>
+                <script>console.log("nope")</script>
+                <style>.x { color: red }</style>
+            </body></html>"#,
+        );
+        assert!(text.contains("Hello."));
+        assert!(!text.contains("console.log"));
+        assert!(!text.contains("color: red"));
+    }
+}