From d8b7c5fad16486b12a0ab995526f60c35ae7cb48 Mon Sep 17 00:00:00 2001
From: Ben Younes <benyounes.ousama@gmail.com>
Date: Wed, 13 May 2026 22:36:17 +0200
Subject: [PATCH] feat(cli): add --dump markdown output mode (#54) (#78)

Surfaces the existing LP.getMarkdown CDP machinery as a CLI flag so
text-pipeline users (LLM agents, scrapers feeding markdown to other
tools) don't need a separate HTML-to-markdown converter step.

  obscura fetch --dump markdown 'https://example.com'

Layout:
- crates/obscura-js/src/markdown.rs holds HTML_TO_MARKDOWN_JS as a
  pub const so every layer can share the same DOM-walker. obscura-js
  is at the bottom of the dep DAG (obscura-browser -> obscura-js,
  obscura-cdp -> obscura-js, obscura-cli -> obscura-js) so a single
  source of truth works without a circular dependency.
- crates/obscura-browser/src/lib.rs re-exports it for ergonomics.
- crates/obscura-cdp/src/domains/lp.rs drops its inline duplicate
  and calls the shared const.
- crates/obscura-cli/src/main.rs adds the `Markdown` DumpFormat
  variant + dump_markdown(&mut Page) which page.evaluate's the
  shared script and prints the result.

Net diff: a new 75-line markdown.rs, a 70+ line deletion in lp.rs
that's reborn as 3 lines, and ~10 lines added across the other four
files. No behavior change for `--dump html|text|links`, and
LP.getMarkdown over CDP keeps the exact same JSON shape.

Four new unit tests in obscura-js cover headings, inline emphasis +
links, ordered/unordered lists, and that <script> / <style> contents
do not leak into the output.

Generated by Ora Studio
Vibe coded by ousamabenyounes

Co-authored-by: Ora Studio <noreply@oratelecom.net>
---
 crates/obscura-browser/src/lib.rs    |  1 +
 crates/obscura-cdp/src/domains/lp.rs | 68 +-------------------------
 crates/obscura-cli/src/main.rs       |  7 +++
 crates/obscura-js/src/lib.rs         |  3 ++
 crates/obscura-js/src/markdown.rs    | 71 ++++++++++++++++++++++++++++
 crates/obscura-js/src/runtime.rs     | 66 ++++++++++++++++++++++++++
 6 files changed, 150 insertions(+), 66 deletions(-)
 create mode 100644 crates/obscura-js/src/markdown.rs

diff --git a/crates/obscura-browser/src/lib.rs b/crates/obscura-browser/src/lib.rs
index 1ece192..8439a9b 100644
--- a/crates/obscura-browser/src/lib.rs
+++ b/crates/obscura-browser/src/lib.rs
@@ -5,3 +5,4 @@ pub mod lifecycle;
 pub use page::{Page, PageError};
 pub use context::BrowserContext;
 pub use lifecycle::{LifecycleState, WaitUntil};
+pub use obscura_js::HTML_TO_MARKDOWN_JS;
diff --git a/crates/obscura-cdp/src/domains/lp.rs b/crates/obscura-cdp/src/domains/lp.rs
index 518df67..5ee5be4 100644
--- a/crates/obscura-cdp/src/domains/lp.rs
+++ b/crates/obscura-cdp/src/domains/lp.rs
@@ -1,3 +1,4 @@
+use obscura_browser::HTML_TO_MARKDOWN_JS;
 use serde_json::{json, Value};
 
 use crate::dispatch::CdpContext;
@@ -11,72 +12,7 @@ pub async fn handle(
     match method {
         "getMarkdown" => {
             let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
-            let code = r#"
-(function() {
-    function toMd(el, depth) {
-        if (!el) return '';
-        var out = '';
-        if (el.nodeType === 3) return el.textContent || '';
-        if (el.nodeType !== 1) return '';
-        var tag = (el.tagName || '').toLowerCase();
-        var children = '';
-        var cn = el.childNodes || [];
-        for (var i = 0; i < cn.length; i++) children += toMd(cn[i], depth);
-        children = children.replace(/\n{3,}/g, '\n\n');
-        switch(tag) {
-            case 'h1': return '\n# ' + children.trim() + '\n\n';
-            case 'h2': return '\n## ' + children.trim() + '\n\n';
-            case 'h3': return '\n### ' + children.trim() + '\n\n';
-            case 'h4': return '\n#### ' + children.trim() + '\n\n';
-            case 'h5': return '\n##### ' + children.trim() + '\n\n';
-            case 'h6': return '\n###### ' + children.trim() + '\n\n';
-            case 'p': return '\n' + children.trim() + '\n\n';
-            case 'br': return '\n';
-            case 'hr': return '\n---\n\n';
-            case 'strong': case 'b': return '**' + children + '**';
-            case 'em': case 'i': return '*' + children + '*';
-            case 'code': return '`' + children + '`';
-            case 'pre': return '\n```\n' + children + '\n```\n\n';
-            case 'blockquote': return '\n> ' + children.trim().replace(/\n/g, '\n> ') + '\n\n';
-            case 'a':
-                var href = el.getAttribute('href') || '';
-                if (href && children.trim()) return '[' + children.trim() + '](' + href + ')';
-                return children;
-            case 'img':
-                var src = el.getAttribute('src') || '';
-                var alt = el.getAttribute('alt') || '';
-                return '![' + alt + '](' + src + ')';
-            case 'ul': case 'ol':
-                return '\n' + children + '\n';
-            case 'li':
-                var parent = el.parentNode;
-                var isOrdered = parent && parent.tagName && parent.tagName.toLowerCase() === 'ol';
-                var bullet = isOrdered ? '1. ' : '- ';
-                return bullet + children.trim() + '\n';
-            case 'table': return '\n' + children + '\n';
-            case 'thead': case 'tbody': case 'tfoot': return children;
-            case 'tr':
-                var cells = [];
-                var tds = el.childNodes || [];
-                for (var j = 0; j < tds.length; j++) {
-                    if (tds[j].nodeType === 1) cells.push(toMd(tds[j], depth).trim());
-                }
-                return '| ' + cells.join(' | ') + ' |\n';
-            case 'th': case 'td': return children;
-            case 'script': case 'style': case 'noscript': case 'link': case 'meta': return '';
-            case 'div': case 'section': case 'article': case 'main': case 'aside': case 'nav': case 'header': case 'footer':
-                return '\n' + children;
-            case 'span': return children;
-            default: return children;
-        }
-    }
-    var body = document.body || document.documentElement;
-    var md = toMd(body, 0);
-    md = md.replace(/\n{3,}/g, '\n\n').trim();
-    return md;
-})()
-"#;
-            let result = page.evaluate(code);
+            let result = page.evaluate(HTML_TO_MARKDOWN_JS);
             let markdown = result.as_str().unwrap_or("").to_string();
             Ok(json!({ "markdown": markdown }))
         }
diff --git a/crates/obscura-cli/src/main.rs b/crates/obscura-cli/src/main.rs
index cf4e059..90628f2 100644
--- a/crates/obscura-cli/src/main.rs
+++ b/crates/obscura-cli/src/main.rs
@@ -109,6 +109,7 @@ enum DumpFormat {
     Html,
     Text,
     Links,
+    Markdown,
 }
 
 fn print_banner(port: u16) {
@@ -395,6 +396,7 @@ async fn run_fetch(
         DumpFormat::Html => dump_html(&page),
         DumpFormat::Text => dump_text(&mut page),
         DumpFormat::Links => dump_links(&page),
+        DumpFormat::Markdown => dump_markdown(&mut page),
     };
     write_or_print(rendered, output.as_ref()).await?;
 
@@ -454,6 +456,11 @@ fn dump_text(page: &mut Page) -> String {
     }).unwrap_or_default()
 }
 
+fn dump_markdown(page: &mut Page) -> String {
+    let result = page.evaluate(obscura_browser::HTML_TO_MARKDOWN_JS);
+    result.as_str().unwrap_or_default().to_string()
+}
+
 fn extract_readable_text(dom: &obscura_dom::DomTree, node_id: obscura_dom::NodeId) -> String {
     use obscura_dom::NodeData;
 
diff --git a/crates/obscura-js/src/lib.rs b/crates/obscura-js/src/lib.rs
index fe1d5cb..ddaebc6 100644
--- a/crates/obscura-js/src/lib.rs
+++ b/crates/obscura-js/src/lib.rs
@@ -5,3 +5,6 @@ pub mod module_loader;
 pub mod runtime;
 pub mod ops;
 pub mod v8_lock;
+pub mod markdown;
+
+pub use markdown::HTML_TO_MARKDOWN_JS;
diff --git a/crates/obscura-js/src/markdown.rs b/crates/obscura-js/src/markdown.rs
new file mode 100644
index 0000000..7f00819
--- /dev/null
+++ b/crates/obscura-js/src/markdown.rs
@@ -0,0 +1,71 @@
+//! Shared markdown extraction script used by the LP.getMarkdown CDP method
+//! and the CLI `--dump markdown` mode. Lives in obscura-browser so both
+//! obscura-cdp and obscura-cli can call it without depending on each other.
+
+/// JS expression that walks `document.body` and returns a markdown string.
+/// Must be evaluated against a Page that has a fully-bootstrapped JS runtime.
+pub const HTML_TO_MARKDOWN_JS: &str = r#"
+(function() {
+    function toMd(el, depth) {
+        if (!el) return '';
+        var out = '';
+        if (el.nodeType === 3) return el.textContent || '';
+        if (el.nodeType !== 1) return '';
+        var tag = (el.tagName || '').toLowerCase();
+        var children = '';
+        var cn = el.childNodes || [];
+        for (var i = 0; i < cn.length; i++) children += toMd(cn[i], depth);
+        children = children.replace(/\n{3,}/g, '\n\n');
+        switch(tag) {
+            case 'h1': return '\n# ' + children.trim() + '\n\n';
+            case 'h2': return '\n## ' + children.trim() + '\n\n';
+            case 'h3': return '\n### ' + children.trim() + '\n\n';
+            case 'h4': return '\n#### ' + children.trim() + '\n\n';
+            case 'h5': return '\n##### ' + children.trim() + '\n\n';
+            case 'h6': return '\n###### ' + children.trim() + '\n\n';
+            case 'p': return '\n' + children.trim() + '\n\n';
+            case 'br': return '\n';
+            case 'hr': return '\n---\n\n';
+            case 'strong': case 'b': return '**' + children + '**';
+            case 'em': case 'i': return '*' + children + '*';
+            case 'code': return '`' + children + '`';
+            case 'pre': return '\n```\n' + children + '\n```\n\n';
+            case 'blockquote': return '\n> ' + children.trim().replace(/\n/g, '\n> ') + '\n\n';
+            case 'a':
+                var href = el.getAttribute('href') || '';
+                if (href && children.trim()) return '[' + children.trim() + '](' + href + ')';
+                return children;
+            case 'img':
+                var src = el.getAttribute('src') || '';
+                var alt = el.getAttribute('alt') || '';
+                return '![' + alt + '](' + src + ')';
+            case 'ul': case 'ol':
+                return '\n' + children + '\n';
+            case 'li':
+                var parent = el.parentNode;
+                var isOrdered = parent && parent.tagName && parent.tagName.toLowerCase() === 'ol';
+                var bullet = isOrdered ? '1. ' : '- ';
+                return bullet + children.trim() + '\n';
+            case 'table': return '\n' + children + '\n';
+            case 'thead': case 'tbody': case 'tfoot': return children;
+            case 'tr':
+                var cells = [];
+                var tds = el.childNodes || [];
+                for (var j = 0; j < tds.length; j++) {
+                    if (tds[j].nodeType === 1) cells.push(toMd(tds[j], depth).trim());
+                }
+                return '| ' + cells.join(' | ') + ' |\n';
+            case 'th': case 'td': return children;
+            case 'script': case 'style': case 'noscript': case 'link': case 'meta': return '';
+            case 'div': case 'section': case 'article': case 'main': case 'aside': case 'nav': case 'header': case 'footer':
+                return '\n' + children;
+            case 'span': return children;
+            default: return children;
+        }
+    }
+    var body = document.body || document.documentElement;
+    var md = toMd(body, 0);
+    md = md.replace(/\n{3,}/g, '\n\n').trim();
+    return md;
+})()
+"#;
diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs
index 904cd3e..46c0bbb 100644
--- a/crates/obscura-js/src/runtime.rs
+++ b/crates/obscura-js/src/runtime.rs
@@ -1466,6 +1466,72 @@ mod tests {
         assert_eq!(kind, serde_json::json!(true));
     }
 
+    #[test]
+    fn test_html_to_markdown_headings() {
+        let mut rt = setup_runtime("<html><body><h1>Title</h1><h2>Sub</h2><p>Body</p></body></html>");
+        let md = rt
+            .evaluate(crate::HTML_TO_MARKDOWN_JS)
+            .unwrap()
+            .as_str()
+            .unwrap()
+            .to_string();
+        assert!(md.contains("# Title"), "missing H1: {}", md);
+        assert!(md.contains("## Sub"), "missing H2: {}", md);
+        assert!(md.contains("Body"), "missing paragraph text: {}", md);
+    }
+
+    #[test]
+    fn test_html_to_markdown_links_and_inline() {
+        let mut rt = setup_runtime(
+            r#"<html><body><p>Hello <strong>world</strong> <a href="https://x.test/">link</a> <em>em</em></p></body></html>"#,
+        );
+        let md = rt
+            .evaluate(crate::HTML_TO_MARKDOWN_JS)
+            .unwrap()
+            .as_str()
+            .unwrap()
+            .to_string();
+        assert!(md.contains("**world**"), "missing strong: {}", md);
+        assert!(md.contains("*em*"), "missing em: {}", md);
+        assert!(
+            md.contains("[link](https://x.test/)"),
+            "missing link: {}",
+            md
+        );
+    }
+
+    #[test]
+    fn test_html_to_markdown_lists() {
+        let mut rt = setup_runtime(
+            "<html><body><ul><li>A</li><li>B</li></ul><ol><li>X</li><li>Y</li></ol></body></html>",
+        );
+        let md = rt
+            .evaluate(crate::HTML_TO_MARKDOWN_JS)
+            .unwrap()
+            .as_str()
+            .unwrap()
+            .to_string();
+        assert!(md.contains("- A"), "missing unordered A: {}", md);
+        assert!(md.contains("- B"), "missing unordered B: {}", md);
+        assert!(md.contains("1. X"), "missing ordered X: {}", md);
+    }
+
+    #[test]
+    fn test_html_to_markdown_skips_script_and_style() {
+        let mut rt = setup_runtime(
+            "<html><body><p>Text</p><script>alert(1)</script><style>body{color:red}</style></body></html>",
+        );
+        let md = rt
+            .evaluate(crate::HTML_TO_MARKDOWN_JS)
+            .unwrap()
+            .as_str()
+            .unwrap()
+            .to_string();
+        assert!(md.contains("Text"), "missing visible text: {}", md);
+        assert!(!md.contains("alert"), "leaked script content: {}", md);
+        assert!(!md.contains("color:red"), "leaked style content: {}", md);
+    }
+
     #[test]
     fn test_page_content_puppeteer_pattern() {
         let mut rt = setup_runtime("<!DOCTYPE html><html><head></head><body><p>Test</p></body></html>");