diff --git a/crates/obscura-browser/src/lib.rs b/crates/obscura-browser/src/lib.rs index 1ece192..8439a9b 100644 --- a/crates/obscura-browser/src/lib.rs +++ b/crates/obscura-browser/src/lib.rs @@ -5,3 +5,4 @@ pub mod lifecycle; pub use page::{Page, PageError}; pub use context::BrowserContext; pub use lifecycle::{LifecycleState, WaitUntil}; +pub use obscura_js::HTML_TO_MARKDOWN_JS; diff --git a/crates/obscura-cdp/src/domains/lp.rs b/crates/obscura-cdp/src/domains/lp.rs index 518df67..5ee5be4 100644 --- a/crates/obscura-cdp/src/domains/lp.rs +++ b/crates/obscura-cdp/src/domains/lp.rs @@ -1,3 +1,4 @@ +use obscura_browser::HTML_TO_MARKDOWN_JS; use serde_json::{json, Value}; use crate::dispatch::CdpContext; @@ -11,72 +12,7 @@ pub async fn handle( match method { "getMarkdown" => { let page = ctx.get_session_page_mut(session_id).ok_or("No page")?; - let code = r#" -(function() { - function toMd(el, depth) { - if (!el) return ''; - var out = ''; - if (el.nodeType === 3) return el.textContent || ''; - if (el.nodeType !== 1) return ''; - var tag = (el.tagName || '').toLowerCase(); - var children = ''; - var cn = el.childNodes || []; - for (var i = 0; i < cn.length; i++) children += toMd(cn[i], depth); - children = children.replace(/\n{3,}/g, '\n\n'); - switch(tag) { - case 'h1': return '\n# ' + children.trim() + '\n\n'; - case 'h2': return '\n## ' + children.trim() + '\n\n'; - case 'h3': return '\n### ' + children.trim() + '\n\n'; - case 'h4': return '\n#### ' + children.trim() + '\n\n'; - case 'h5': return '\n##### ' + children.trim() + '\n\n'; - case 'h6': return '\n###### ' + children.trim() + '\n\n'; - case 'p': return '\n' + children.trim() + '\n\n'; - case 'br': return '\n'; - case 'hr': return '\n---\n\n'; - case 'strong': case 'b': return '**' + children + '**'; - case 'em': case 'i': return '*' + children + '*'; - case 'code': return '`' + children + '`'; - case 'pre': return '\n```\n' + children + '\n```\n\n'; - case 'blockquote': return '\n> ' + children.trim().replace(/\n/g, '\n> ') + '\n\n'; - case 'a': - var href = el.getAttribute('href') || ''; - if (href && children.trim()) return '[' + children.trim() + '](' + href + ')'; - return children; - case 'img': - var src = el.getAttribute('src') || ''; - var alt = el.getAttribute('alt') || ''; - return '![' + alt + '](' + src + ')'; - case 'ul': case 'ol': - return '\n' + children + '\n'; - case 'li': - var parent = el.parentNode; - var isOrdered = parent && parent.tagName && parent.tagName.toLowerCase() === 'ol'; - var bullet = isOrdered ? '1. ' : '- '; - return bullet + children.trim() + '\n'; - case 'table': return '\n' + children + '\n'; - case 'thead': case 'tbody': case 'tfoot': return children; - case 'tr': - var cells = []; - var tds = el.childNodes || []; - for (var j = 0; j < tds.length; j++) { - if (tds[j].nodeType === 1) cells.push(toMd(tds[j], depth).trim()); - } - return '| ' + cells.join(' | ') + ' |\n'; - case 'th': case 'td': return children; - case 'script': case 'style': case 'noscript': case 'link': case 'meta': return ''; - case 'div': case 'section': case 'article': case 'main': case 'aside': case 'nav': case 'header': case 'footer': - return '\n' + children; - case 'span': return children; - default: return children; - } - } - var body = document.body || document.documentElement; - var md = toMd(body, 0); - md = md.replace(/\n{3,}/g, '\n\n').trim(); - return md; -})() -"#; - let result = page.evaluate(code); + let result = page.evaluate(HTML_TO_MARKDOWN_JS); let markdown = result.as_str().unwrap_or("").to_string(); Ok(json!({ "markdown": markdown })) } diff --git a/crates/obscura-cli/src/main.rs b/crates/obscura-cli/src/main.rs index cf4e059..90628f2 100644 --- a/crates/obscura-cli/src/main.rs +++ b/crates/obscura-cli/src/main.rs @@ -109,6 +109,7 @@ enum DumpFormat { Html, Text, Links, + Markdown, } fn print_banner(port: u16) { @@ -395,6 +396,7 @@ async fn run_fetch( DumpFormat::Html => dump_html(&page), DumpFormat::Text => dump_text(&mut page), DumpFormat::Links => dump_links(&page), + DumpFormat::Markdown => dump_markdown(&mut page), }; write_or_print(rendered, output.as_ref()).await?; @@ -454,6 +456,11 @@ fn dump_text(page: &mut Page) -> String { }).unwrap_or_default() } +fn dump_markdown(page: &mut Page) -> String { + let result = page.evaluate(obscura_browser::HTML_TO_MARKDOWN_JS); + result.as_str().unwrap_or_default().to_string() +} + fn extract_readable_text(dom: &obscura_dom::DomTree, node_id: obscura_dom::NodeId) -> String { use obscura_dom::NodeData; diff --git a/crates/obscura-js/src/lib.rs b/crates/obscura-js/src/lib.rs index fe1d5cb..ddaebc6 100644 --- a/crates/obscura-js/src/lib.rs +++ b/crates/obscura-js/src/lib.rs @@ -5,3 +5,6 @@ pub mod module_loader; pub mod runtime; pub mod ops; pub mod v8_lock; +pub mod markdown; + +pub use markdown::HTML_TO_MARKDOWN_JS; diff --git a/crates/obscura-js/src/markdown.rs b/crates/obscura-js/src/markdown.rs new file mode 100644 index 0000000..7f00819 --- /dev/null +++ b/crates/obscura-js/src/markdown.rs @@ -0,0 +1,71 @@ +//! Shared markdown extraction script used by the LP.getMarkdown CDP method +//! and the CLI `--dump markdown` mode. Lives in obscura-browser so both +//! obscura-cdp and obscura-cli can call it without depending on each other. + +/// JS expression that walks `document.body` and returns a markdown string. +/// Must be evaluated against a Page that has a fully-bootstrapped JS runtime. +pub const HTML_TO_MARKDOWN_JS: &str = r#" +(function() { + function toMd(el, depth) { + if (!el) return ''; + var out = ''; + if (el.nodeType === 3) return el.textContent || ''; + if (el.nodeType !== 1) return ''; + var tag = (el.tagName || '').toLowerCase(); + var children = ''; + var cn = el.childNodes || []; + for (var i = 0; i < cn.length; i++) children += toMd(cn[i], depth); + children = children.replace(/\n{3,}/g, '\n\n'); + switch(tag) { + case 'h1': return '\n# ' + children.trim() + '\n\n'; + case 'h2': return '\n## ' + children.trim() + '\n\n'; + case 'h3': return '\n### ' + children.trim() + '\n\n'; + case 'h4': return '\n#### ' + children.trim() + '\n\n'; + case 'h5': return '\n##### ' + children.trim() + '\n\n'; + case 'h6': return '\n###### ' + children.trim() + '\n\n'; + case 'p': return '\n' + children.trim() + '\n\n'; + case 'br': return '\n'; + case 'hr': return '\n---\n\n'; + case 'strong': case 'b': return '**' + children + '**'; + case 'em': case 'i': return '*' + children + '*'; + case 'code': return '`' + children + '`'; + case 'pre': return '\n```\n' + children + '\n```\n\n'; + case 'blockquote': return '\n> ' + children.trim().replace(/\n/g, '\n> ') + '\n\n'; + case 'a': + var href = el.getAttribute('href') || ''; + if (href && children.trim()) return '[' + children.trim() + '](' + href + ')'; + return children; + case 'img': + var src = el.getAttribute('src') || ''; + var alt = el.getAttribute('alt') || ''; + return '![' + alt + '](' + src + ')'; + case 'ul': case 'ol': + return '\n' + children + '\n'; + case 'li': + var parent = el.parentNode; + var isOrdered = parent && parent.tagName && parent.tagName.toLowerCase() === 'ol'; + var bullet = isOrdered ? '1. ' : '- '; + return bullet + children.trim() + '\n'; + case 'table': return '\n' + children + '\n'; + case 'thead': case 'tbody': case 'tfoot': return children; + case 'tr': + var cells = []; + var tds = el.childNodes || []; + for (var j = 0; j < tds.length; j++) { + if (tds[j].nodeType === 1) cells.push(toMd(tds[j], depth).trim()); + } + return '| ' + cells.join(' | ') + ' |\n'; + case 'th': case 'td': return children; + case 'script': case 'style': case 'noscript': case 'link': case 'meta': return ''; + case 'div': case 'section': case 'article': case 'main': case 'aside': case 'nav': case 'header': case 'footer': + return '\n' + children; + case 'span': return children; + default: return children; + } + } + var body = document.body || document.documentElement; + var md = toMd(body, 0); + md = md.replace(/\n{3,}/g, '\n\n').trim(); + return md; +})() +"#; diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs index 904cd3e..46c0bbb 100644 --- a/crates/obscura-js/src/runtime.rs +++ b/crates/obscura-js/src/runtime.rs @@ -1466,6 +1466,72 @@ mod tests { assert_eq!(kind, serde_json::json!(true)); } + #[test] + fn test_html_to_markdown_headings() { + let mut rt = setup_runtime("

Title

Sub

Body

"); + let md = rt + .evaluate(crate::HTML_TO_MARKDOWN_JS) + .unwrap() + .as_str() + .unwrap() + .to_string(); + assert!(md.contains("# Title"), "missing H1: {}", md); + assert!(md.contains("## Sub"), "missing H2: {}", md); + assert!(md.contains("Body"), "missing paragraph text: {}", md); + } + + #[test] + fn test_html_to_markdown_links_and_inline() { + let mut rt = setup_runtime( + r#"

Hello world link em

"#, + ); + let md = rt + .evaluate(crate::HTML_TO_MARKDOWN_JS) + .unwrap() + .as_str() + .unwrap() + .to_string(); + assert!(md.contains("**world**"), "missing strong: {}", md); + assert!(md.contains("*em*"), "missing em: {}", md); + assert!( + md.contains("[link](https://x.test/)"), + "missing link: {}", + md + ); + } + + #[test] + fn test_html_to_markdown_lists() { + let mut rt = setup_runtime( + "
  1. X
  2. Y
", + ); + let md = rt + .evaluate(crate::HTML_TO_MARKDOWN_JS) + .unwrap() + .as_str() + .unwrap() + .to_string(); + assert!(md.contains("- A"), "missing unordered A: {}", md); + assert!(md.contains("- B"), "missing unordered B: {}", md); + assert!(md.contains("1. X"), "missing ordered X: {}", md); + } + + #[test] + fn test_html_to_markdown_skips_script_and_style() { + let mut rt = setup_runtime( + "

Text

", + ); + let md = rt + .evaluate(crate::HTML_TO_MARKDOWN_JS) + .unwrap() + .as_str() + .unwrap() + .to_string(); + assert!(md.contains("Text"), "missing visible text: {}", md); + assert!(!md.contains("alert"), "leaked script content: {}", md); + assert!(!md.contains("color:red"), "leaked style content: {}", md); + } + #[test] fn test_page_content_puppeteer_pattern() { let mut rt = setup_runtime("

Test

");