Surfaces the existing LP.getMarkdown CDP machinery as a CLI flag so text-pipeline users (LLM agents, scrapers feeding markdown to other tools) don't need a separate HTML-to-markdown converter step. obscura fetch --dump markdown 'https://example.com' Layout: - crates/obscura-js/src/markdown.rs holds HTML_TO_MARKDOWN_JS as a pub const so every layer can share the same DOM-walker. obscura-js is at the bottom of the dep DAG (obscura-browser -> obscura-js, obscura-cdp -> obscura-js, obscura-cli -> obscura-js) so a single source of truth works without a circular dependency. - crates/obscura-browser/src/lib.rs re-exports it for ergonomics. - crates/obscura-cdp/src/domains/lp.rs drops its inline duplicate and calls the shared const. - crates/obscura-cli/src/main.rs adds the `Markdown` DumpFormat variant + dump_markdown(&mut Page) which page.evaluate's the shared script and prints the result. Net diff: a new 75-line markdown.rs, a 70+ line deletion in lp.rs that's reborn as 3 lines, and ~10 lines added across the other four files. No behavior change for `--dump html|text|links`, and LP.getMarkdown over CDP keeps the exact same JSON shape. Four new unit tests in obscura-js cover headings, inline emphasis + links, ordered/unordered lists, and that <script> / <style> contents do not leak into the output. Generated by Ora Studio Vibe coded by ousamabenyounes Co-authored-by: Ora Studio <[email protected]>
This commit is contained in:
1 parent
9fbd0c73a3
commit
d8b7c5fad1
6 files changed
+150
-66
No files matched your search
@@ -1466,6 +1466,72 @@ mod tests {
|
||||
assert_eq!(kind, serde_json::json!(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_html_to_markdown_headings() {
|
||||
let mut rt = setup_runtime("<html><body><h1>Title</h1><h2>Sub</h2><p>Body</p></body></html>");
|
||||
let md = rt
|
||||
.evaluate(crate::HTML_TO_MARKDOWN_JS)
|
||||
.unwrap()
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
assert!(md.contains("# Title"), "missing H1: {}", md);
|
||||
assert!(md.contains("## Sub"), "missing H2: {}", md);
|
||||
assert!(md.contains("Body"), "missing paragraph text: {}", md);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_html_to_markdown_links_and_inline() {
|
||||
let mut rt = setup_runtime(
|
||||
r#"<html><body><p>Hello <strong>world</strong> <a href="https://x.test/">link</a> <em>em</em></p></body></html>"#,
|
||||
);
|
||||
let md = rt
|
||||
.evaluate(crate::HTML_TO_MARKDOWN_JS)
|
||||
.unwrap()
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
assert!(md.contains("**world**"), "missing strong: {}", md);
|
||||
assert!(md.contains("*em*"), "missing em: {}", md);
|
||||
assert!(
|
||||
md.contains("[link](https://x.test/)"),
|
||||
"missing link: {}",
|
||||
md
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_html_to_markdown_lists() {
|
||||
let mut rt = setup_runtime(
|
||||
"<html><body><ul><li>A</li><li>B</li></ul><ol><li>X</li><li>Y</li></ol></body></html>",
|
||||
);
|
||||
let md = rt
|
||||
.evaluate(crate::HTML_TO_MARKDOWN_JS)
|
||||
.unwrap()
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
assert!(md.contains("- A"), "missing unordered A: {}", md);
|
||||
assert!(md.contains("- B"), "missing unordered B: {}", md);
|
||||
assert!(md.contains("1. X"), "missing ordered X: {}", md);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_html_to_markdown_skips_script_and_style() {
|
||||
let mut rt = setup_runtime(
|
||||
"<html><body><p>Text</p><script>alert(1)</script><style>body{color:red}</style></body></html>",
|
||||
);
|
||||
let md = rt
|
||||
.evaluate(crate::HTML_TO_MARKDOWN_JS)
|
||||
.unwrap()
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
assert!(md.contains("Text"), "missing visible text: {}", md);
|
||||
assert!(!md.contains("alert"), "leaked script content: {}", md);
|
||||
assert!(!md.contains("color:red"), "leaked style content: {}", md);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_page_content_puppeteer_pattern() {
|
||||
let mut rt = setup_runtime("<!DOCTYPE html><html><head></head><body><p>Test</p></body></html>");
|
||||
|
||||
Reference in new issue
Block a user