Surfaces the existing LP.getMarkdown CDP machinery as a CLI flag so text-pipeline users (LLM agents, scrapers feeding markdown to other tools) don't need a separate HTML-to-markdown converter step. obscura fetch --dump markdown 'https://example.com' Layout: - crates/obscura-js/src/markdown.rs holds HTML_TO_MARKDOWN_JS as a pub const so every layer can share the same DOM-walker. obscura-js is at the bottom of the dep DAG (obscura-browser -> obscura-js, obscura-cdp -> obscura-js, obscura-cli -> obscura-js) so a single source of truth works without a circular dependency. - crates/obscura-browser/src/lib.rs re-exports it for ergonomics. - crates/obscura-cdp/src/domains/lp.rs drops its inline duplicate and calls the shared const. - crates/obscura-cli/src/main.rs adds the `Markdown` DumpFormat variant + dump_markdown(&mut Page) which page.evaluate's the shared script and prints the result. Net diff: a new 75-line markdown.rs, a 70+ line deletion in lp.rs that's reborn as 3 lines, and ~10 lines added across the other four files. No behavior change for `--dump html|text|links`, and LP.getMarkdown over CDP keeps the exact same JSON shape. Four new unit tests in obscura-js cover headings, inline emphasis + links, ordered/unordered lists, and that <script> / <style> contents do not leak into the output. Generated by Ora Studio Vibe coded by ousamabenyounes Co-authored-by: Ora Studio <[email protected]>
This commit is contained in:
1 parent
9fbd0c73a3
commit
d8b7c5fad1
6 files changed
+150
-66
No files matched your search
@@ -1,3 +1,4 @@
|
||||
use obscura_browser::HTML_TO_MARKDOWN_JS;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
@@ -11,72 +12,7 @@ pub async fn handle(
|
||||
match method {
|
||||
"getMarkdown" => {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
let code = r#"
|
||||
(function() {
|
||||
function toMd(el, depth) {
|
||||
if (!el) return '';
|
||||
var out = '';
|
||||
if (el.nodeType === 3) return el.textContent || '';
|
||||
if (el.nodeType !== 1) return '';
|
||||
var tag = (el.tagName || '').toLowerCase();
|
||||
var children = '';
|
||||
var cn = el.childNodes || [];
|
||||
for (var i = 0; i < cn.length; i++) children += toMd(cn[i], depth);
|
||||
children = children.replace(/\n{3,}/g, '\n\n');
|
||||
switch(tag) {
|
||||
case 'h1': return '\n# ' + children.trim() + '\n\n';
|
||||
case 'h2': return '\n## ' + children.trim() + '\n\n';
|
||||
case 'h3': return '\n### ' + children.trim() + '\n\n';
|
||||
case 'h4': return '\n#### ' + children.trim() + '\n\n';
|
||||
case 'h5': return '\n##### ' + children.trim() + '\n\n';
|
||||
case 'h6': return '\n###### ' + children.trim() + '\n\n';
|
||||
case 'p': return '\n' + children.trim() + '\n\n';
|
||||
case 'br': return '\n';
|
||||
case 'hr': return '\n---\n\n';
|
||||
case 'strong': case 'b': return '**' + children + '**';
|
||||
case 'em': case 'i': return '*' + children + '*';
|
||||
case 'code': return '`' + children + '`';
|
||||
case 'pre': return '\n```\n' + children + '\n```\n\n';
|
||||
case 'blockquote': return '\n> ' + children.trim().replace(/\n/g, '\n> ') + '\n\n';
|
||||
case 'a':
|
||||
var href = el.getAttribute('href') || '';
|
||||
if (href && children.trim()) return '[' + children.trim() + '](' + href + ')';
|
||||
return children;
|
||||
case 'img':
|
||||
var src = el.getAttribute('src') || '';
|
||||
var alt = el.getAttribute('alt') || '';
|
||||
return '';
|
||||
case 'ul': case 'ol':
|
||||
return '\n' + children + '\n';
|
||||
case 'li':
|
||||
var parent = el.parentNode;
|
||||
var isOrdered = parent && parent.tagName && parent.tagName.toLowerCase() === 'ol';
|
||||
var bullet = isOrdered ? '1. ' : '- ';
|
||||
return bullet + children.trim() + '\n';
|
||||
case 'table': return '\n' + children + '\n';
|
||||
case 'thead': case 'tbody': case 'tfoot': return children;
|
||||
case 'tr':
|
||||
var cells = [];
|
||||
var tds = el.childNodes || [];
|
||||
for (var j = 0; j < tds.length; j++) {
|
||||
if (tds[j].nodeType === 1) cells.push(toMd(tds[j], depth).trim());
|
||||
}
|
||||
return '| ' + cells.join(' | ') + ' |\n';
|
||||
case 'th': case 'td': return children;
|
||||
case 'script': case 'style': case 'noscript': case 'link': case 'meta': return '';
|
||||
case 'div': case 'section': case 'article': case 'main': case 'aside': case 'nav': case 'header': case 'footer':
|
||||
return '\n' + children;
|
||||
case 'span': return children;
|
||||
default: return children;
|
||||
}
|
||||
}
|
||||
var body = document.body || document.documentElement;
|
||||
var md = toMd(body, 0);
|
||||
md = md.replace(/\n{3,}/g, '\n\n').trim();
|
||||
return md;
|
||||
})()
|
||||
"#;
|
||||
let result = page.evaluate(code);
|
||||
let result = page.evaluate(HTML_TO_MARKDOWN_JS);
|
||||
let markdown = result.as_str().unwrap_or("").to_string();
|
||||
Ok(json!({ "markdown": markdown }))
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user