From 81b9cf0f42b007ac521f3f395e29da9cc8abd4c5 Mon Sep 17 00:00:00 2001 From: F0Rextasy <99847357+F0Rextasy@users.noreply.github.com> Date: Sat, 9 May 2026 18:30:24 +0300 Subject: [PATCH] Add fetch output file option (#95) --- README.md | 4 ++ crates/obscura-cli/src/main.rs | 100 +++++++++++++++++++++++---------- 2 files changed, 73 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index caca1d3..28256e6 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,9 @@ obscura fetch https://example.com --dump links # Render JavaScript and dump HTML obscura fetch https://news.ycombinator.com --dump html +# Write dump or eval output to a file +obscura fetch https://example.com --dump text --output page.txt + # Wait for dynamic content obscura fetch https://example.com --wait-until networkidle0 @@ -249,6 +252,7 @@ Fetch and render a single page. | `--timeout` | `30` | Maximum navigation time in seconds | | `--selector` | — | Wait for CSS selector | | `--stealth` | off | Anti-detection mode | +| `--output` | — | Write dump or eval output to a file | | `--quiet` | off | Suppress banner | ### `obscura scrape ` diff --git a/crates/obscura-cli/src/main.rs b/crates/obscura-cli/src/main.rs index 2e0e07a..768032e 100644 --- a/crates/obscura-cli/src/main.rs +++ b/crates/obscura-cli/src/main.rs @@ -75,6 +75,9 @@ enum Command { #[arg(long, short)] eval: Option, + #[arg(long, short = 'o')] + output: Option, + #[arg(long, short)] quiet: bool, }, @@ -175,8 +178,8 @@ async fn main() -> anyhow::Result<()> { obscura_cdp::start_with_full_options(port, proxy, stealth, user_agent).await?; } } - Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, quiet }) => { - run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, quiet).await?; + Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, output, quiet }) => { + run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, output, quiet).await?; } Some(Command::Scrape { urls, eval, concurrency, format, timeout, quiet }) => { run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout, quiet).await?; @@ -330,6 +333,7 @@ async fn run_fetch( user_agent: Option, stealth: bool, eval: Option, + output: Option, quiet: bool, ) -> anyhow::Result<()> { let context = Arc::new(BrowserContext::with_options("fetch".to_string(), None, stealth)); @@ -367,29 +371,36 @@ async fn run_fetch( if let Some(ref expr) = eval { let result = page.evaluate(expr); - match result { - serde_json::Value::String(s) => println!("{}", s), - serde_json::Value::Null => println!("null"), - other => println!("{}", other), - } + let rendered = match result { + serde_json::Value::String(s) => s, + serde_json::Value::Null => "null".to_string(), + other => other.to_string(), + }; + write_or_print(rendered, output.as_ref()).await?; return Ok(()); } - match dump { - DumpFormat::Html => { - dump_html(&page); - } - DumpFormat::Text => { - dump_text(&mut page); - } - DumpFormat::Links => { - dump_links(&page); - } - } + let rendered = match dump { + DumpFormat::Html => dump_html(&page), + DumpFormat::Text => dump_text(&mut page), + DumpFormat::Links => dump_links(&page), + }; + write_or_print(rendered, output.as_ref()).await?; Ok(()) } +async fn write_or_print(content: String, output: Option<&std::path::PathBuf>) -> anyhow::Result<()> { + if let Some(path) = output { + tokio::fs::write(path, content) + .await + .map_err(|e| anyhow::anyhow!("Failed to write {}: {}", path.display(), e))?; + } else { + println!("{}", content); + } + Ok(()) +} + async fn wait_for_selector(page: &mut Page, selector: &str, timeout_secs: u64) -> bool { let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_secs(timeout_secs); loop { @@ -409,27 +420,27 @@ async fn wait_for_selector(page: &mut Page, selector: &str, timeout_secs: u64) - } } -fn dump_html(page: &Page) { +fn dump_html(page: &Page) -> String { page.with_dom(|dom| { if let Ok(Some(html_node)) = dom.query_selector("html") { let html = dom.outer_html(html_node); - println!(""); - println!("{}", html); + format!("\n{}", html) } else { let doc = dom.document(); - let html = dom.inner_html(doc); - println!("{}", html); + dom.inner_html(doc) } - }); + }).unwrap_or_default() } -fn dump_text(page: &mut Page) { +fn dump_text(page: &mut Page) -> String { page.with_dom(|dom| { if let Ok(Some(body)) = dom.query_selector("body") { let text = extract_readable_text(dom, body); - println!("{}", text.trim()); + text.trim().to_string() + } else { + String::new() } - }); + }).unwrap_or_default() } fn extract_readable_text(dom: &obscura_dom::DomTree, node_id: obscura_dom::NodeId) -> String { @@ -719,9 +730,10 @@ async fn run_parallel_scrape( Ok(()) } -fn dump_links(page: &Page) { +fn dump_links(page: &Page) -> String { let base_url = page.url.clone(); page.with_dom(|dom| { + let mut rendered = Vec::new(); let links = dom.query_selector_all("a").unwrap_or_default(); for link_id in links { if let Some(node) = dom.get_node(link_id) { @@ -739,14 +751,40 @@ fn dump_links(page: &Page) { if !full_url.is_empty() { if text.is_empty() { - println!("{}", full_url); + rendered.push(full_url); } else { - println!("{}\t{}", full_url, text); + rendered.push(format!("{}\t{}", full_url, text)); } } } } - }); + rendered.join("\n") + }).unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::write_or_print; + + #[tokio::test(flavor = "current_thread")] + async fn write_or_print_writes_output_file_with_tokio_fs() { + let path = std::env::temp_dir().join(format!( + "obscura-fetch-output-test-{}.txt", + std::process::id() + )); + let _ = tokio::fs::remove_file(&path).await; + + write_or_print("rendered output".to_string(), Some(&path)) + .await + .expect("write output file"); + + let content = tokio::fs::read_to_string(&path) + .await + .expect("read output file"); + let _ = tokio::fs::remove_file(&path).await; + + assert_eq!(content, "rendered output"); + } } #[cfg(test)] mod tests {