diff --git a/README.md b/README.md index 3f99715..caca1d3 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,9 @@ obscura scrape url1 url2 url3 ... \ --concurrency 25 \ --eval "document.querySelector('h1').textContent" \ --format json + +# Suppress scrape progress on stderr for script-friendly output +obscura scrape https://example.com --quiet --format json ``` ## Puppeteer / Playwright @@ -257,6 +260,7 @@ Scrape multiple URLs in parallel with worker processes. | `--concurrency` | `10` | Parallel workers | | `--eval` | — | JS expression per page | | `--format` | `json` | Output: `json` or `text` | +| `--quiet` | off | Suppress scrape progress on stderr | ## License diff --git a/crates/obscura-cli/src/main.rs b/crates/obscura-cli/src/main.rs index 357e4e8..2e0e07a 100644 --- a/crates/obscura-cli/src/main.rs +++ b/crates/obscura-cli/src/main.rs @@ -93,6 +93,9 @@ enum Command { #[arg(long, default_value_t = 60, value_parser = clap::value_parser!(u64).range(1..))] timeout: u64, + + #[arg(long, short)] + quiet: bool, }, } @@ -175,8 +178,8 @@ async fn main() -> anyhow::Result<()> { Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, quiet }) => { run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, quiet).await?; } - Some(Command::Scrape { urls, eval, concurrency, format, timeout }) => { - run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout).await?; + Some(Command::Scrape { urls, eval, concurrency, format, timeout, quiet }) => { + run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout, quiet).await?; } None => { print_banner(args.port); @@ -489,6 +492,7 @@ async fn run_parallel_scrape( concurrency: usize, format: &str, timeout_secs: u64, + quiet: bool, ) -> anyhow::Result<()> { let total = urls.len(); let start = Instant::now(); @@ -497,10 +501,12 @@ async fn run_parallel_scrape( anyhow::bail!("No URLs provided. Pass at least one URL to scrape."); } - eprintln!( - "Scraping {} URLs with {} concurrent workers (per-worker timeout: {}s)...", - total, concurrency, timeout_secs - ); + if !quiet { + eprintln!( + "Scraping {} URLs with {} concurrent workers (per-worker timeout: {}s)...", + total, concurrency, timeout_secs + ); + } let worker_path = std::env::current_exe() .ok() @@ -700,12 +706,14 @@ async fn run_parallel_scrape( println!("{}ms\t{}\t{}", time, url, eval); } } - eprintln!( - "\nTotal: {}ms for {} URLs ({} concurrent)", - total_time.as_millis(), - total, - concurrency - ); + if !quiet { + eprintln!( + "\nTotal: {}ms for {} URLs ({} concurrent)", + total_time.as_millis(), + total, + concurrency + ); + } } Ok(())