Add quiet mode for scrape (#91)

This commit is contained in:
F0Rextasy authored and GitHub committed 2026-05-09 17:28:52 +02:00
1 parent 84115ca06e
commit e00729e7c6
2 files changed
+14 -2

No files matched your search

+4
View File
@@ -116,6 +116,9 @@ obscura scrape url1 url2 url3 ... \
--concurrency 25 \ --concurrency 25 \
--eval "document.querySelector('h1').textContent" \ --eval "document.querySelector('h1').textContent" \
--format json --format json
# Suppress scrape progress on stderr for script-friendly output
obscura scrape https://example.com --quiet --format json
``` ```
## Puppeteer / Playwright ## Puppeteer / Playwright
@@ -257,6 +260,7 @@ Scrape multiple URLs in parallel with worker processes.
| `--concurrency` | `10` | Parallel workers | | `--concurrency` | `10` | Parallel workers |
| `--eval` | — | JS expression per page | | `--eval` | — | JS expression per page |
| `--format` | `json` | Output: `json` or `text` | | `--format` | `json` | Output: `json` or `text` |
| `--quiet` | off | Suppress scrape progress on stderr |
## License ## License
+10 -2
View File
@@ -93,6 +93,9 @@ enum Command {
#[arg(long, default_value_t = 60, value_parser = clap::value_parser!(u64).range(1..))] #[arg(long, default_value_t = 60, value_parser = clap::value_parser!(u64).range(1..))]
timeout: u64, timeout: u64,
#[arg(long, short)]
quiet: bool,
}, },
} }
@@ -175,8 +178,8 @@ async fn main() -> anyhow::Result<()> {
Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, quiet }) => { Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, quiet }) => {
run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, quiet).await?; run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, quiet).await?;
} }
Some(Command::Scrape { urls, eval, concurrency, format, timeout }) => { Some(Command::Scrape { urls, eval, concurrency, format, timeout, quiet }) => {
run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout).await?; run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout, quiet).await?;
} }
None => { None => {
print_banner(args.port); print_banner(args.port);
@@ -489,6 +492,7 @@ async fn run_parallel_scrape(
concurrency: usize, concurrency: usize,
format: &str, format: &str,
timeout_secs: u64, timeout_secs: u64,
quiet: bool,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let total = urls.len(); let total = urls.len();
let start = Instant::now(); let start = Instant::now();
@@ -497,10 +501,12 @@ async fn run_parallel_scrape(
anyhow::bail!("No URLs provided. Pass at least one URL to scrape."); anyhow::bail!("No URLs provided. Pass at least one URL to scrape.");
} }
if !quiet {
eprintln!( eprintln!(
"Scraping {} URLs with {} concurrent workers (per-worker timeout: {}s)...", "Scraping {} URLs with {} concurrent workers (per-worker timeout: {}s)...",
total, concurrency, timeout_secs total, concurrency, timeout_secs
); );
}
let worker_path = std::env::current_exe() let worker_path = std::env::current_exe()
.ok() .ok()
@@ -700,6 +706,7 @@ async fn run_parallel_scrape(
println!("{}ms\t{}\t{}", time, url, eval); println!("{}ms\t{}\t{}", time, url, eval);
} }
} }
if !quiet {
eprintln!( eprintln!(
"\nTotal: {}ms for {} URLs ({} concurrent)", "\nTotal: {}ms for {} URLs ({} concurrent)",
total_time.as_millis(), total_time.as_millis(),
@@ -707,6 +714,7 @@ async fn run_parallel_scrape(
concurrency concurrency
); );
} }
}
Ok(()) Ok(())
} }