Add quiet mode for scrape (#91)
This commit is contained in:
1 parent
84115ca06e
commit
e00729e7c6
2 files changed
+24
-12
No files matched your search
@@ -116,6 +116,9 @@ obscura scrape url1 url2 url3 ... \
|
||||
--concurrency 25 \
|
||||
--eval "document.querySelector('h1').textContent" \
|
||||
--format json
|
||||
|
||||
# Suppress scrape progress on stderr for script-friendly output
|
||||
obscura scrape https://example.com --quiet --format json
|
||||
```
|
||||
|
||||
## Puppeteer / Playwright
|
||||
@@ -257,6 +260,7 @@ Scrape multiple URLs in parallel with worker processes.
|
||||
| `--concurrency` | `10` | Parallel workers |
|
||||
| `--eval` | — | JS expression per page |
|
||||
| `--format` | `json` | Output: `json` or `text` |
|
||||
| `--quiet` | off | Suppress scrape progress on stderr |
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -93,6 +93,9 @@ enum Command {
|
||||
|
||||
#[arg(long, default_value_t = 60, value_parser = clap::value_parser!(u64).range(1..))]
|
||||
timeout: u64,
|
||||
|
||||
#[arg(long, short)]
|
||||
quiet: bool,
|
||||
},
|
||||
|
||||
}
|
||||
@@ -175,8 +178,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, quiet }) => {
|
||||
run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, quiet).await?;
|
||||
}
|
||||
Some(Command::Scrape { urls, eval, concurrency, format, timeout }) => {
|
||||
run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout).await?;
|
||||
Some(Command::Scrape { urls, eval, concurrency, format, timeout, quiet }) => {
|
||||
run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout, quiet).await?;
|
||||
}
|
||||
None => {
|
||||
print_banner(args.port);
|
||||
@@ -489,6 +492,7 @@ async fn run_parallel_scrape(
|
||||
concurrency: usize,
|
||||
format: &str,
|
||||
timeout_secs: u64,
|
||||
quiet: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
let total = urls.len();
|
||||
let start = Instant::now();
|
||||
@@ -497,10 +501,12 @@ async fn run_parallel_scrape(
|
||||
anyhow::bail!("No URLs provided. Pass at least one URL to scrape.");
|
||||
}
|
||||
|
||||
eprintln!(
|
||||
"Scraping {} URLs with {} concurrent workers (per-worker timeout: {}s)...",
|
||||
total, concurrency, timeout_secs
|
||||
);
|
||||
if !quiet {
|
||||
eprintln!(
|
||||
"Scraping {} URLs with {} concurrent workers (per-worker timeout: {}s)...",
|
||||
total, concurrency, timeout_secs
|
||||
);
|
||||
}
|
||||
|
||||
let worker_path = std::env::current_exe()
|
||||
.ok()
|
||||
@@ -700,12 +706,14 @@ async fn run_parallel_scrape(
|
||||
println!("{}ms\t{}\t{}", time, url, eval);
|
||||
}
|
||||
}
|
||||
eprintln!(
|
||||
"\nTotal: {}ms for {} URLs ({} concurrent)",
|
||||
total_time.as_millis(),
|
||||
total,
|
||||
concurrency
|
||||
);
|
||||
if !quiet {
|
||||
eprintln!(
|
||||
"\nTotal: {}ms for {} URLs ({} concurrent)",
|
||||
total_time.as_millis(),
|
||||
total,
|
||||
concurrency
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
Reference in new issue
Block a user