thebadking/obscura
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add quiet mode for scrape (#91)

Browse Source
This commit is contained in:
F0Rextasy authored and GitHub committed 2026-05-09 17:28:52 +02:00
1 parent 84115ca06e
commit e00729e7c6
2 files changed
+24 -12
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

README.md
+4
View File
@@ -116,6 +116,9 @@ obscura scrape url1 url2 url3 ... \
--concurrency 25 \ --concurrency 25 \
--eval "document.querySelector('h1').textContent" \ --eval "document.querySelector('h1').textContent" \
--format json --format json
# Suppress scrape progress on stderr for script-friendly output
obscura scrape https://example.com --quiet --format json
``` ```
## Puppeteer / Playwright ## Puppeteer / Playwright
@@ -257,6 +260,7 @@ Scrape multiple URLs in parallel with worker processes.
| `--concurrency` | `10` | Parallel workers | | `--concurrency` | `10` | Parallel workers |
| `--eval` | — | JS expression per page | | `--eval` | — | JS expression per page |
| `--format` | `json` | Output: `json` or `text` | | `--format` | `json` | Output: `json` or `text` |
| `--quiet` | off | Suppress scrape progress on stderr |
## License ## License
crates/obscura-cli/src/main.rs
+20 -12
View File
@@ -93,6 +93,9 @@ enum Command {
#[arg(long, default_value_t = 60, value_parser = clap::value_parser!(u64).range(1..))] #[arg(long, default_value_t = 60, value_parser = clap::value_parser!(u64).range(1..))]
timeout: u64, timeout: u64,
#[arg(long, short)]
quiet: bool,
}, },
} }
@@ -175,8 +178,8 @@ async fn main() -> anyhow::Result<()> {
Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, quiet }) => { Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, quiet }) => {
run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, quiet).await?; run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, quiet).await?;
} }
Some(Command::Scrape { urls, eval, concurrency, format, timeout }) => { Some(Command::Scrape { urls, eval, concurrency, format, timeout, quiet }) => {
run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout).await?; run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout, quiet).await?;
} }
None => { None => {
print_banner(args.port); print_banner(args.port);
@@ -489,6 +492,7 @@ async fn run_parallel_scrape(
concurrency: usize, concurrency: usize,
format: &str, format: &str,
timeout_secs: u64, timeout_secs: u64,
quiet: bool,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let total = urls.len(); let total = urls.len();
let start = Instant::now(); let start = Instant::now();
@@ -497,10 +501,12 @@ async fn run_parallel_scrape(
anyhow::bail!("No URLs provided. Pass at least one URL to scrape."); anyhow::bail!("No URLs provided. Pass at least one URL to scrape.");
} }
eprintln!( if !quiet {
"Scraping {} URLs with {} concurrent workers (per-worker timeout: {}s)...", eprintln!(
total, concurrency, timeout_secs "Scraping {} URLs with {} concurrent workers (per-worker timeout: {}s)...",
); total, concurrency, timeout_secs
);
}
let worker_path = std::env::current_exe() let worker_path = std::env::current_exe()
.ok() .ok()
@@ -700,12 +706,14 @@ async fn run_parallel_scrape(
println!("{}ms\t{}\t{}", time, url, eval); println!("{}ms\t{}\t{}", time, url, eval);
} }
} }
eprintln!( if !quiet {
"\nTotal: {}ms for {} URLs ({} concurrent)", eprintln!(
total_time.as_millis(), "\nTotal: {}ms for {} URLs ({} concurrent)",
total, total_time.as_millis(),
concurrency total,
); concurrency
);
}
} }
Ok(()) Ok(())