From 9fbd0c73a32d1144acf4652dc0da67ba6d8991c6 Mon Sep 17 00:00:00 2001 From: F0Rextasy <99847357+F0Rextasy@users.noreply.github.com> Date: Wed, 13 May 2026 23:34:02 +0300 Subject: [PATCH] Plumb proxy into fetch and scrape (#88) --- README.md | 8 ++++++ crates/obscura-cli/src/main.rs | 44 +++++++++++++++++++++++++++----- crates/obscura-cli/src/worker.rs | 6 ++++- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6614d9f..30819ef 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,9 @@ obscura fetch https://news.ycombinator.com --dump html # Write dump or eval output to a file obscura fetch https://example.com --dump text --output page.txt +# Fetch through an HTTP or SOCKS proxy +obscura --proxy socks5://127.0.0.1:1080 fetch https://example.com --dump text + # Wait for dynamic content obscura fetch https://example.com --wait-until networkidle0 @@ -125,6 +128,9 @@ obscura scrape url1 url2 url3 ... \ # Suppress scrape progress on stderr for script-friendly output obscura scrape https://example.com --quiet --format json + +# Scrape workers inherit the global proxy +obscura --proxy http://127.0.0.1:8080 scrape https://example.com https://news.ycombinator.com ``` ## Puppeteer / Playwright @@ -257,6 +263,7 @@ Fetch and render a single page. | `--stealth` | off | Anti-detection mode | | `--output` | — | Write dump or eval output to a file | | `--quiet` | off | Suppress banner | +| `--proxy` | — | Inherited global HTTP/SOCKS5 proxy URL | ### `obscura scrape ` @@ -268,6 +275,7 @@ Scrape multiple URLs in parallel with worker processes. | `--eval` | — | JS expression per page | | `--format` | `json` | Output: `json` or `text` | | `--quiet` | off | Suppress scrape progress on stderr | +| `--proxy` | — | Inherited global HTTP/SOCKS5 proxy URL for all workers | ## License diff --git a/crates/obscura-cli/src/main.rs b/crates/obscura-cli/src/main.rs index 45d474d..cf4e059 100644 --- a/crates/obscura-cli/src/main.rs +++ b/crates/obscura-cli/src/main.rs @@ -19,7 +19,7 @@ struct Args { #[arg(short, long, default_value_t = 9222)] port: u16, - #[arg(long)] + #[arg(long, global = true)] proxy: Option, #[arg(long)] @@ -136,7 +136,14 @@ fn select_log_filter(verbose: bool, quiet: bool) -> &'static str { } fn is_quiet_command(cmd: &Option) -> bool { - matches!(cmd, Some(Command::Fetch { quiet: true, .. })) + matches!( + cmd, + Some(Command::Fetch { quiet: true, .. }) | Some(Command::Scrape { quiet: true, .. }) + ) +} + +fn merge_proxy(global_proxy: Option, command_proxy: Option) -> Option { + command_proxy.or(global_proxy) } #[tokio::main(flavor = "current_thread")] @@ -153,8 +160,11 @@ async fn main() -> anyhow::Result<()> { .with_writer(std::io::stderr) .init(); + let global_proxy = args.proxy.clone(); + match args.command { Some(Command::Serve { port, proxy, user_agent, stealth, workers }) => { + let proxy = merge_proxy(global_proxy.clone(), proxy); print_banner(port); if let Some(ref proxy) = proxy { tracing::info!("Using proxy: {}", proxy); @@ -179,10 +189,10 @@ async fn main() -> anyhow::Result<()> { } } Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, output, quiet }) => { - run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, output, quiet).await?; + run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, output, quiet, global_proxy).await?; } Some(Command::Scrape { urls, eval, concurrency, format, timeout, quiet }) => { - run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout, quiet).await?; + run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout, quiet, global_proxy).await?; } None => { print_banner(args.port); @@ -335,8 +345,9 @@ async fn run_fetch( eval: Option, output: Option, quiet: bool, + proxy: Option, ) -> anyhow::Result<()> { - let context = Arc::new(BrowserContext::with_options("fetch".to_string(), None, stealth)); + let context = Arc::new(BrowserContext::with_options("fetch".to_string(), proxy, stealth)); let mut page = Page::new("fetch-page".to_string(), context); if let Some(ref ua) = user_agent { @@ -510,6 +521,7 @@ async fn run_parallel_scrape( format: &str, timeout_secs: u64, quiet: bool, + proxy: Option, ) -> anyhow::Result<()> { let total = urls.len(); let start = Instant::now(); @@ -550,6 +562,7 @@ async fn run_parallel_scrape( let sem = semaphore.clone(); let eval = eval.clone(); let worker_path = worker_path.clone(); + let proxy = proxy.clone(); let handle = tokio::spawn(async move { let _permit = sem.acquire().await.unwrap(); @@ -559,6 +572,7 @@ async fn run_parallel_scrape( .stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::null()) + .env("OBSCURA_PROXY", proxy.as_deref().unwrap_or("")) .spawn() { Ok(c) => c, @@ -771,7 +785,8 @@ fn dump_links(page: &Page) -> String { #[cfg(test)] mod tests { use super::{ - extract_readable_text, is_quiet_command, select_log_filter, write_or_print, Args, Command, + extract_readable_text, is_quiet_command, merge_proxy, select_log_filter, write_or_print, + Args, Command, }; use clap::Parser; use obscura_dom::parse_html; @@ -922,4 +937,21 @@ mod tests { assert!(!text.contains("console.log")); assert!(!text.contains("color: red")); } + + #[test] + fn command_proxy_overrides_global_proxy() { + let proxy = merge_proxy( + Some("http://global.example:8080".to_string()), + Some("socks5://127.0.0.1:1080".to_string()), + ); + + assert_eq!(proxy.as_deref(), Some("socks5://127.0.0.1:1080")); + } + + #[test] + fn global_proxy_is_used_when_command_proxy_is_absent() { + let proxy = merge_proxy(Some("http://global.example:8080".to_string()), None); + + assert_eq!(proxy.as_deref(), Some("http://global.example:8080")); + } } diff --git a/crates/obscura-cli/src/worker.rs b/crates/obscura-cli/src/worker.rs index 57a6123..9a1ad45 100644 --- a/crates/obscura-cli/src/worker.rs +++ b/crates/obscura-cli/src/worker.rs @@ -47,7 +47,11 @@ async fn main() { .with_writer(std::io::stderr) .init(); - let context = Arc::new(BrowserContext::new("worker".to_string())); + let proxy = std::env::var("OBSCURA_PROXY") + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + let context = Arc::new(BrowserContext::with_options("worker".to_string(), proxy, false)); let mut page = Page::new("page-1".to_string(), context); let stdin = tokio::io::stdin();