feat(cli): --allow-private-network for localhost development (#33) (#203)

Picks up @Timtech4u's PR #33, which has been waiting since April. The
  design (CLI flag + threading through BrowserContext + http client) is
  theirs and is correct; this is a clean rewrite on top of current main
  because repeated rebase rounds had been reverting unrelated work that
  landed in the meantime.

  Issue #4 blocked all loopback / RFC1918 / link-local fetches to prevent
  SSRF. The only escape was OBSCURA_ALLOW_PRIVATE_NETWORK=1, which is
  process-wide and clunky in pipelines. Multiple users (@Timtech4u in
  the original PR, then @duartemvix and @necrogami in the comment
  thread) reported they cannot use obscura for development against
  http://localhost:N without it.

  Adds --allow-private-network as a global CLI flag, works on fetch,
  serve, and scrape. Threads through BrowserContext::with_storage_and_network
  and the http client's validate_url. CdpContext gets a new_full
  builder and obscura_cdp::start_with_full_serve_options entry point so
  'obscura serve --allow-private-network' lets Puppeteer/Playwright
  navigate to localhost too. Default remains blocked.

  Backward compat: the env var still works (OR'd with the flag inside
  validate_url). All existing constructors and entry points unchanged.

  Verified end-to-end against a local http.server on 127.0.0.1: fetch
  and CDP Page.navigate both blocked by default, both succeed with the
  flag. Public URLs unaffected. Integration suite 28/28, full workspace
  cargo test all green.
This commit is contained in:
SGavrl authored and GitHub committed 2026-05-29 17:53:23 +00:00
1 parent 1a1253118f
commit e7bb0c676d
7 files changed
+140 -40

No files matched your search

+14 -3
View File
@@ -35,6 +35,14 @@ struct Args {
#[arg(long)]
storage_dir: Option<std::path::PathBuf>,
/// Permit fetches to loopback, RFC1918, and link-local addresses.
/// Default is to block them (SSRF fix from #4). Use this for local
/// development against http://localhost:N or http://192.168.x.y.
/// Equivalent to `OBSCURA_ALLOW_PRIVATE_NETWORK=1` but per-process
/// and survives in command pipelines.
#[arg(long, global = true)]
allow_private_network: bool,
/// Pass raw flags to V8, in the same form V8/Chromium/Node accept
/// (e.g. `"--max-old-space-size=4096 --max-semi-space-size=64 --expose-gc"`).
/// Applied once at startup before any isolate is created.
@@ -304,14 +312,15 @@ async fn main() -> anyhow::Result<()> {
tracing::info!("{} worker processes", workers);
run_multi_worker_serve(port, workers, proxy, stealth, user_agent).await?;
} else {
obscura_cdp::start_with_host_and_security(
obscura_cdp::start_with_full_serve_options(
port, &host, proxy, stealth, user_agent, allow_file_access, storage_dir,
args.allow_private_network,
).await?;
}
}
Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, output, quiet, storage_dir }) => {
reject_stealth_with_socks5(global_proxy.as_deref(), stealth)?;
run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, output, quiet, global_proxy, storage_dir).await?;
run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, output, quiet, global_proxy, storage_dir, args.allow_private_network).await?;
}
Some(Command::Scrape { urls, eval, concurrency, format, timeout, quiet }) => {
run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout, quiet, global_proxy).await?;
@@ -478,6 +487,7 @@ async fn run_fetch(
quiet: bool,
proxy: Option<String>,
storage_dir: Option<std::path::PathBuf>,
allow_private_network: bool,
) -> anyhow::Result<()> {
// --dump original short-circuits the browser stack entirely: fetch the raw
// response body via HTTP and stream the bytes verbatim. Useful for binary
@@ -495,12 +505,13 @@ async fn run_fetch(
return Ok(());
}
let context = Arc::new(BrowserContext::with_storage_full(
let context = Arc::new(BrowserContext::with_storage_and_network(
"fetch".to_string(),
proxy,
stealth,
user_agent.clone(),
storage_dir.clone(),
allow_private_network,
));
let mut page = Page::new("fetch-page".to_string(), context.clone());