diff --git a/crates/obscura-browser/src/context.rs b/crates/obscura-browser/src/context.rs index eb9f462..f4509b4 100644 --- a/crates/obscura-browser/src/context.rs +++ b/crates/obscura-browser/src/context.rs @@ -21,11 +21,17 @@ pub struct BrowserContext { /// the CDP server. pub allow_file_access: bool, pub storage_dir: Option, + /// When true, the http client allows fetching localhost / RFC1918 / + /// link-local addresses. Set via `--allow-private-network` (issue #33). + /// Independent of `allow_file_access` because they cover different threat + /// models: file:// is a local file-system read, while private-network is + /// the broader SSRF gate from issue #4. + pub allow_private_network: bool, } impl BrowserContext { pub fn new(id: String) -> Self { - Self::_new_inner(id, None, false, None, None) + Self::_new_inner(id, None, false, None, None, false) } /// Create a BrowserContext with an optional storage directory. @@ -35,7 +41,7 @@ impl BrowserContext { id: String, storage_dir: Option, ) -> Self { - Self::_new_inner(id, None, false, None, storage_dir) + Self::_new_inner(id, None, false, None, storage_dir, false) } /// Create a BrowserContext with full options including storage_dir. @@ -46,7 +52,21 @@ impl BrowserContext { user_agent: Option, storage_dir: Option, ) -> Self { - Self::_new_inner(id, proxy_url, stealth, user_agent, storage_dir) + Self::_new_inner(id, proxy_url, stealth, user_agent, storage_dir, false) + } + + /// Variant that also accepts the `allow_private_network` opt-in. All + /// pre-existing constructors default it to `false`; callers that want the + /// CLI's `--allow-private-network` (issue #33) behaviour go through here. + pub fn with_storage_and_network( + id: String, + proxy_url: Option, + stealth: bool, + user_agent: Option, + storage_dir: Option, + allow_private_network: bool, + ) -> Self { + Self::_new_inner(id, proxy_url, stealth, user_agent, storage_dir, allow_private_network) } fn _new_inner( @@ -55,6 +75,7 @@ impl BrowserContext { stealth: bool, user_agent: Option, storage_dir: Option, + allow_private_network: bool, ) -> Self { let cookie_jar = Arc::new(CookieJar::new()); @@ -74,9 +95,10 @@ impl BrowserContext { } } - let mut client = ObscuraHttpClient::with_options( + let mut client = ObscuraHttpClient::with_full_options( cookie_jar.clone(), proxy_url.as_deref(), + allow_private_network, ); if stealth { client.block_trackers = true; @@ -102,6 +124,7 @@ impl BrowserContext { stealth, allow_file_access: false, storage_dir, + allow_private_network, } } @@ -115,7 +138,7 @@ impl BrowserContext { stealth: bool, user_agent: Option, ) -> Self { - Self::_new_inner(id, proxy_url, stealth, user_agent, None) + Self::_new_inner(id, proxy_url, stealth, user_agent, None, false) } pub fn with_proxy(id: String, proxy_url: Option) -> Self { diff --git a/crates/obscura-cdp/src/dispatch.rs b/crates/obscura-cdp/src/dispatch.rs index 51c9f5d..3c89f5c 100644 --- a/crates/obscura-cdp/src/dispatch.rs +++ b/crates/obscura-cdp/src/dispatch.rs @@ -74,7 +74,7 @@ impl CdpContext { user_agent: Option, storage_dir: Option, ) -> Self { - Self::_new_inner(proxy, stealth, user_agent, storage_dir, false) + Self::_new_inner(proxy, stealth, user_agent, storage_dir, false, false) } pub fn new_with_security( @@ -83,7 +83,22 @@ impl CdpContext { user_agent: Option, allow_file_access: bool, ) -> Self { - Self::_new_inner(proxy, stealth, user_agent, None, allow_file_access) + Self::_new_inner(proxy, stealth, user_agent, None, allow_file_access, false) + } + + /// Kitchen-sink constructor that also threads `allow_private_network` + /// (issue #33). Older `new_with_*` builders stay as-is. + pub fn new_full( + proxy: Option, + stealth: bool, + user_agent: Option, + storage_dir: Option, + allow_file_access: bool, + allow_private_network: bool, + ) -> Self { + Self::_new_inner( + proxy, stealth, user_agent, storage_dir, allow_file_access, allow_private_network, + ) } fn _new_inner( @@ -92,23 +107,16 @@ impl CdpContext { user_agent: Option, storage_dir: Option, allow_file_access: bool, + allow_private_network: bool, ) -> Self { - let mut ctx = if let Some(ref dir) = storage_dir { - BrowserContext::with_storage_full( - "default".to_string(), - proxy, - stealth, - user_agent, - Some(dir.clone()), - ) - } else { - BrowserContext::with_full_options( - "default".to_string(), - proxy, - stealth, - user_agent, - ) - }; + let mut ctx = BrowserContext::with_storage_and_network( + "default".to_string(), + proxy, + stealth, + user_agent, + storage_dir, + allow_private_network, + ); ctx.allow_file_access = allow_file_access; let default_context = Arc::new(ctx); // Pre-seed with the default-frame execution context ids that diff --git a/crates/obscura-cdp/src/lib.rs b/crates/obscura-cdp/src/lib.rs index e037ba5..9e0f7e8 100644 --- a/crates/obscura-cdp/src/lib.rs +++ b/crates/obscura-cdp/src/lib.rs @@ -6,6 +6,6 @@ pub mod cookie_params; pub(crate) mod util; pub use server::{ - start, start_with_full_options, start_with_host, start_with_host_and_security, - start_with_options, + start, start_with_full_options, start_with_full_serve_options, start_with_host, + start_with_host_and_security, start_with_options, }; diff --git a/crates/obscura-cdp/src/server.rs b/crates/obscura-cdp/src/server.rs index 23c6732..194c85d 100644 --- a/crates/obscura-cdp/src/server.rs +++ b/crates/obscura-cdp/src/server.rs @@ -80,7 +80,10 @@ pub async fn start_with_host_and_security( allow_file_access: bool, storage_dir: Option, ) -> anyhow::Result<()> { - start_with_host_security_and_storage(port, host, proxy, stealth, user_agent, allow_file_access, storage_dir).await + start_with_full_serve_options( + port, host, proxy, stealth, user_agent, allow_file_access, storage_dir, false, + ) + .await } pub async fn start_with_host_security_and_storage( @@ -91,6 +94,25 @@ pub async fn start_with_host_security_and_storage( user_agent: Option, allow_file_access: bool, storage_dir: Option, +) -> anyhow::Result<()> { + start_with_full_serve_options( + port, host, proxy, stealth, user_agent, allow_file_access, storage_dir, false, + ) + .await +} + +/// Full serve entry point that also accepts `allow_private_network` (issue +/// #33). Older entry points default it to `false` so existing callers and +/// public API consumers are unaffected. +pub async fn start_with_full_serve_options( + port: u16, + host: &str, + proxy: Option, + stealth: bool, + user_agent: Option, + allow_file_access: bool, + storage_dir: Option, + allow_private_network: bool, ) -> anyhow::Result<()> { let ip: std::net::IpAddr = host .parse() @@ -162,6 +184,7 @@ pub async fn start_with_host_security_and_storage( let _processor_handle = tokio::task::spawn_local(cdp_processor( msg_rx, proxy, stealth, user_agent, allow_file_access, storage_dir, + allow_private_network, )); while let Some(stream) = ws_rx.recv().await { @@ -300,13 +323,16 @@ async fn cdp_processor( user_agent: Option, allow_file_access: bool, storage_dir: Option, + allow_private_network: bool, ) { - let mut ctx = if storage_dir.is_some() { - // Use storage-aware context creation with security settings - CdpContext::new_with_storage(proxy, stealth, user_agent, storage_dir) - } else { - CdpContext::new_with_security(proxy, stealth, user_agent, allow_file_access) - }; + let mut ctx = CdpContext::new_full( + proxy, + stealth, + user_agent, + storage_dir, + allow_file_access, + allow_private_network, + ); let (itx, irx) = mpsc::unbounded_channel::(); ctx.intercept_tx = Some(itx); let mut intercept_rx: Option> = Some(irx); diff --git a/crates/obscura-cli/src/main.rs b/crates/obscura-cli/src/main.rs index 194ee20..56f35a9 100644 --- a/crates/obscura-cli/src/main.rs +++ b/crates/obscura-cli/src/main.rs @@ -35,6 +35,14 @@ struct Args { #[arg(long)] storage_dir: Option, + /// Permit fetches to loopback, RFC1918, and link-local addresses. + /// Default is to block them (SSRF fix from #4). Use this for local + /// development against http://localhost:N or http://192.168.x.y. + /// Equivalent to `OBSCURA_ALLOW_PRIVATE_NETWORK=1` but per-process + /// and survives in command pipelines. + #[arg(long, global = true)] + allow_private_network: bool, + /// Pass raw flags to V8, in the same form V8/Chromium/Node accept /// (e.g. `"--max-old-space-size=4096 --max-semi-space-size=64 --expose-gc"`). /// Applied once at startup before any isolate is created. @@ -304,14 +312,15 @@ async fn main() -> anyhow::Result<()> { tracing::info!("{} worker processes", workers); run_multi_worker_serve(port, workers, proxy, stealth, user_agent).await?; } else { - obscura_cdp::start_with_host_and_security( + obscura_cdp::start_with_full_serve_options( port, &host, proxy, stealth, user_agent, allow_file_access, storage_dir, + args.allow_private_network, ).await?; } } Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, output, quiet, storage_dir }) => { reject_stealth_with_socks5(global_proxy.as_deref(), stealth)?; - run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, output, quiet, global_proxy, storage_dir).await?; + run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, output, quiet, global_proxy, storage_dir, args.allow_private_network).await?; } Some(Command::Scrape { urls, eval, concurrency, format, timeout, quiet }) => { run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout, quiet, global_proxy).await?; @@ -478,6 +487,7 @@ async fn run_fetch( quiet: bool, proxy: Option, storage_dir: Option, + allow_private_network: bool, ) -> anyhow::Result<()> { // --dump original short-circuits the browser stack entirely: fetch the raw // response body via HTTP and stream the bytes verbatim. Useful for binary @@ -495,12 +505,13 @@ async fn run_fetch( return Ok(()); } - let context = Arc::new(BrowserContext::with_storage_full( + let context = Arc::new(BrowserContext::with_storage_and_network( "fetch".to_string(), proxy, stealth, user_agent.clone(), storage_dir.clone(), + allow_private_network, )); let mut page = Page::new("fetch-page".to_string(), context.clone()); diff --git a/crates/obscura-net/src/client.rs b/crates/obscura-net/src/client.rs index e283f80..d7dca39 100644 --- a/crates/obscura-net/src/client.rs +++ b/crates/obscura-net/src/client.rs @@ -72,8 +72,24 @@ pub enum ResourceType { pub type RequestCallback = Arc; pub type ResponseCallback = Arc; -fn validate_url(url: &Url) -> Result<(), ObscuraNetError> { - let allow_private_network = std::env::var_os("OBSCURA_ALLOW_PRIVATE_NETWORK").is_some(); +/// Process-wide opt-in via env var. Older flow that issue #4 introduced. The +/// new `--allow-private-network` CLI flag (issue #33) sets a per-client field +/// that is OR'd with this so existing scripts and Docker setups that pin the +/// env var keep working unchanged. +pub fn env_allows_private_network() -> bool { + matches!( + std::env::var("OBSCURA_ALLOW_PRIVATE_NETWORK") + .ok() + .as_deref() + .map(str::trim) + .map(str::to_ascii_lowercase) + .as_deref(), + Some("1") | Some("true") | Some("yes") | Some("on") + ) +} + +fn validate_url(url: &Url, allow_private_network: bool) -> Result<(), ObscuraNetError> { + let allow_private_network = allow_private_network || env_allows_private_network(); let scheme = url.scheme(); if scheme != "http" && scheme != "https" && scheme != "file" { return Err(ObscuraNetError::Network(format!( @@ -175,6 +191,10 @@ pub struct ObscuraHttpClient { pub timeout: Duration, pub in_flight: Arc, pub block_trackers: bool, + /// When true, `validate_url` lets localhost / RFC1918 / link-local addresses + /// through in addition to the `OBSCURA_ALLOW_PRIVATE_NETWORK` env var. + /// Set via `--allow-private-network` on the CLI (issue #33). + pub allow_private_network: bool, } impl ObscuraHttpClient { @@ -187,6 +207,14 @@ impl ObscuraHttpClient { } pub fn with_options(cookie_jar: Arc, proxy_url: Option<&str>) -> Self { + Self::with_full_options(cookie_jar, proxy_url, false) + } + + pub fn with_full_options( + cookie_jar: Arc, + proxy_url: Option<&str>, + allow_private_network: bool, + ) -> Self { ObscuraHttpClient { client: tokio::sync::OnceCell::new(), proxy_url: proxy_url.map(|s| s.to_string()), @@ -201,6 +229,7 @@ impl ObscuraHttpClient { in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)), timeout: Duration::from_secs(30), block_trackers: false, + allow_private_network, } } @@ -244,7 +273,7 @@ impl ObscuraHttpClient { url: &Url, initial_body: Option>, ) -> Result { - validate_url(url)?; + validate_url(url, self.allow_private_network)?; if url.scheme() == "file" { return fetch_file_url(url).await; @@ -427,7 +456,7 @@ impl ObscuraHttpClient { let next_url = current_url.join(location_str).map_err(|e| { ObscuraNetError::Network(format!("Invalid redirect URL: {}", e)) })?; - validate_url(&next_url)?; + validate_url(&next_url, self.allow_private_network)?; redirects.push(current_url.clone()); current_url = next_url; if status == reqwest::StatusCode::MOVED_PERMANENTLY diff --git a/crates/obscura-net/src/lib.rs b/crates/obscura-net/src/lib.rs index b319d08..0aaa061 100644 --- a/crates/obscura-net/src/lib.rs +++ b/crates/obscura-net/src/lib.rs @@ -7,7 +7,10 @@ pub mod blocklist; #[cfg(feature = "stealth")] pub mod wreq_client; -pub use client::{ObscuraHttpClient, ObscuraNetError, RequestInfo, ResourceType, Response}; +pub use client::{ + env_allows_private_network, ObscuraHttpClient, ObscuraNetError, RequestInfo, ResourceType, + Response, +}; pub use cookies::{CookieInfo, CookieJar}; pub use encoding::{decode_non_html, decode_response}; pub use robots::RobotsCache;