Picks up @Timtech4u's PR #33, which has been waiting since April. The design (CLI flag + threading through BrowserContext + http client) is theirs and is correct; this is a clean rewrite on top of current main because repeated rebase rounds had been reverting unrelated work that landed in the meantime. Issue #4 blocked all loopback / RFC1918 / link-local fetches to prevent SSRF. The only escape was OBSCURA_ALLOW_PRIVATE_NETWORK=1, which is process-wide and clunky in pipelines. Multiple users (@Timtech4u in the original PR, then @duartemvix and @necrogami in the comment thread) reported they cannot use obscura for development against http://localhost:N without it. Adds --allow-private-network as a global CLI flag, works on fetch, serve, and scrape. Threads through BrowserContext::with_storage_and_network and the http client's validate_url. CdpContext gets a new_full builder and obscura_cdp::start_with_full_serve_options entry point so 'obscura serve --allow-private-network' lets Puppeteer/Playwright navigate to localhost too. Default remains blocked. Backward compat: the env var still works (OR'd with the flag inside validate_url). All existing constructors and entry points unchanged. Verified end-to-end against a local http.server on 127.0.0.1: fetch and CDP Page.navigate both blocked by default, both succeed with the flag. Public URLs unaffected. Integration suite 28/28, full workspace cargo test all green.
200 lines
7.1 KiB
Rust
200 lines
7.1 KiB
Rust
use std::path::PathBuf;
|
|
use std::sync::Arc;
|
|
|
|
use obscura_net::{CookieJar, ObscuraHttpClient, RobotsCache};
|
|
|
|
pub struct BrowserContext {
|
|
pub id: String,
|
|
pub cookie_jar: Arc<CookieJar>,
|
|
pub http_client: Arc<ObscuraHttpClient>,
|
|
pub user_agent: String,
|
|
pub proxy_url: Option<String>,
|
|
pub robots_cache: Arc<RobotsCache>,
|
|
pub obey_robots: bool,
|
|
pub stealth: bool,
|
|
/// When true, CDP-driven navigation to file:// URLs is permitted.
|
|
/// Default is false: a remote CDP client cannot point the browser
|
|
/// at /etc/shadow even if Obscura is running as a privileged user.
|
|
/// Flip on via `obscura serve --allow-file-access` for legitimate
|
|
/// local-HTML testing workflows. The CLI's own `obscura fetch
|
|
/// file://...` path is unaffected because it does not go through
|
|
/// the CDP server.
|
|
pub allow_file_access: bool,
|
|
pub storage_dir: Option<PathBuf>,
|
|
/// When true, the http client allows fetching localhost / RFC1918 /
|
|
/// link-local addresses. Set via `--allow-private-network` (issue #33).
|
|
/// Independent of `allow_file_access` because they cover different threat
|
|
/// models: file:// is a local file-system read, while private-network is
|
|
/// the broader SSRF gate from issue #4.
|
|
pub allow_private_network: bool,
|
|
}
|
|
|
|
impl BrowserContext {
|
|
pub fn new(id: String) -> Self {
|
|
Self::_new_inner(id, None, false, None, None, false)
|
|
}
|
|
|
|
/// Create a BrowserContext with an optional storage directory.
|
|
/// When `storage_dir` is set, cookies are automatically loaded from
|
|
/// `{storage_dir}/cookies.json` on creation.
|
|
pub fn with_storage(
|
|
id: String,
|
|
storage_dir: Option<PathBuf>,
|
|
) -> Self {
|
|
Self::_new_inner(id, None, false, None, storage_dir, false)
|
|
}
|
|
|
|
/// Create a BrowserContext with full options including storage_dir.
|
|
pub fn with_storage_full(
|
|
id: String,
|
|
proxy_url: Option<String>,
|
|
stealth: bool,
|
|
user_agent: Option<String>,
|
|
storage_dir: Option<PathBuf>,
|
|
) -> Self {
|
|
Self::_new_inner(id, proxy_url, stealth, user_agent, storage_dir, false)
|
|
}
|
|
|
|
/// Variant that also accepts the `allow_private_network` opt-in. All
|
|
/// pre-existing constructors default it to `false`; callers that want the
|
|
/// CLI's `--allow-private-network` (issue #33) behaviour go through here.
|
|
pub fn with_storage_and_network(
|
|
id: String,
|
|
proxy_url: Option<String>,
|
|
stealth: bool,
|
|
user_agent: Option<String>,
|
|
storage_dir: Option<PathBuf>,
|
|
allow_private_network: bool,
|
|
) -> Self {
|
|
Self::_new_inner(id, proxy_url, stealth, user_agent, storage_dir, allow_private_network)
|
|
}
|
|
|
|
fn _new_inner(
|
|
id: String,
|
|
proxy_url: Option<String>,
|
|
stealth: bool,
|
|
user_agent: Option<String>,
|
|
storage_dir: Option<PathBuf>,
|
|
allow_private_network: bool,
|
|
) -> Self {
|
|
let cookie_jar = Arc::new(CookieJar::new());
|
|
|
|
// Restore cookies from disk if storage_dir is configured
|
|
if let Some(ref dir) = storage_dir {
|
|
let cookie_path = dir.join("cookies.json");
|
|
if cookie_path.exists() {
|
|
match cookie_jar.load_from_file(&cookie_path) {
|
|
Ok(n) if n > 0 => {
|
|
tracing::info!("Loaded {} cookies from {}", n, cookie_path.display());
|
|
}
|
|
Ok(_) => {}
|
|
Err(e) => {
|
|
tracing::warn!("Failed to load cookies from {}: {}", cookie_path.display(), e);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
let mut client = ObscuraHttpClient::with_full_options(
|
|
cookie_jar.clone(),
|
|
proxy_url.as_deref(),
|
|
allow_private_network,
|
|
);
|
|
if stealth {
|
|
client.block_trackers = true;
|
|
}
|
|
let resolved_ua = user_agent.unwrap_or_else(|| {
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36".to_string()
|
|
});
|
|
// Sync the http client's UA at construction so navigation requests pick it
|
|
// up before any async setup runs. The lock has no other holders here, so
|
|
// try_write always succeeds; we fall back silently if it ever fails.
|
|
if let Ok(mut guard) = client.user_agent.try_write() {
|
|
*guard = resolved_ua.clone();
|
|
}
|
|
let http_client = Arc::new(client);
|
|
BrowserContext {
|
|
id,
|
|
cookie_jar,
|
|
http_client,
|
|
user_agent: resolved_ua,
|
|
proxy_url,
|
|
robots_cache: Arc::new(RobotsCache::new()),
|
|
obey_robots: false,
|
|
stealth,
|
|
allow_file_access: false,
|
|
storage_dir,
|
|
allow_private_network,
|
|
}
|
|
}
|
|
|
|
pub fn with_options(id: String, proxy_url: Option<String>, stealth: bool) -> Self {
|
|
Self::with_full_options(id, proxy_url, stealth, None)
|
|
}
|
|
|
|
pub fn with_full_options(
|
|
id: String,
|
|
proxy_url: Option<String>,
|
|
stealth: bool,
|
|
user_agent: Option<String>,
|
|
) -> Self {
|
|
Self::_new_inner(id, proxy_url, stealth, user_agent, None, false)
|
|
}
|
|
|
|
pub fn with_proxy(id: String, proxy_url: Option<String>) -> Self {
|
|
Self::with_options(id, proxy_url, false)
|
|
}
|
|
|
|
/// Persist cookies to disk if storage_dir is configured.
|
|
/// Called during graceful shutdown.
|
|
pub fn save_cookies(&self) {
|
|
if let Some(ref dir) = self.storage_dir {
|
|
let _ = std::fs::create_dir_all(dir);
|
|
let cookie_path = dir.join("cookies.json");
|
|
if let Err(e) = self.cookie_jar.save_to_file(&cookie_path) {
|
|
tracing::warn!("Failed to save cookies to {}: {}", cookie_path.display(), e);
|
|
} else {
|
|
tracing::info!("Saved cookies to {}", cookie_path.display());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[tokio::test(flavor = "current_thread")]
|
|
async fn with_full_options_propagates_user_agent_to_http_client() {
|
|
let ctx = BrowserContext::with_full_options(
|
|
"test".to_string(),
|
|
None,
|
|
false,
|
|
Some("Custom-UA/1.0".to_string()),
|
|
);
|
|
assert_eq!(ctx.user_agent, "Custom-UA/1.0");
|
|
let client_ua = ctx.http_client.user_agent.read().await.clone();
|
|
assert_eq!(client_ua, "Custom-UA/1.0");
|
|
}
|
|
|
|
#[tokio::test(flavor = "current_thread")]
|
|
async fn with_full_options_falls_back_to_chrome_default() {
|
|
let ctx = BrowserContext::with_full_options(
|
|
"test".to_string(),
|
|
None,
|
|
false,
|
|
None,
|
|
);
|
|
assert!(ctx.user_agent.contains("Chrome"));
|
|
let client_ua = ctx.http_client.user_agent.read().await.clone();
|
|
assert!(client_ua.contains("Chrome"));
|
|
assert_eq!(ctx.user_agent, client_ua);
|
|
}
|
|
|
|
#[tokio::test(flavor = "current_thread")]
|
|
async fn with_options_keeps_default_user_agent() {
|
|
let ctx = BrowserContext::with_options("test".to_string(), None, false);
|
|
assert!(ctx.user_agent.contains("Chrome"));
|
|
}
|
|
}
|