use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use std::net::{IpAddr, SocketAddr}; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use reqwest::header::{HeaderMap, HeaderName, HeaderValue, USER_AGENT}; use reqwest::redirect::Policy; use reqwest::{Client, Method}; use tokio::sync::RwLock; use url::Url; use crate::cookies::CookieJar; use crate::interceptor::{InterceptAction, RequestInterceptor}; #[derive(Debug, Clone)] pub struct Response { pub url: Url, pub status: u16, pub headers: HashMap, pub body: Vec, pub redirected_from: Vec, } impl Response { /// Decode the body as text, honoring the response charset. /// /// Uses the HTTP `Content-Type` header's `charset=` parameter, then for /// HTML responses falls back to sniffing `` in the first /// 1KB, then UTF-8. Mirrors browser behaviour per the HTML5 spec. pub fn text(&self) -> String { if self.is_html() { crate::encoding::decode_response(&self.body, self.content_type()) } else { crate::encoding::decode_non_html(&self.body, self.content_type()) } } pub fn header(&self, name: &str) -> Option<&str> { self.headers.get(&name.to_lowercase()).map(|s| s.as_str()) } pub fn content_type(&self) -> Option<&str> { self.header("content-type") } pub fn is_html(&self) -> bool { self.content_type() .map(|ct| ct.contains("text/html")) .unwrap_or(false) } } #[derive(Debug, Clone)] pub struct RequestInfo { pub url: Url, pub method: String, pub headers: HashMap, pub resource_type: ResourceType, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ResourceType { Document, Script, Stylesheet, Image, Font, Xhr, Fetch, Other, } pub type RequestCallback = Arc; pub type ResponseCallback = Arc; /// Process-wide opt-in via env var. Older flow that issue #4 introduced. The /// new `--allow-private-network` CLI flag (issue #33) sets a per-client field /// that is OR'd with this so existing scripts and Docker setups that pin the /// env var keep working unchanged. pub fn env_allows_private_network() -> bool { matches!( std::env::var("OBSCURA_ALLOW_PRIVATE_NETWORK") .ok() .as_deref() .map(str::trim) .map(str::to_ascii_lowercase) .as_deref(), Some("1") | Some("true") | Some("yes") | Some("on") ) } /// True when `ip` must never be the target of an outbound request from the /// engine: loopback, RFC1918 private, link-local (incl. the 169.254.169.254 /// cloud-metadata endpoint), broadcast, documentation, the unspecified address /// (0.0.0.0 / ::, which the OS routes to localhost), IPv6 unique-local /// (fc00::/7), and any IPv4-mapped/compatible IPv6 form of the above. /// Centralizes the SSRF deny-set so the literal-host check and the /// DNS-resolution check (`SsrfGuardResolver`) can never disagree. pub fn is_forbidden_ip(ip: IpAddr) -> bool { match ip { IpAddr::V4(v4) => { v4.is_loopback() || v4.is_private() || v4.is_link_local() || v4.is_broadcast() || v4.is_documentation() || v4.is_unspecified() } IpAddr::V6(v6) => { if v6.is_loopback() || v6.is_unspecified() || v6.is_unique_local() || v6.is_unicast_link_local() { return true; } // Unwrap IPv4-mapped (::ffff:a.b.c.d) and IPv4-compatible (::a.b.c.d) // forms and re-check the embedded v4 so e.g. [::ffff:127.0.0.1] or // [::ffff:169.254.169.254] cannot slip past the v6 arm. if let Some(v4) = v6.to_ipv4_mapped().or_else(|| v6.to_ipv4()) { return is_forbidden_ip(IpAddr::V4(v4)); } false } } } /// reqwest DNS resolver that performs the lookup and then rejects the whole /// request if ANY resolved address is in the SSRF deny-set. This closes the /// DNS-rebinding bypass a host-string check alone cannot: a public name that /// resolves to 127.0.0.1 / 169.254.169.254 / an RFC1918 address is blocked at /// connect time, using the very addresses reqwest will dial. When private /// access is permitted (`--allow-private-network` or /// `OBSCURA_ALLOW_PRIVATE_NETWORK`) the lookup passes through unfiltered. pub struct SsrfGuardResolver { allow_private: bool, } impl SsrfGuardResolver { pub fn new(allow_private: bool) -> Self { Self { allow_private } } } impl Resolve for SsrfGuardResolver { fn resolve(&self, name: Name) -> Resolving { let allow = self.allow_private || env_allows_private_network(); let host = name.as_str().to_string(); Box::pin(async move { let addrs: Vec = tokio::net::lookup_host((host.as_str(), 0)) .await .map_err(|e| -> Box { Box::new(e) })? .collect(); if !allow { if let Some(bad) = addrs.iter().find(|sa| is_forbidden_ip(sa.ip())) { return Err(format!( "SSRF blocked: '{}' resolves to forbidden address {}", host, bad.ip() ) .into()); } } let iter: Addrs = Box::new(addrs.into_iter()); Ok(iter) }) } } fn validate_url(url: &Url, allow_private_network: bool) -> Result<(), ObscuraNetError> { let allow_private_network = allow_private_network || env_allows_private_network(); let scheme = url.scheme(); if scheme != "http" && scheme != "https" && scheme != "file" { return Err(ObscuraNetError::Network(format!( "Forbidden URL scheme '{}' - only http, https, and file are allowed", scheme ))); } if scheme == "file" || allow_private_network { return Ok(()); } if let Some(host) = url.host() { match host { url::Host::Ipv4(ip) => { if is_forbidden_ip(IpAddr::V4(ip)) { return Err(ObscuraNetError::Network(format!( "Access to private/internal IP address {} is not allowed", ip ))); } } url::Host::Ipv6(ip) => { if is_forbidden_ip(IpAddr::V6(ip)) { return Err(ObscuraNetError::Network(format!( "Access to private/internal IPv6 address {} is not allowed", ip ))); } } url::Host::Domain(domain) => { let lower_domain = domain.to_lowercase(); if lower_domain == "localhost" || lower_domain.ends_with(".localhost") || lower_domain == "127.0.0.1" || lower_domain == "::1" { return Err(ObscuraNetError::Network(format!( "Access to localhost domain '{}' is not allowed", domain ))); } } } } Ok(()) } async fn fetch_file_url(url: &Url) -> Result { let path = url .to_file_path() .map_err(|_| ObscuraNetError::Network("Invalid file URL".to_string()))?; let body = tokio::fs::read(&path) .await .map_err(|e| ObscuraNetError::Network(format!("Failed to read file: {}", e)))?; let mut headers = HashMap::new(); if let Some(ext) = path.extension().and_then(|e| e.to_str()) { let ct = match ext.to_lowercase().as_str() { "html" | "htm" => "text/html", "css" => "text/css", "js" | "mjs" => "application/javascript", "json" => "application/json", "png" => "image/png", "jpg" | "jpeg" => "image/jpeg", "gif" => "image/gif", "svg" => "image/svg+xml", "webp" => "image/webp", "ico" => "image/x-icon", _ => "application/octet-stream", }; headers.insert("content-type".to_string(), ct.to_string()); } Ok(Response { url: url.clone(), status: 200, headers, body, redirected_from: Vec::new(), }) } pub struct ObscuraHttpClient { client: tokio::sync::OnceCell, proxy_url: Option, pub cookie_jar: Arc, pub user_agent: RwLock, pub extra_headers: RwLock>, pub interceptor: RwLock>>, pub on_request: RwLock>, pub on_response: RwLock>, pub timeout: Duration, pub in_flight: Arc, pub block_trackers: bool, /// When true, `validate_url` lets localhost / RFC1918 / link-local addresses /// through in addition to the `OBSCURA_ALLOW_PRIVATE_NETWORK` env var. /// Set via `--allow-private-network` on the CLI (issue #33). pub allow_private_network: bool, } /// Derive the sec-ch-ua and sec-ch-ua-platform client-hint header values from a /// User-Agent string, using Chromium's per-major-version GREASE algorithm so /// the non-stealth HTTP path agrees with navigator.userAgentData instead of /// shipping a fixed Linux/Chrome-145 hint that contradicts a Windows profile. fn chrome_client_hints(ua: &str) -> (String, String) { let major: usize = ua .split("Chrome/") .nth(1) .and_then(|s| s.split('.').next()) .and_then(|s| s.parse::().ok()) .unwrap_or(145); const GREASE_CHARS: [char; 11] = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_']; const GREASE_VER: [&str; 3] = ["8", "99", "24"]; const PERMS: [[usize; 3]; 6] = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]]; let grease_brand = format!( "Not{}A{}Brand", GREASE_CHARS[major % 11], GREASE_CHARS[(major + 1) % 11] ); let brands = [ (grease_brand, GREASE_VER[major % 3].to_string()), ("Chromium".to_string(), major.to_string()), ("Google Chrome".to_string(), major.to_string()), ]; let p = PERMS[major % 6]; let sec_ch_ua = p .iter() .map(|&i| format!("\"{}\";v=\"{}\"", brands[i].0, brands[i].1)) .collect::>() .join(", "); let platform = if ua.contains("Windows NT") { "\"Windows\"" } else if ua.contains("Macintosh") { "\"macOS\"" } else { "\"Linux\"" }; (sec_ch_ua, platform.to_string()) } impl ObscuraHttpClient { pub fn new() -> Self { Self::with_cookie_jar(Arc::new(CookieJar::new())) } pub fn with_cookie_jar(cookie_jar: Arc) -> Self { Self::with_options(cookie_jar, None) } pub fn with_options(cookie_jar: Arc, proxy_url: Option<&str>) -> Self { Self::with_full_options(cookie_jar, proxy_url, false) } pub fn with_full_options( cookie_jar: Arc, proxy_url: Option<&str>, allow_private_network: bool, ) -> Self { ObscuraHttpClient { client: tokio::sync::OnceCell::new(), proxy_url: proxy_url.map(|s| s.to_string()), cookie_jar, user_agent: RwLock::new( "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36".to_string(), ), extra_headers: RwLock::new(HashMap::new()), interceptor: RwLock::new(None), on_request: RwLock::new(Vec::new()), on_response: RwLock::new(Vec::new()), in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)), timeout: Duration::from_secs(30), block_trackers: false, allow_private_network, } } async fn get_client(&self) -> &Client { self.client.get_or_init(|| async { let mut builder = Client::builder() .redirect(Policy::none()) .timeout(Duration::from_secs(30)) .danger_accept_invalid_certs(false) // SSRF guard: reject hostnames that resolve to a private/loopback IP. .dns_resolver(Arc::new(SsrfGuardResolver::new(self.allow_private_network))) ; if let Some(ref proxy) = self.proxy_url { if let Ok(p) = reqwest::Proxy::all(proxy.as_str()) { builder = builder.proxy(p); } } builder.build().expect("failed to build HTTP client") }).await } /// Read-only accessor for the proxy URL the client was configured with /// (if any). Exposed so callers outside the `obscura-net` crate — notably /// `op_fetch_url` in `obscura-js` (#139) — can route their own reqwest /// requests through the same upstream proxy. pub fn proxy_url(&self) -> Option<&str> { self.proxy_url.as_deref() } pub async fn fetch(&self, url: &Url) -> Result { self.fetch_with_method(Method::GET, url, None).await } pub async fn post_form(&self, url: &Url, body: &str) -> Result { self.fetch_with_method(Method::POST, url, Some(body.as_bytes().to_vec())).await } pub async fn fetch_with_method( &self, initial_method: Method, url: &Url, initial_body: Option>, ) -> Result { validate_url(url, self.allow_private_network)?; if url.scheme() == "file" { return fetch_file_url(url).await; } let mut method = initial_method; let mut body = initial_body; if self.block_trackers { if let Some(host) = url.host_str() { if crate::blocklist::is_blocked(host) { tracing::debug!("Blocked tracker: {}", url); return Ok(Response { status: 0, url: url.clone(), headers: HashMap::new(), body: Vec::new(), redirected_from: Vec::new(), }); } } } let mut current_url = url.clone(); let mut redirects = Vec::new(); let max_redirects = 20; for _redirect_count in 0..max_redirects { let request_info = RequestInfo { url: current_url.clone(), method: method.to_string(), headers: self.extra_headers.read().await.clone(), resource_type: ResourceType::Document, }; if let Some(interceptor) = self.interceptor.read().await.as_ref() { match interceptor.intercept(&request_info).await { InterceptAction::Continue => {} InterceptAction::Block => { return Err(ObscuraNetError::Blocked(current_url.to_string())); } InterceptAction::Fulfill(response) => { return Ok(response); } InterceptAction::ModifyHeaders(headers) => { let mut extra = self.extra_headers.write().await; extra.extend(headers); } } } for cb in self.on_request.read().await.iter() { cb(&request_info); } let ua = self.user_agent.read().await.clone(); let (sec_ch_ua, sec_ch_ua_platform) = chrome_client_hints(&ua); let mut headers = HeaderMap::new(); // Chrome's top-level navigation header order. (reqwest appends // accept-encoding/host after these, so accept-encoding lands after // accept-language rather than before it; the rest matches Chrome.) headers.insert( HeaderName::from_static("sec-ch-ua"), HeaderValue::from_str(&sec_ch_ua) .unwrap_or_else(|_| HeaderValue::from_static("\"Not:A-Brand\";v=\"99\", \"Google Chrome\";v=\"145\", \"Chromium\";v=\"145\"")), ); headers.insert(HeaderName::from_static("sec-ch-ua-mobile"), HeaderValue::from_static("?0")); headers.insert( HeaderName::from_static("sec-ch-ua-platform"), HeaderValue::from_str(&sec_ch_ua_platform) .unwrap_or_else(|_| HeaderValue::from_static("\"Windows\"")), ); headers.insert(HeaderName::from_static("upgrade-insecure-requests"), HeaderValue::from_static("1")); headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap_or_else(|_| { HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36") })); headers.insert( reqwest::header::ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"), ); headers.insert(HeaderName::from_static("sec-fetch-site"), HeaderValue::from_static("none")); headers.insert(HeaderName::from_static("sec-fetch-mode"), HeaderValue::from_static("navigate")); headers.insert(HeaderName::from_static("sec-fetch-user"), HeaderValue::from_static("?1")); headers.insert(HeaderName::from_static("sec-fetch-dest"), HeaderValue::from_static("document")); headers.insert( reqwest::header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"), ); let cookie_header = self.cookie_jar.get_cookie_header(¤t_url); tracing::debug!( "Cookie header for {}: {} cookies ({} bytes)", current_url.host_str().unwrap_or("?"), cookie_header.split("; ").filter(|s| !s.is_empty()).count(), cookie_header.len(), ); if !cookie_header.is_empty() { match HeaderValue::from_str(&cookie_header) { Ok(val) => { headers.insert(reqwest::header::COOKIE, val); } Err(_) => { let filtered: String = cookie_header .split("; ") .filter(|pair| HeaderValue::from_str(pair).is_ok()) .collect::>() .join("; "); if !filtered.is_empty() { if let Ok(val) = HeaderValue::from_str(&filtered) { headers.insert(reqwest::header::COOKIE, val); } } tracing::debug!( "Cookie header invalid chars, filtered {} -> {} bytes", cookie_header.len(), filtered.len(), ); } } } for (k, v) in self.extra_headers.read().await.iter() { if let (Ok(name), Ok(val)) = ( HeaderName::from_bytes(k.as_bytes()), HeaderValue::from_str(v), ) { headers.insert(name, val); } } let mut req_builder = self.get_client().await.request(method.clone(), current_url.as_str()) .headers(headers); if let Some(ref b) = body { if method == Method::POST { req_builder = req_builder.header( reqwest::header::CONTENT_TYPE, "application/x-www-form-urlencoded", ); } req_builder = req_builder.body(b.clone()); } self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let resp = req_builder.send().await.map_err(|e| { self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); ObscuraNetError::Network(format!("{}: {}", current_url, e)) })?; self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); let status = resp.status(); for val in resp.headers().get_all(reqwest::header::SET_COOKIE) { if let Ok(s) = val.to_str() { self.cookie_jar.set_cookie(s, ¤t_url); } } let response_headers: HashMap = resp .headers() .iter() .map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string())) .collect(); if status.is_redirection() { if let Some(location) = resp.headers().get(reqwest::header::LOCATION) { let location_str = location.to_str().map_err(|_| { ObscuraNetError::Network("Invalid redirect Location header".into()) })?; let next_url = current_url.join(location_str).map_err(|e| { ObscuraNetError::Network(format!("Invalid redirect URL: {}", e)) })?; validate_url(&next_url, self.allow_private_network)?; redirects.push(current_url.clone()); current_url = next_url; if status == reqwest::StatusCode::MOVED_PERMANENTLY || status == reqwest::StatusCode::FOUND || status == reqwest::StatusCode::SEE_OTHER { method = Method::GET; body = None; } continue; } } let body_bytes = resp.bytes().await.map_err(|e| { ObscuraNetError::Network(format!("Failed to read body: {}", e)) })?.to_vec(); let response = Response { url: current_url, status: status.as_u16(), headers: response_headers, body: body_bytes, redirected_from: redirects, }; for cb in self.on_response.read().await.iter() { cb(&request_info, &response); } return Ok(response); } Err(ObscuraNetError::TooManyRedirects(current_url.to_string())) } pub async fn set_user_agent(&self, ua: &str) { *self.user_agent.write().await = ua.to_string(); } pub async fn set_extra_headers(&self, headers: HashMap) { *self.extra_headers.write().await = headers; } pub fn active_requests(&self) -> u32 { self.in_flight.load(std::sync::atomic::Ordering::Relaxed) } pub fn is_network_idle(&self) -> bool { self.active_requests() == 0 } } impl Default for ObscuraHttpClient { fn default() -> Self { Self::new() } } #[derive(Debug, thiserror::Error)] pub enum ObscuraNetError { #[error("Network error: {0}")] Network(String), #[error("Too many redirects: {0}")] TooManyRedirects(String), #[error("Request blocked: {0}")] Blocked(String), } #[cfg(test)] mod ssrf_tests { use super::{is_forbidden_ip, validate_url, SsrfGuardResolver}; use reqwest::dns::{Name, Resolve}; use std::net::IpAddr; use std::str::FromStr; use url::Url; fn ip(s: &str) -> IpAddr { IpAddr::from_str(s).unwrap() } #[test] fn ipv4_private_and_special_ranges_are_forbidden() { for s in [ "127.0.0.1", "127.5.6.7", "10.0.0.1", "172.16.0.1", "192.168.1.1", "169.254.169.254", // cloud-metadata endpoint "0.0.0.0", // unspecified -> localhost (was a bypass) "255.255.255.255", // broadcast "192.0.2.1", // documentation ] { assert!(is_forbidden_ip(ip(s)), "{s} should be forbidden"); } } #[test] fn public_ipv4_is_allowed() { for s in ["1.1.1.1", "8.8.8.8", "93.184.216.34"] { assert!(!is_forbidden_ip(ip(s)), "{s} should be allowed"); } } #[test] fn ipv6_loopback_ula_linklocal_and_mapped_are_forbidden() { for s in [ "::1", // loopback "::", // unspecified "fc00::1", // unique-local (was a bypass) "fd12:3456:789a::1", // unique-local "fe80::1", // link-local "::ffff:127.0.0.1", // v4-mapped loopback (was a bypass) "::ffff:169.254.169.254", // v4-mapped metadata ] { assert!(is_forbidden_ip(ip(s)), "{s} should be forbidden"); } } #[test] fn public_ipv6_is_allowed() { assert!(!is_forbidden_ip(ip("2606:4700:4700::1111"))); // cloudflare dns } #[test] fn validate_url_blocks_unspecified_and_allows_public() { // 0.0.0.0 previously slipped through validate_url's literal-host check. assert!(validate_url(&Url::parse("http://0.0.0.0:8080/").unwrap(), false).is_err()); assert!(validate_url(&Url::parse("http://127.0.0.1/").unwrap(), false).is_err()); assert!(validate_url(&Url::parse("http://example.com/").unwrap(), false).is_ok()); // The allow flag bypasses the guard (local-dev escape hatch). assert!(validate_url(&Url::parse("http://127.0.0.1/").unwrap(), true).is_ok()); } #[tokio::test] async fn resolver_blocks_hostname_that_resolves_to_loopback() { // localtest.me is a public DNS name that resolves to 127.0.0.1 — the // canonical DNS-rebinding test. The guard must reject it. If DNS is // unavailable the lookup itself errors (also Err), so the assertion // holds either way. let r = SsrfGuardResolver::new(false); let res = r.resolve(Name::from_str("localtest.me").unwrap()).await; assert!(res.is_err(), "localtest.me -> 127.0.0.1 must be blocked"); } #[tokio::test] async fn resolver_does_not_ssrf_block_public_host() { // A public host must not be SSRF-blocked. Tolerate a no-network sandbox // by only failing on an actual SSRF rejection, not a lookup failure. let r = SsrfGuardResolver::new(false); match r.resolve(Name::from_str("example.com").unwrap()).await { Ok(_) => {} Err(e) => assert!( !e.to_string().contains("SSRF blocked"), "example.com wrongly SSRF-blocked: {e}" ), } } }