use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use reqwest::header::{HeaderMap, HeaderName, HeaderValue, USER_AGENT}; use reqwest::redirect::Policy; use reqwest::{Client, Method}; use tokio::sync::RwLock; use url::Url; use crate::cookies::CookieJar; use crate::interceptor::{InterceptAction, RequestInterceptor}; #[derive(Debug, Clone)] pub struct Response { pub url: Url, pub status: u16, pub headers: HashMap, pub body: Vec, pub redirected_from: Vec, } impl Response { /// Decode the body as text, honoring the response charset. /// /// Uses the HTTP `Content-Type` header's `charset=` parameter, then for /// HTML responses falls back to sniffing `` in the first /// 1KB, then UTF-8. Mirrors browser behaviour per the HTML5 spec. pub fn text(&self) -> String { if self.is_html() { crate::encoding::decode_response(&self.body, self.content_type()) } else { crate::encoding::decode_non_html(&self.body, self.content_type()) } } pub fn header(&self, name: &str) -> Option<&str> { self.headers.get(&name.to_lowercase()).map(|s| s.as_str()) } pub fn content_type(&self) -> Option<&str> { self.header("content-type") } pub fn is_html(&self) -> bool { self.content_type() .map(|ct| ct.contains("text/html")) .unwrap_or(false) } } #[derive(Debug, Clone)] pub struct RequestInfo { pub url: Url, pub method: String, pub headers: HashMap, pub resource_type: ResourceType, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum ResourceType { Document, Script, Stylesheet, Image, Font, Xhr, Fetch, Other, } pub type RequestCallback = Arc; pub type ResponseCallback = Arc; fn validate_url(url: &Url) -> Result<(), ObscuraNetError> { let allow_private_network = std::env::var_os("OBSCURA_ALLOW_PRIVATE_NETWORK").is_some(); let scheme = url.scheme(); if scheme != "http" && scheme != "https" && scheme != "file" { return Err(ObscuraNetError::Network(format!( "Forbidden URL scheme '{}' - only http, https, and file are allowed", scheme ))); } if scheme == "file" || allow_private_network { return Ok(()); } if let Some(host) = url.host() { match host { url::Host::Ipv4(ip) => { if ip.is_loopback() || ip.is_private() || ip.is_link_local() || ip.is_broadcast() || ip.is_documentation() { return Err(ObscuraNetError::Network(format!( "Access to private/internal IP address {} is not allowed", ip ))); } } url::Host::Ipv6(ip) => { if ip.is_loopback() || ip.is_unicast_link_local() { return Err(ObscuraNetError::Network(format!( "Access to private/internal IPv6 address {} is not allowed", ip ))); } } url::Host::Domain(domain) => { let lower_domain = domain.to_lowercase(); if lower_domain == "localhost" || lower_domain.ends_with(".localhost") || lower_domain == "127.0.0.1" || lower_domain == "::1" { return Err(ObscuraNetError::Network(format!( "Access to localhost domain '{}' is not allowed", domain ))); } } } } Ok(()) } async fn fetch_file_url(url: &Url) -> Result { let path = url .to_file_path() .map_err(|_| ObscuraNetError::Network("Invalid file URL".to_string()))?; let body = tokio::fs::read(&path) .await .map_err(|e| ObscuraNetError::Network(format!("Failed to read file: {}", e)))?; let mut headers = HashMap::new(); if let Some(ext) = path.extension().and_then(|e| e.to_str()) { let ct = match ext.to_lowercase().as_str() { "html" | "htm" => "text/html", "css" => "text/css", "js" | "mjs" => "application/javascript", "json" => "application/json", "png" => "image/png", "jpg" | "jpeg" => "image/jpeg", "gif" => "image/gif", "svg" => "image/svg+xml", "webp" => "image/webp", "ico" => "image/x-icon", _ => "application/octet-stream", }; headers.insert("content-type".to_string(), ct.to_string()); } Ok(Response { url: url.clone(), status: 200, headers, body, redirected_from: Vec::new(), }) } pub struct ObscuraHttpClient { client: tokio::sync::OnceCell, proxy_url: Option, pub cookie_jar: Arc, pub user_agent: RwLock, pub extra_headers: RwLock>, pub interceptor: RwLock>>, pub on_request: RwLock>, pub on_response: RwLock>, pub timeout: Duration, pub in_flight: Arc, pub block_trackers: bool, } impl ObscuraHttpClient { pub fn new() -> Self { Self::with_cookie_jar(Arc::new(CookieJar::new())) } pub fn with_cookie_jar(cookie_jar: Arc) -> Self { Self::with_options(cookie_jar, None) } pub fn with_options(cookie_jar: Arc, proxy_url: Option<&str>) -> Self { ObscuraHttpClient { client: tokio::sync::OnceCell::new(), proxy_url: proxy_url.map(|s| s.to_string()), cookie_jar, user_agent: RwLock::new( "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36".to_string(), ), extra_headers: RwLock::new(HashMap::new()), interceptor: RwLock::new(None), on_request: RwLock::new(Vec::new()), on_response: RwLock::new(Vec::new()), in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)), timeout: Duration::from_secs(30), block_trackers: false, } } async fn get_client(&self) -> &Client { self.client.get_or_init(|| async { let mut builder = Client::builder() .redirect(Policy::none()) .timeout(Duration::from_secs(30)) .danger_accept_invalid_certs(false) ; if let Some(ref proxy) = self.proxy_url { if let Ok(p) = reqwest::Proxy::all(proxy.as_str()) { builder = builder.proxy(p); } } builder.build().expect("failed to build HTTP client") }).await } /// Read-only accessor for the proxy URL the client was configured with /// (if any). Exposed so callers outside the `obscura-net` crate — notably /// `op_fetch_url` in `obscura-js` (#139) — can route their own reqwest /// requests through the same upstream proxy. pub fn proxy_url(&self) -> Option<&str> { self.proxy_url.as_deref() } pub async fn fetch(&self, url: &Url) -> Result { self.fetch_with_method(Method::GET, url, None).await } pub async fn post_form(&self, url: &Url, body: &str) -> Result { self.fetch_with_method(Method::POST, url, Some(body.as_bytes().to_vec())).await } pub async fn fetch_with_method( &self, initial_method: Method, url: &Url, initial_body: Option>, ) -> Result { validate_url(url)?; if url.scheme() == "file" { return fetch_file_url(url).await; } let mut method = initial_method; let mut body = initial_body; if self.block_trackers { if let Some(host) = url.host_str() { if crate::blocklist::is_blocked(host) { tracing::debug!("Blocked tracker: {}", url); return Ok(Response { status: 0, url: url.clone(), headers: HashMap::new(), body: Vec::new(), redirected_from: Vec::new(), }); } } } let mut current_url = url.clone(); let mut redirects = Vec::new(); let max_redirects = 20; for _redirect_count in 0..max_redirects { let request_info = RequestInfo { url: current_url.clone(), method: method.to_string(), headers: self.extra_headers.read().await.clone(), resource_type: ResourceType::Document, }; if let Some(interceptor) = self.interceptor.read().await.as_ref() { match interceptor.intercept(&request_info).await { InterceptAction::Continue => {} InterceptAction::Block => { return Err(ObscuraNetError::Blocked(current_url.to_string())); } InterceptAction::Fulfill(response) => { return Ok(response); } InterceptAction::ModifyHeaders(headers) => { let mut extra = self.extra_headers.write().await; extra.extend(headers); } } } for cb in self.on_request.read().await.iter() { cb(&request_info); } let ua = self.user_agent.read().await.clone(); let mut headers = HeaderMap::new(); headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap_or_else(|_| { HeaderValue::from_static("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36") })); headers.insert( reqwest::header::ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"), ); headers.insert( reqwest::header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"), ); headers.insert( HeaderName::from_static("sec-ch-ua"), HeaderValue::from_static("\"Chromium\";v=\"145\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"145\""), ); headers.insert( HeaderName::from_static("sec-ch-ua-mobile"), HeaderValue::from_static("?0"), ); headers.insert( HeaderName::from_static("sec-ch-ua-platform"), HeaderValue::from_static("\"Linux\""), ); headers.insert( HeaderName::from_static("sec-fetch-dest"), HeaderValue::from_static("document"), ); headers.insert( HeaderName::from_static("sec-fetch-mode"), HeaderValue::from_static("navigate"), ); headers.insert( HeaderName::from_static("sec-fetch-site"), HeaderValue::from_static("none"), ); headers.insert( HeaderName::from_static("sec-fetch-user"), HeaderValue::from_static("?1"), ); headers.insert( HeaderName::from_static("upgrade-insecure-requests"), HeaderValue::from_static("1"), ); let cookie_header = self.cookie_jar.get_cookie_header(¤t_url); tracing::debug!( "Cookie header for {}: {} cookies ({} bytes)", current_url.host_str().unwrap_or("?"), cookie_header.split("; ").filter(|s| !s.is_empty()).count(), cookie_header.len(), ); if !cookie_header.is_empty() { match HeaderValue::from_str(&cookie_header) { Ok(val) => { headers.insert(reqwest::header::COOKIE, val); } Err(_) => { let filtered: String = cookie_header .split("; ") .filter(|pair| HeaderValue::from_str(pair).is_ok()) .collect::>() .join("; "); if !filtered.is_empty() { if let Ok(val) = HeaderValue::from_str(&filtered) { headers.insert(reqwest::header::COOKIE, val); } } tracing::debug!( "Cookie header invalid chars, filtered {} -> {} bytes", cookie_header.len(), filtered.len(), ); } } } for (k, v) in self.extra_headers.read().await.iter() { if let (Ok(name), Ok(val)) = ( HeaderName::from_bytes(k.as_bytes()), HeaderValue::from_str(v), ) { headers.insert(name, val); } } let mut req_builder = self.get_client().await.request(method.clone(), current_url.as_str()) .headers(headers); if let Some(ref b) = body { if method == Method::POST { req_builder = req_builder.header( reqwest::header::CONTENT_TYPE, "application/x-www-form-urlencoded", ); } req_builder = req_builder.body(b.clone()); } self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let resp = req_builder.send().await.map_err(|e| { self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); ObscuraNetError::Network(format!("{}: {}", current_url, e)) })?; self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); let status = resp.status(); for val in resp.headers().get_all(reqwest::header::SET_COOKIE) { if let Ok(s) = val.to_str() { self.cookie_jar.set_cookie(s, ¤t_url); } } let response_headers: HashMap = resp .headers() .iter() .map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string())) .collect(); if status.is_redirection() { if let Some(location) = resp.headers().get(reqwest::header::LOCATION) { let location_str = location.to_str().map_err(|_| { ObscuraNetError::Network("Invalid redirect Location header".into()) })?; let next_url = current_url.join(location_str).map_err(|e| { ObscuraNetError::Network(format!("Invalid redirect URL: {}", e)) })?; validate_url(&next_url)?; redirects.push(current_url.clone()); current_url = next_url; if status == reqwest::StatusCode::MOVED_PERMANENTLY || status == reqwest::StatusCode::FOUND || status == reqwest::StatusCode::SEE_OTHER { method = Method::GET; body = None; } continue; } } let body_bytes = resp.bytes().await.map_err(|e| { ObscuraNetError::Network(format!("Failed to read body: {}", e)) })?.to_vec(); let response = Response { url: current_url, status: status.as_u16(), headers: response_headers, body: body_bytes, redirected_from: redirects, }; for cb in self.on_response.read().await.iter() { cb(&request_info, &response); } return Ok(response); } Err(ObscuraNetError::TooManyRedirects(current_url.to_string())) } pub async fn set_user_agent(&self, ua: &str) { *self.user_agent.write().await = ua.to_string(); } pub async fn set_extra_headers(&self, headers: HashMap) { *self.extra_headers.write().await = headers; } pub fn active_requests(&self) -> u32 { self.in_flight.load(std::sync::atomic::Ordering::Relaxed) } pub fn is_network_idle(&self) -> bool { self.active_requests() == 0 } } impl Default for ObscuraHttpClient { fn default() -> Self { Self::new() } } #[derive(Debug, thiserror::Error)] pub enum ObscuraNetError { #[error("Network error: {0}")] Network(String), #[error("Too many redirects: {0}")] TooManyRedirects(String), #[error("Request blocked: {0}")] Blocked(String), }