SGavrl
768cbc056b
The default HTTP path emitted headers with user-agent first and sec-fetch-dest before sec-fetch-site, an order no real browser produces and a known fingerprint signal. Reorder the explicit headers to Chrome's top-level navigation order (sec-ch-ua family, upgrade-insecure-requests, user-agent, accept, sec-fetch-site/mode/user/dest, accept-language). reqwest still appends accept-encoding and host after these, so accept-encoding lands after accept-language instead of before it; the rest matches Chrome exactly.
727 lines
27 KiB
Rust
727 lines
27 KiB
Rust
use std::collections::HashMap;
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
|
|
use std::net::{IpAddr, SocketAddr};
|
|
|
|
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
|
|
use reqwest::header::{HeaderMap, HeaderName, HeaderValue, USER_AGENT};
|
|
use reqwest::redirect::Policy;
|
|
use reqwest::{Client, Method};
|
|
use tokio::sync::RwLock;
|
|
use url::Url;
|
|
|
|
use crate::cookies::CookieJar;
|
|
use crate::interceptor::{InterceptAction, RequestInterceptor};
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct Response {
|
|
pub url: Url,
|
|
pub status: u16,
|
|
pub headers: HashMap<String, String>,
|
|
pub body: Vec<u8>,
|
|
pub redirected_from: Vec<Url>,
|
|
}
|
|
|
|
impl Response {
|
|
/// Decode the body as text, honoring the response charset.
|
|
///
|
|
/// Uses the HTTP `Content-Type` header's `charset=` parameter, then for
|
|
/// HTML responses falls back to sniffing `<meta charset>` in the first
|
|
/// 1KB, then UTF-8. Mirrors browser behaviour per the HTML5 spec.
|
|
pub fn text(&self) -> String {
|
|
if self.is_html() {
|
|
crate::encoding::decode_response(&self.body, self.content_type())
|
|
} else {
|
|
crate::encoding::decode_non_html(&self.body, self.content_type())
|
|
}
|
|
}
|
|
|
|
pub fn header(&self, name: &str) -> Option<&str> {
|
|
self.headers.get(&name.to_lowercase()).map(|s| s.as_str())
|
|
}
|
|
|
|
pub fn content_type(&self) -> Option<&str> {
|
|
self.header("content-type")
|
|
}
|
|
|
|
pub fn is_html(&self) -> bool {
|
|
self.content_type()
|
|
.map(|ct| ct.contains("text/html"))
|
|
.unwrap_or(false)
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct RequestInfo {
|
|
pub url: Url,
|
|
pub method: String,
|
|
pub headers: HashMap<String, String>,
|
|
pub resource_type: ResourceType,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum ResourceType {
|
|
Document,
|
|
Script,
|
|
Stylesheet,
|
|
Image,
|
|
Font,
|
|
Xhr,
|
|
Fetch,
|
|
Other,
|
|
}
|
|
|
|
pub type RequestCallback = Arc<dyn Fn(&RequestInfo) + Send + Sync>;
|
|
pub type ResponseCallback = Arc<dyn Fn(&RequestInfo, &Response) + Send + Sync>;
|
|
|
|
/// Process-wide opt-in via env var. Older flow that issue #4 introduced. The
|
|
/// new `--allow-private-network` CLI flag (issue #33) sets a per-client field
|
|
/// that is OR'd with this so existing scripts and Docker setups that pin the
|
|
/// env var keep working unchanged.
|
|
pub fn env_allows_private_network() -> bool {
|
|
matches!(
|
|
std::env::var("OBSCURA_ALLOW_PRIVATE_NETWORK")
|
|
.ok()
|
|
.as_deref()
|
|
.map(str::trim)
|
|
.map(str::to_ascii_lowercase)
|
|
.as_deref(),
|
|
Some("1") | Some("true") | Some("yes") | Some("on")
|
|
)
|
|
}
|
|
|
|
/// True when `ip` must never be the target of an outbound request from the
|
|
/// engine: loopback, RFC1918 private, link-local (incl. the 169.254.169.254
|
|
/// cloud-metadata endpoint), broadcast, documentation, the unspecified address
|
|
/// (0.0.0.0 / ::, which the OS routes to localhost), IPv6 unique-local
|
|
/// (fc00::/7), and any IPv4-mapped/compatible IPv6 form of the above.
|
|
/// Centralizes the SSRF deny-set so the literal-host check and the
|
|
/// DNS-resolution check (`SsrfGuardResolver`) can never disagree.
|
|
pub fn is_forbidden_ip(ip: IpAddr) -> bool {
|
|
match ip {
|
|
IpAddr::V4(v4) => {
|
|
v4.is_loopback()
|
|
|| v4.is_private()
|
|
|| v4.is_link_local()
|
|
|| v4.is_broadcast()
|
|
|| v4.is_documentation()
|
|
|| v4.is_unspecified()
|
|
}
|
|
IpAddr::V6(v6) => {
|
|
if v6.is_loopback()
|
|
|| v6.is_unspecified()
|
|
|| v6.is_unique_local()
|
|
|| v6.is_unicast_link_local()
|
|
{
|
|
return true;
|
|
}
|
|
// Unwrap IPv4-mapped (::ffff:a.b.c.d) and IPv4-compatible (::a.b.c.d)
|
|
// forms and re-check the embedded v4 so e.g. [::ffff:127.0.0.1] or
|
|
// [::ffff:169.254.169.254] cannot slip past the v6 arm.
|
|
if let Some(v4) = v6.to_ipv4_mapped().or_else(|| v6.to_ipv4()) {
|
|
return is_forbidden_ip(IpAddr::V4(v4));
|
|
}
|
|
false
|
|
}
|
|
}
|
|
}
|
|
|
|
/// reqwest DNS resolver that performs the lookup and then rejects the whole
|
|
/// request if ANY resolved address is in the SSRF deny-set. This closes the
|
|
/// DNS-rebinding bypass a host-string check alone cannot: a public name that
|
|
/// resolves to 127.0.0.1 / 169.254.169.254 / an RFC1918 address is blocked at
|
|
/// connect time, using the very addresses reqwest will dial. When private
|
|
/// access is permitted (`--allow-private-network` or
|
|
/// `OBSCURA_ALLOW_PRIVATE_NETWORK`) the lookup passes through unfiltered.
|
|
pub struct SsrfGuardResolver {
|
|
allow_private: bool,
|
|
}
|
|
|
|
impl SsrfGuardResolver {
|
|
pub fn new(allow_private: bool) -> Self {
|
|
Self { allow_private }
|
|
}
|
|
}
|
|
|
|
impl Resolve for SsrfGuardResolver {
|
|
fn resolve(&self, name: Name) -> Resolving {
|
|
let allow = self.allow_private || env_allows_private_network();
|
|
let host = name.as_str().to_string();
|
|
Box::pin(async move {
|
|
let addrs: Vec<SocketAddr> = tokio::net::lookup_host((host.as_str(), 0))
|
|
.await
|
|
.map_err(|e| -> Box<dyn std::error::Error + Send + Sync> { Box::new(e) })?
|
|
.collect();
|
|
if !allow {
|
|
if let Some(bad) = addrs.iter().find(|sa| is_forbidden_ip(sa.ip())) {
|
|
return Err(format!(
|
|
"SSRF blocked: '{}' resolves to forbidden address {}",
|
|
host,
|
|
bad.ip()
|
|
)
|
|
.into());
|
|
}
|
|
}
|
|
let iter: Addrs = Box::new(addrs.into_iter());
|
|
Ok(iter)
|
|
})
|
|
}
|
|
}
|
|
|
|
fn validate_url(url: &Url, allow_private_network: bool) -> Result<(), ObscuraNetError> {
|
|
let allow_private_network = allow_private_network || env_allows_private_network();
|
|
let scheme = url.scheme();
|
|
if scheme != "http" && scheme != "https" && scheme != "file" {
|
|
return Err(ObscuraNetError::Network(format!(
|
|
"Forbidden URL scheme '{}' - only http, https, and file are allowed",
|
|
scheme
|
|
)));
|
|
}
|
|
|
|
if scheme == "file" || allow_private_network {
|
|
return Ok(());
|
|
}
|
|
|
|
if let Some(host) = url.host() {
|
|
match host {
|
|
url::Host::Ipv4(ip) => {
|
|
if is_forbidden_ip(IpAddr::V4(ip)) {
|
|
return Err(ObscuraNetError::Network(format!(
|
|
"Access to private/internal IP address {} is not allowed",
|
|
ip
|
|
)));
|
|
}
|
|
}
|
|
url::Host::Ipv6(ip) => {
|
|
if is_forbidden_ip(IpAddr::V6(ip)) {
|
|
return Err(ObscuraNetError::Network(format!(
|
|
"Access to private/internal IPv6 address {} is not allowed",
|
|
ip
|
|
)));
|
|
}
|
|
}
|
|
url::Host::Domain(domain) => {
|
|
let lower_domain = domain.to_lowercase();
|
|
if lower_domain == "localhost"
|
|
|| lower_domain.ends_with(".localhost")
|
|
|| lower_domain == "127.0.0.1"
|
|
|| lower_domain == "::1"
|
|
{
|
|
return Err(ObscuraNetError::Network(format!(
|
|
"Access to localhost domain '{}' is not allowed",
|
|
domain
|
|
)));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
async fn fetch_file_url(url: &Url) -> Result<Response, ObscuraNetError> {
|
|
let path = url
|
|
.to_file_path()
|
|
.map_err(|_| ObscuraNetError::Network("Invalid file URL".to_string()))?;
|
|
let body = tokio::fs::read(&path)
|
|
.await
|
|
.map_err(|e| ObscuraNetError::Network(format!("Failed to read file: {}", e)))?;
|
|
|
|
let mut headers = HashMap::new();
|
|
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
|
|
let ct = match ext.to_lowercase().as_str() {
|
|
"html" | "htm" => "text/html",
|
|
"css" => "text/css",
|
|
"js" | "mjs" => "application/javascript",
|
|
"json" => "application/json",
|
|
"png" => "image/png",
|
|
"jpg" | "jpeg" => "image/jpeg",
|
|
"gif" => "image/gif",
|
|
"svg" => "image/svg+xml",
|
|
"webp" => "image/webp",
|
|
"ico" => "image/x-icon",
|
|
_ => "application/octet-stream",
|
|
};
|
|
headers.insert("content-type".to_string(), ct.to_string());
|
|
}
|
|
|
|
Ok(Response {
|
|
url: url.clone(),
|
|
status: 200,
|
|
headers,
|
|
body,
|
|
redirected_from: Vec::new(),
|
|
})
|
|
}
|
|
|
|
pub struct ObscuraHttpClient {
|
|
client: tokio::sync::OnceCell<Client>,
|
|
proxy_url: Option<String>,
|
|
pub cookie_jar: Arc<CookieJar>,
|
|
pub user_agent: RwLock<String>,
|
|
pub extra_headers: RwLock<HashMap<String, String>>,
|
|
pub interceptor: RwLock<Option<Box<dyn RequestInterceptor + Send + Sync>>>,
|
|
pub on_request: RwLock<Vec<RequestCallback>>,
|
|
pub on_response: RwLock<Vec<ResponseCallback>>,
|
|
pub timeout: Duration,
|
|
pub in_flight: Arc<std::sync::atomic::AtomicU32>,
|
|
pub block_trackers: bool,
|
|
/// When true, `validate_url` lets localhost / RFC1918 / link-local addresses
|
|
/// through in addition to the `OBSCURA_ALLOW_PRIVATE_NETWORK` env var.
|
|
/// Set via `--allow-private-network` on the CLI (issue #33).
|
|
pub allow_private_network: bool,
|
|
}
|
|
|
|
/// Derive the sec-ch-ua and sec-ch-ua-platform client-hint header values from a
|
|
/// User-Agent string, using Chromium's per-major-version GREASE algorithm so
|
|
/// the non-stealth HTTP path agrees with navigator.userAgentData instead of
|
|
/// shipping a fixed Linux/Chrome-145 hint that contradicts a Windows profile.
|
|
fn chrome_client_hints(ua: &str) -> (String, String) {
|
|
let major: usize = ua
|
|
.split("Chrome/")
|
|
.nth(1)
|
|
.and_then(|s| s.split('.').next())
|
|
.and_then(|s| s.parse::<usize>().ok())
|
|
.unwrap_or(145);
|
|
const GREASE_CHARS: [char; 11] = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_'];
|
|
const GREASE_VER: [&str; 3] = ["8", "99", "24"];
|
|
const PERMS: [[usize; 3]; 6] = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]];
|
|
let grease_brand = format!(
|
|
"Not{}A{}Brand",
|
|
GREASE_CHARS[major % 11],
|
|
GREASE_CHARS[(major + 1) % 11]
|
|
);
|
|
let brands = [
|
|
(grease_brand, GREASE_VER[major % 3].to_string()),
|
|
("Chromium".to_string(), major.to_string()),
|
|
("Google Chrome".to_string(), major.to_string()),
|
|
];
|
|
let p = PERMS[major % 6];
|
|
let sec_ch_ua = p
|
|
.iter()
|
|
.map(|&i| format!("\"{}\";v=\"{}\"", brands[i].0, brands[i].1))
|
|
.collect::<Vec<_>>()
|
|
.join(", ");
|
|
let platform = if ua.contains("Windows NT") {
|
|
"\"Windows\""
|
|
} else if ua.contains("Macintosh") {
|
|
"\"macOS\""
|
|
} else {
|
|
"\"Linux\""
|
|
};
|
|
(sec_ch_ua, platform.to_string())
|
|
}
|
|
|
|
impl ObscuraHttpClient {
|
|
pub fn new() -> Self {
|
|
Self::with_cookie_jar(Arc::new(CookieJar::new()))
|
|
}
|
|
|
|
pub fn with_cookie_jar(cookie_jar: Arc<CookieJar>) -> Self {
|
|
Self::with_options(cookie_jar, None)
|
|
}
|
|
|
|
pub fn with_options(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
|
|
Self::with_full_options(cookie_jar, proxy_url, false)
|
|
}
|
|
|
|
pub fn with_full_options(
|
|
cookie_jar: Arc<CookieJar>,
|
|
proxy_url: Option<&str>,
|
|
allow_private_network: bool,
|
|
) -> Self {
|
|
ObscuraHttpClient {
|
|
client: tokio::sync::OnceCell::new(),
|
|
proxy_url: proxy_url.map(|s| s.to_string()),
|
|
cookie_jar,
|
|
user_agent: RwLock::new(
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36".to_string(),
|
|
),
|
|
extra_headers: RwLock::new(HashMap::new()),
|
|
interceptor: RwLock::new(None),
|
|
on_request: RwLock::new(Vec::new()),
|
|
on_response: RwLock::new(Vec::new()),
|
|
in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),
|
|
timeout: Duration::from_secs(30),
|
|
block_trackers: false,
|
|
allow_private_network,
|
|
}
|
|
}
|
|
|
|
async fn get_client(&self) -> &Client {
|
|
self.client.get_or_init(|| async {
|
|
let mut builder = Client::builder()
|
|
.redirect(Policy::none())
|
|
.timeout(Duration::from_secs(30))
|
|
.danger_accept_invalid_certs(false)
|
|
// SSRF guard: reject hostnames that resolve to a private/loopback IP.
|
|
.dns_resolver(Arc::new(SsrfGuardResolver::new(self.allow_private_network)))
|
|
;
|
|
|
|
if let Some(ref proxy) = self.proxy_url {
|
|
if let Ok(p) = reqwest::Proxy::all(proxy.as_str()) {
|
|
builder = builder.proxy(p);
|
|
}
|
|
}
|
|
|
|
builder.build().expect("failed to build HTTP client")
|
|
}).await
|
|
}
|
|
|
|
/// Read-only accessor for the proxy URL the client was configured with
|
|
/// (if any). Exposed so callers outside the `obscura-net` crate — notably
|
|
/// `op_fetch_url` in `obscura-js` (#139) — can route their own reqwest
|
|
/// requests through the same upstream proxy.
|
|
pub fn proxy_url(&self) -> Option<&str> {
|
|
self.proxy_url.as_deref()
|
|
}
|
|
|
|
pub async fn fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
|
self.fetch_with_method(Method::GET, url, None).await
|
|
}
|
|
|
|
pub async fn post_form(&self, url: &Url, body: &str) -> Result<Response, ObscuraNetError> {
|
|
self.fetch_with_method(Method::POST, url, Some(body.as_bytes().to_vec())).await
|
|
}
|
|
|
|
pub async fn fetch_with_method(
|
|
&self,
|
|
initial_method: Method,
|
|
url: &Url,
|
|
initial_body: Option<Vec<u8>>,
|
|
) -> Result<Response, ObscuraNetError> {
|
|
validate_url(url, self.allow_private_network)?;
|
|
|
|
if url.scheme() == "file" {
|
|
return fetch_file_url(url).await;
|
|
}
|
|
|
|
let mut method = initial_method;
|
|
let mut body = initial_body;
|
|
if self.block_trackers {
|
|
if let Some(host) = url.host_str() {
|
|
if crate::blocklist::is_blocked(host) {
|
|
tracing::debug!("Blocked tracker: {}", url);
|
|
return Ok(Response {
|
|
status: 0,
|
|
url: url.clone(),
|
|
headers: HashMap::new(),
|
|
body: Vec::new(),
|
|
redirected_from: Vec::new(),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
let mut current_url = url.clone();
|
|
let mut redirects = Vec::new();
|
|
let max_redirects = 20;
|
|
|
|
for _redirect_count in 0..max_redirects {
|
|
let request_info = RequestInfo {
|
|
url: current_url.clone(),
|
|
method: method.to_string(),
|
|
headers: self.extra_headers.read().await.clone(),
|
|
resource_type: ResourceType::Document,
|
|
};
|
|
|
|
if let Some(interceptor) = self.interceptor.read().await.as_ref() {
|
|
match interceptor.intercept(&request_info).await {
|
|
InterceptAction::Continue => {}
|
|
InterceptAction::Block => {
|
|
return Err(ObscuraNetError::Blocked(current_url.to_string()));
|
|
}
|
|
InterceptAction::Fulfill(response) => {
|
|
return Ok(response);
|
|
}
|
|
InterceptAction::ModifyHeaders(headers) => {
|
|
let mut extra = self.extra_headers.write().await;
|
|
extra.extend(headers);
|
|
}
|
|
}
|
|
}
|
|
|
|
for cb in self.on_request.read().await.iter() {
|
|
cb(&request_info);
|
|
}
|
|
|
|
let ua = self.user_agent.read().await.clone();
|
|
let (sec_ch_ua, sec_ch_ua_platform) = chrome_client_hints(&ua);
|
|
let mut headers = HeaderMap::new();
|
|
// Chrome's top-level navigation header order. (reqwest appends
|
|
// accept-encoding/host after these, so accept-encoding lands after
|
|
// accept-language rather than before it; the rest matches Chrome.)
|
|
headers.insert(
|
|
HeaderName::from_static("sec-ch-ua"),
|
|
HeaderValue::from_str(&sec_ch_ua)
|
|
.unwrap_or_else(|_| HeaderValue::from_static("\"Not:A-Brand\";v=\"99\", \"Google Chrome\";v=\"145\", \"Chromium\";v=\"145\"")),
|
|
);
|
|
headers.insert(HeaderName::from_static("sec-ch-ua-mobile"), HeaderValue::from_static("?0"));
|
|
headers.insert(
|
|
HeaderName::from_static("sec-ch-ua-platform"),
|
|
HeaderValue::from_str(&sec_ch_ua_platform)
|
|
.unwrap_or_else(|_| HeaderValue::from_static("\"Windows\"")),
|
|
);
|
|
headers.insert(HeaderName::from_static("upgrade-insecure-requests"), HeaderValue::from_static("1"));
|
|
headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap_or_else(|_| {
|
|
HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36")
|
|
}));
|
|
headers.insert(
|
|
reqwest::header::ACCEPT,
|
|
HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"),
|
|
);
|
|
headers.insert(HeaderName::from_static("sec-fetch-site"), HeaderValue::from_static("none"));
|
|
headers.insert(HeaderName::from_static("sec-fetch-mode"), HeaderValue::from_static("navigate"));
|
|
headers.insert(HeaderName::from_static("sec-fetch-user"), HeaderValue::from_static("?1"));
|
|
headers.insert(HeaderName::from_static("sec-fetch-dest"), HeaderValue::from_static("document"));
|
|
headers.insert(
|
|
reqwest::header::ACCEPT_LANGUAGE,
|
|
HeaderValue::from_static("en-US,en;q=0.9"),
|
|
);
|
|
|
|
let cookie_header = self.cookie_jar.get_cookie_header(¤t_url);
|
|
tracing::debug!(
|
|
"Cookie header for {}: {} cookies ({} bytes)",
|
|
current_url.host_str().unwrap_or("?"),
|
|
cookie_header.split("; ").filter(|s| !s.is_empty()).count(),
|
|
cookie_header.len(),
|
|
);
|
|
if !cookie_header.is_empty() {
|
|
match HeaderValue::from_str(&cookie_header) {
|
|
Ok(val) => {
|
|
headers.insert(reqwest::header::COOKIE, val);
|
|
}
|
|
Err(_) => {
|
|
let filtered: String = cookie_header
|
|
.split("; ")
|
|
.filter(|pair| HeaderValue::from_str(pair).is_ok())
|
|
.collect::<Vec<_>>()
|
|
.join("; ");
|
|
if !filtered.is_empty() {
|
|
if let Ok(val) = HeaderValue::from_str(&filtered) {
|
|
headers.insert(reqwest::header::COOKIE, val);
|
|
}
|
|
}
|
|
tracing::debug!(
|
|
"Cookie header invalid chars, filtered {} -> {} bytes",
|
|
cookie_header.len(), filtered.len(),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
for (k, v) in self.extra_headers.read().await.iter() {
|
|
if let (Ok(name), Ok(val)) = (
|
|
HeaderName::from_bytes(k.as_bytes()),
|
|
HeaderValue::from_str(v),
|
|
) {
|
|
headers.insert(name, val);
|
|
}
|
|
}
|
|
|
|
let mut req_builder = self.get_client().await.request(method.clone(), current_url.as_str())
|
|
.headers(headers);
|
|
|
|
if let Some(ref b) = body {
|
|
if method == Method::POST {
|
|
req_builder = req_builder.header(
|
|
reqwest::header::CONTENT_TYPE,
|
|
"application/x-www-form-urlencoded",
|
|
);
|
|
}
|
|
req_builder = req_builder.body(b.clone());
|
|
}
|
|
|
|
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
|
let resp = req_builder.send().await.map_err(|e| {
|
|
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
|
ObscuraNetError::Network(format!("{}: {}", current_url, e))
|
|
})?;
|
|
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
|
|
|
let status = resp.status();
|
|
|
|
for val in resp.headers().get_all(reqwest::header::SET_COOKIE) {
|
|
if let Ok(s) = val.to_str() {
|
|
self.cookie_jar.set_cookie(s, ¤t_url);
|
|
}
|
|
}
|
|
|
|
let response_headers: HashMap<String, String> = resp
|
|
.headers()
|
|
.iter()
|
|
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
|
.collect();
|
|
|
|
if status.is_redirection() {
|
|
if let Some(location) = resp.headers().get(reqwest::header::LOCATION) {
|
|
let location_str = location.to_str().map_err(|_| {
|
|
ObscuraNetError::Network("Invalid redirect Location header".into())
|
|
})?;
|
|
let next_url = current_url.join(location_str).map_err(|e| {
|
|
ObscuraNetError::Network(format!("Invalid redirect URL: {}", e))
|
|
})?;
|
|
validate_url(&next_url, self.allow_private_network)?;
|
|
redirects.push(current_url.clone());
|
|
current_url = next_url;
|
|
if status == reqwest::StatusCode::MOVED_PERMANENTLY
|
|
|| status == reqwest::StatusCode::FOUND
|
|
|| status == reqwest::StatusCode::SEE_OTHER
|
|
{
|
|
method = Method::GET;
|
|
body = None;
|
|
}
|
|
continue;
|
|
}
|
|
}
|
|
|
|
let body_bytes = resp.bytes().await.map_err(|e| {
|
|
ObscuraNetError::Network(format!("Failed to read body: {}", e))
|
|
})?.to_vec();
|
|
|
|
let response = Response {
|
|
url: current_url,
|
|
status: status.as_u16(),
|
|
headers: response_headers,
|
|
body: body_bytes,
|
|
redirected_from: redirects,
|
|
};
|
|
|
|
for cb in self.on_response.read().await.iter() {
|
|
cb(&request_info, &response);
|
|
}
|
|
|
|
return Ok(response);
|
|
}
|
|
|
|
Err(ObscuraNetError::TooManyRedirects(current_url.to_string()))
|
|
}
|
|
|
|
pub async fn set_user_agent(&self, ua: &str) {
|
|
*self.user_agent.write().await = ua.to_string();
|
|
}
|
|
|
|
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
|
|
*self.extra_headers.write().await = headers;
|
|
}
|
|
|
|
pub fn active_requests(&self) -> u32 {
|
|
self.in_flight.load(std::sync::atomic::Ordering::Relaxed)
|
|
}
|
|
|
|
pub fn is_network_idle(&self) -> bool {
|
|
self.active_requests() == 0
|
|
}
|
|
}
|
|
|
|
impl Default for ObscuraHttpClient {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum ObscuraNetError {
|
|
#[error("Network error: {0}")]
|
|
Network(String),
|
|
|
|
#[error("Too many redirects: {0}")]
|
|
TooManyRedirects(String),
|
|
|
|
#[error("Request blocked: {0}")]
|
|
Blocked(String),
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod ssrf_tests {
|
|
use super::{is_forbidden_ip, validate_url, SsrfGuardResolver};
|
|
use reqwest::dns::{Name, Resolve};
|
|
use std::net::IpAddr;
|
|
use std::str::FromStr;
|
|
use url::Url;
|
|
|
|
fn ip(s: &str) -> IpAddr {
|
|
IpAddr::from_str(s).unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn ipv4_private_and_special_ranges_are_forbidden() {
|
|
for s in [
|
|
"127.0.0.1",
|
|
"127.5.6.7",
|
|
"10.0.0.1",
|
|
"172.16.0.1",
|
|
"192.168.1.1",
|
|
"169.254.169.254", // cloud-metadata endpoint
|
|
"0.0.0.0", // unspecified -> localhost (was a bypass)
|
|
"255.255.255.255", // broadcast
|
|
"192.0.2.1", // documentation
|
|
] {
|
|
assert!(is_forbidden_ip(ip(s)), "{s} should be forbidden");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn public_ipv4_is_allowed() {
|
|
for s in ["1.1.1.1", "8.8.8.8", "93.184.216.34"] {
|
|
assert!(!is_forbidden_ip(ip(s)), "{s} should be allowed");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn ipv6_loopback_ula_linklocal_and_mapped_are_forbidden() {
|
|
for s in [
|
|
"::1", // loopback
|
|
"::", // unspecified
|
|
"fc00::1", // unique-local (was a bypass)
|
|
"fd12:3456:789a::1", // unique-local
|
|
"fe80::1", // link-local
|
|
"::ffff:127.0.0.1", // v4-mapped loopback (was a bypass)
|
|
"::ffff:169.254.169.254", // v4-mapped metadata
|
|
] {
|
|
assert!(is_forbidden_ip(ip(s)), "{s} should be forbidden");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn public_ipv6_is_allowed() {
|
|
assert!(!is_forbidden_ip(ip("2606:4700:4700::1111"))); // cloudflare dns
|
|
}
|
|
|
|
#[test]
|
|
fn validate_url_blocks_unspecified_and_allows_public() {
|
|
// 0.0.0.0 previously slipped through validate_url's literal-host check.
|
|
assert!(validate_url(&Url::parse("http://0.0.0.0:8080/").unwrap(), false).is_err());
|
|
assert!(validate_url(&Url::parse("http://127.0.0.1/").unwrap(), false).is_err());
|
|
assert!(validate_url(&Url::parse("http://example.com/").unwrap(), false).is_ok());
|
|
// The allow flag bypasses the guard (local-dev escape hatch).
|
|
assert!(validate_url(&Url::parse("http://127.0.0.1/").unwrap(), true).is_ok());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn resolver_blocks_hostname_that_resolves_to_loopback() {
|
|
// localtest.me is a public DNS name that resolves to 127.0.0.1 — the
|
|
// canonical DNS-rebinding test. The guard must reject it. If DNS is
|
|
// unavailable the lookup itself errors (also Err), so the assertion
|
|
// holds either way.
|
|
let r = SsrfGuardResolver::new(false);
|
|
let res = r.resolve(Name::from_str("localtest.me").unwrap()).await;
|
|
assert!(res.is_err(), "localtest.me -> 127.0.0.1 must be blocked");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn resolver_does_not_ssrf_block_public_host() {
|
|
// A public host must not be SSRF-blocked. Tolerate a no-network sandbox
|
|
// by only failing on an actual SSRF rejection, not a lookup failure.
|
|
let r = SsrfGuardResolver::new(false);
|
|
match r.resolve(Name::from_str("example.com").unwrap()).await {
|
|
Ok(_) => {}
|
|
Err(e) => assert!(
|
|
!e.to_string().contains("SSRF blocked"),
|
|
"example.com wrongly SSRF-blocked: {e}"
|
|
),
|
|
}
|
|
}
|
|
}
|