Initial Release
This commit is contained in:
h4ckf0r0day committed
1 parent
2e5d63cc35
commit
8749725b45
52 files changed
+19804
No files matched your search
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "obscura-net"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
stealth = ["wreq", "wreq-util"]
|
||||
|
||||
[dependencies]
|
||||
reqwest = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
url = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
async-trait = "0.1"
|
||||
wreq = { version = "6.0.0-rc.28", optional = true, features = ["prefix-symbols"] }
|
||||
wreq-util = { version = "3.0.0-rc.10", optional = true }
|
||||
@@ -0,0 +1,77 @@
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const PGL_LIST: &str = include_str!("pgl_domains.txt");
|
||||
|
||||
fn blocklist() -> &'static HashSet<&'static str> {
|
||||
static BLOCKLIST: OnceLock<HashSet<&str>> = OnceLock::new();
|
||||
BLOCKLIST.get_or_init(|| {
|
||||
let mut set = HashSet::with_capacity(4000);
|
||||
for line in PGL_LIST.lines() {
|
||||
let domain = line.trim();
|
||||
if !domain.is_empty() && !domain.starts_with('#') {
|
||||
set.insert(domain);
|
||||
}
|
||||
}
|
||||
for domain in EXTRA_DOMAINS {
|
||||
set.insert(*domain);
|
||||
}
|
||||
set
|
||||
})
|
||||
}
|
||||
|
||||
pub fn is_blocked(host: &str) -> bool {
|
||||
let bl = blocklist();
|
||||
|
||||
if bl.contains(host) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let mut domain = host;
|
||||
while let Some(pos) = domain.find('.') {
|
||||
domain = &domain[pos + 1..];
|
||||
if bl.contains(domain) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
static EXTRA_DOMAINS: &[&str] = &[];
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_exact_match() {
|
||||
assert!(is_blocked("google-analytics.com"));
|
||||
assert!(is_blocked("doubleclick.net"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_subdomain_match() {
|
||||
assert!(is_blocked("www.google-analytics.com"));
|
||||
assert!(is_blocked("ssl.google-analytics.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_blocked() {
|
||||
assert!(!is_blocked("google.com"));
|
||||
assert!(!is_blocked("example.com"));
|
||||
assert!(!is_blocked("github.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pgl_domains() {
|
||||
assert!(is_blocked("adnxs.com"));
|
||||
assert!(is_blocked("criteo.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blocklist_size() {
|
||||
assert!(blocklist().len() > 3500);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
use std::collections::HashMap;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::header::{HeaderMap, HeaderName, HeaderValue, USER_AGENT};
|
||||
use reqwest::redirect::Policy;
|
||||
use reqwest::{Client, Method};
|
||||
use tokio::sync::RwLock;
|
||||
use url::Url;
|
||||
|
||||
use crate::cookies::CookieJar;
|
||||
use crate::interceptor::{InterceptAction, RequestInterceptor};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Response {
|
||||
pub url: Url,
|
||||
pub status: u16,
|
||||
pub headers: HashMap<String, String>,
|
||||
pub body: Vec<u8>,
|
||||
pub redirected_from: Vec<Url>,
|
||||
}
|
||||
|
||||
impl Response {
|
||||
pub fn text(&self) -> Result<String, std::string::FromUtf8Error> {
|
||||
String::from_utf8(self.body.clone())
|
||||
}
|
||||
|
||||
pub fn header(&self, name: &str) -> Option<&str> {
|
||||
self.headers.get(&name.to_lowercase()).map(|s| s.as_str())
|
||||
}
|
||||
|
||||
pub fn content_type(&self) -> Option<&str> {
|
||||
self.header("content-type")
|
||||
}
|
||||
|
||||
pub fn is_html(&self) -> bool {
|
||||
self.content_type()
|
||||
.map(|ct| ct.contains("text/html"))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RequestInfo {
|
||||
pub url: Url,
|
||||
pub method: String,
|
||||
pub headers: HashMap<String, String>,
|
||||
pub resource_type: ResourceType,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ResourceType {
|
||||
Document,
|
||||
Script,
|
||||
Stylesheet,
|
||||
Image,
|
||||
Font,
|
||||
Xhr,
|
||||
Fetch,
|
||||
Other,
|
||||
}
|
||||
|
||||
pub type RequestCallback = Arc<dyn Fn(&RequestInfo) + Send + Sync>;
|
||||
pub type ResponseCallback = Arc<dyn Fn(&RequestInfo, &Response) + Send + Sync>;
|
||||
|
||||
fn validate_url(url: &Url) -> Result<(), ObscuraNetError> {
|
||||
let scheme = url.scheme();
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Forbidden URL scheme '{}' - only http and https are allowed",
|
||||
scheme
|
||||
)));
|
||||
}
|
||||
|
||||
if let Some(host) = url.host() {
|
||||
match host {
|
||||
url::Host::Ipv4(ip) => {
|
||||
if ip.is_loopback()
|
||||
|| ip.is_private()
|
||||
|| ip.is_link_local()
|
||||
|| ip.is_broadcast()
|
||||
|| ip.is_documentation()
|
||||
{
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Access to private/internal IP address {} is not allowed",
|
||||
ip
|
||||
)));
|
||||
}
|
||||
}
|
||||
url::Host::Ipv6(ip) => {
|
||||
if ip.is_loopback() || ip.is_unicast_link_local() {
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Access to private/internal IPv6 address {} is not allowed",
|
||||
ip
|
||||
)));
|
||||
}
|
||||
}
|
||||
url::Host::Domain(domain) => {
|
||||
let lower_domain = domain.to_lowercase();
|
||||
if lower_domain == "localhost"
|
||||
|| lower_domain.ends_with(".localhost")
|
||||
|| lower_domain == "127.0.0.1"
|
||||
|| lower_domain == "::1"
|
||||
{
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Access to localhost domain '{}' is not allowed",
|
||||
domain
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct ObscuraHttpClient {
|
||||
client: tokio::sync::OnceCell<Client>,
|
||||
proxy_url: Option<String>,
|
||||
pub cookie_jar: Arc<CookieJar>,
|
||||
pub user_agent: RwLock<String>,
|
||||
pub extra_headers: RwLock<HashMap<String, String>>,
|
||||
pub interceptor: RwLock<Option<Box<dyn RequestInterceptor + Send + Sync>>>,
|
||||
pub on_request: RwLock<Vec<RequestCallback>>,
|
||||
pub on_response: RwLock<Vec<ResponseCallback>>,
|
||||
pub timeout: Duration,
|
||||
pub in_flight: Arc<std::sync::atomic::AtomicU32>,
|
||||
pub block_trackers: bool,
|
||||
}
|
||||
|
||||
impl ObscuraHttpClient {
|
||||
pub fn new() -> Self {
|
||||
Self::with_cookie_jar(Arc::new(CookieJar::new()))
|
||||
}
|
||||
|
||||
pub fn with_cookie_jar(cookie_jar: Arc<CookieJar>) -> Self {
|
||||
Self::with_options(cookie_jar, None)
|
||||
}
|
||||
|
||||
pub fn with_options(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
|
||||
ObscuraHttpClient {
|
||||
client: tokio::sync::OnceCell::new(),
|
||||
proxy_url: proxy_url.map(|s| s.to_string()),
|
||||
cookie_jar,
|
||||
user_agent: RwLock::new(
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36".to_string(),
|
||||
),
|
||||
extra_headers: RwLock::new(HashMap::new()),
|
||||
interceptor: RwLock::new(None),
|
||||
on_request: RwLock::new(Vec::new()),
|
||||
on_response: RwLock::new(Vec::new()),
|
||||
in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),
|
||||
timeout: Duration::from_secs(30),
|
||||
block_trackers: false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_client(&self) -> &Client {
|
||||
self.client.get_or_init(|| async {
|
||||
let mut builder = Client::builder()
|
||||
.redirect(Policy::none())
|
||||
.timeout(Duration::from_secs(30))
|
||||
.danger_accept_invalid_certs(false)
|
||||
;
|
||||
|
||||
if let Some(ref proxy) = self.proxy_url {
|
||||
if let Ok(p) = reqwest::Proxy::all(proxy.as_str()) {
|
||||
builder = builder.proxy(p);
|
||||
}
|
||||
}
|
||||
|
||||
builder.build().expect("failed to build HTTP client")
|
||||
}).await
|
||||
}
|
||||
|
||||
pub async fn fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
||||
self.fetch_with_method(Method::GET, url, None).await
|
||||
}
|
||||
|
||||
pub async fn post_form(&self, url: &Url, body: &str) -> Result<Response, ObscuraNetError> {
|
||||
self.fetch_with_method(Method::POST, url, Some(body.as_bytes().to_vec())).await
|
||||
}
|
||||
|
||||
pub async fn fetch_with_method(
|
||||
&self,
|
||||
initial_method: Method,
|
||||
url: &Url,
|
||||
initial_body: Option<Vec<u8>>,
|
||||
) -> Result<Response, ObscuraNetError> {
|
||||
validate_url(url)?;
|
||||
|
||||
let mut method = initial_method;
|
||||
let mut body = initial_body;
|
||||
if self.block_trackers {
|
||||
if let Some(host) = url.host_str() {
|
||||
if crate::blocklist::is_blocked(host) {
|
||||
tracing::debug!("Blocked tracker: {}", url);
|
||||
return Ok(Response {
|
||||
status: 0,
|
||||
url: url.clone(),
|
||||
headers: HashMap::new(),
|
||||
body: Vec::new(),
|
||||
redirected_from: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut current_url = url.clone();
|
||||
let mut redirects = Vec::new();
|
||||
let max_redirects = 20;
|
||||
|
||||
for _redirect_count in 0..max_redirects {
|
||||
let request_info = RequestInfo {
|
||||
url: current_url.clone(),
|
||||
method: method.to_string(),
|
||||
headers: self.extra_headers.read().await.clone(),
|
||||
resource_type: ResourceType::Document,
|
||||
};
|
||||
|
||||
if let Some(interceptor) = self.interceptor.read().await.as_ref() {
|
||||
match interceptor.intercept(&request_info).await {
|
||||
InterceptAction::Continue => {}
|
||||
InterceptAction::Block => {
|
||||
return Err(ObscuraNetError::Blocked(current_url.to_string()));
|
||||
}
|
||||
InterceptAction::Fulfill(response) => {
|
||||
return Ok(response);
|
||||
}
|
||||
InterceptAction::ModifyHeaders(headers) => {
|
||||
let mut extra = self.extra_headers.write().await;
|
||||
extra.extend(headers);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for cb in self.on_request.read().await.iter() {
|
||||
cb(&request_info);
|
||||
}
|
||||
|
||||
let ua = self.user_agent.read().await.clone();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap_or_else(|_| {
|
||||
HeaderValue::from_static("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36")
|
||||
}));
|
||||
headers.insert(
|
||||
reqwest::header::ACCEPT,
|
||||
HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"),
|
||||
);
|
||||
headers.insert(
|
||||
reqwest::header::ACCEPT_LANGUAGE,
|
||||
HeaderValue::from_static("en-US,en;q=0.9"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua"),
|
||||
HeaderValue::from_static("\"Chromium\";v=\"145\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"145\""),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua-mobile"),
|
||||
HeaderValue::from_static("?0"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua-platform"),
|
||||
HeaderValue::from_static("\"Linux\""),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-dest"),
|
||||
HeaderValue::from_static("document"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-mode"),
|
||||
HeaderValue::from_static("navigate"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-site"),
|
||||
HeaderValue::from_static("none"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-user"),
|
||||
HeaderValue::from_static("?1"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("upgrade-insecure-requests"),
|
||||
HeaderValue::from_static("1"),
|
||||
);
|
||||
|
||||
let cookie_header = self.cookie_jar.get_cookie_header(¤t_url);
|
||||
if !cookie_header.is_empty() {
|
||||
if let Ok(val) = HeaderValue::from_str(&cookie_header) {
|
||||
headers.insert(reqwest::header::COOKIE, val);
|
||||
}
|
||||
}
|
||||
|
||||
for (k, v) in self.extra_headers.read().await.iter() {
|
||||
if let (Ok(name), Ok(val)) = (
|
||||
HeaderName::from_bytes(k.as_bytes()),
|
||||
HeaderValue::from_str(v),
|
||||
) {
|
||||
headers.insert(name, val);
|
||||
}
|
||||
}
|
||||
|
||||
let mut req_builder = self.get_client().await.request(method.clone(), current_url.as_str())
|
||||
.headers(headers);
|
||||
|
||||
if let Some(ref b) = body {
|
||||
if method == Method::POST {
|
||||
req_builder = req_builder.header(
|
||||
reqwest::header::CONTENT_TYPE,
|
||||
"application/x-www-form-urlencoded",
|
||||
);
|
||||
}
|
||||
req_builder = req_builder.body(b.clone());
|
||||
}
|
||||
|
||||
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let resp = req_builder.send().await.map_err(|e| {
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
ObscuraNetError::Network(format!("{}: {}", current_url, e))
|
||||
})?;
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let status = resp.status();
|
||||
|
||||
for val in resp.headers().get_all(reqwest::header::SET_COOKIE) {
|
||||
if let Ok(s) = val.to_str() {
|
||||
self.cookie_jar.set_cookie(s, ¤t_url);
|
||||
}
|
||||
}
|
||||
|
||||
let response_headers: HashMap<String, String> = resp
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
|
||||
if status.is_redirection() {
|
||||
if let Some(location) = resp.headers().get(reqwest::header::LOCATION) {
|
||||
let location_str = location.to_str().map_err(|_| {
|
||||
ObscuraNetError::Network("Invalid redirect Location header".into())
|
||||
})?;
|
||||
let next_url = current_url.join(location_str).map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Invalid redirect URL: {}", e))
|
||||
})?;
|
||||
validate_url(&next_url)?;
|
||||
redirects.push(current_url.clone());
|
||||
current_url = next_url;
|
||||
if status == reqwest::StatusCode::MOVED_PERMANENTLY
|
||||
|| status == reqwest::StatusCode::FOUND
|
||||
|| status == reqwest::StatusCode::SEE_OTHER
|
||||
{
|
||||
method = Method::GET;
|
||||
body = None;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let body_bytes = resp.bytes().await.map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Failed to read body: {}", e))
|
||||
})?.to_vec();
|
||||
|
||||
let response = Response {
|
||||
url: current_url,
|
||||
status: status.as_u16(),
|
||||
headers: response_headers,
|
||||
body: body_bytes,
|
||||
redirected_from: redirects,
|
||||
};
|
||||
|
||||
for cb in self.on_response.read().await.iter() {
|
||||
cb(&request_info, &response);
|
||||
}
|
||||
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
Err(ObscuraNetError::TooManyRedirects(current_url.to_string()))
|
||||
}
|
||||
|
||||
pub async fn set_user_agent(&self, ua: &str) {
|
||||
*self.user_agent.write().await = ua.to_string();
|
||||
}
|
||||
|
||||
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
|
||||
*self.extra_headers.write().await = headers;
|
||||
}
|
||||
|
||||
pub fn active_requests(&self) -> u32 {
|
||||
self.in_flight.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn is_network_idle(&self) -> bool {
|
||||
self.active_requests() == 0
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ObscuraHttpClient {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ObscuraNetError {
|
||||
#[error("Network error: {0}")]
|
||||
Network(String),
|
||||
|
||||
#[error("Too many redirects: {0}")]
|
||||
TooManyRedirects(String),
|
||||
|
||||
#[error("Request blocked: {0}")]
|
||||
Blocked(String),
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::RwLock;
|
||||
use url::Url;
|
||||
|
||||
pub struct CookieJar {
|
||||
cookies: RwLock<HashMap<String, HashMap<String, CookieEntry>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct CookieEntry {
|
||||
name: String,
|
||||
value: String,
|
||||
path: String,
|
||||
domain: String,
|
||||
secure: bool,
|
||||
http_only: bool,
|
||||
expires: Option<u64>,
|
||||
same_site: String,
|
||||
}
|
||||
|
||||
impl CookieJar {
|
||||
pub fn new() -> Self {
|
||||
CookieJar {
|
||||
cookies: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_cookie(&self, set_cookie_str: &str, url: &Url) {
|
||||
let parts: Vec<&str> = set_cookie_str.splitn(2, ';').collect();
|
||||
let name_value = parts[0].trim();
|
||||
let (name, value) = match name_value.split_once('=') {
|
||||
Some((n, v)) => (n.trim().to_string(), v.trim().to_string()),
|
||||
None => return,
|
||||
};
|
||||
|
||||
let mut domain = url.host_str().unwrap_or("").to_lowercase();
|
||||
let mut path = url.path().to_string();
|
||||
let mut secure = false;
|
||||
let mut http_only = false;
|
||||
let mut expires: Option<u64> = None;
|
||||
let mut same_site = "Lax".to_string();
|
||||
|
||||
if parts.len() > 1 {
|
||||
for attr in parts[1].split(';') {
|
||||
let attr = attr.trim();
|
||||
if let Some((key, val)) = attr.split_once('=') {
|
||||
match key.trim().to_lowercase().as_str() {
|
||||
"domain" => {
|
||||
domain = val.trim().trim_start_matches('.').to_lowercase();
|
||||
}
|
||||
"path" => {
|
||||
path = val.trim().to_string();
|
||||
}
|
||||
"expires" => {
|
||||
if let Ok(ts) = parse_http_date(val.trim()) {
|
||||
expires = Some(ts);
|
||||
}
|
||||
}
|
||||
"max-age" => {
|
||||
if let Ok(secs) = val.trim().parse::<i64>() {
|
||||
if secs <= 0 {
|
||||
expires = Some(0);
|
||||
} else {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
expires = Some(now + secs as u64);
|
||||
}
|
||||
}
|
||||
}
|
||||
"samesite" => {
|
||||
same_site = val.trim().to_string();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
} else {
|
||||
match attr.to_lowercase().as_str() {
|
||||
"secure" => secure = true,
|
||||
"httponly" => http_only = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(exp) = expires {
|
||||
if exp == 0 {
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
if let Some(domain_cookies) = cookies.get_mut(&domain) {
|
||||
domain_cookies.remove(&name);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
if exp < now {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let entry = CookieEntry {
|
||||
name: name.clone(),
|
||||
value,
|
||||
path,
|
||||
domain: domain.clone(),
|
||||
secure,
|
||||
http_only,
|
||||
expires,
|
||||
same_site,
|
||||
};
|
||||
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
cookies.entry(domain).or_default().insert(name, entry);
|
||||
}
|
||||
|
||||
pub fn get_cookie_header(&self, url: &Url) -> String {
|
||||
let host = url.host_str().unwrap_or("");
|
||||
let path = url.path();
|
||||
let is_secure = url.scheme() == "https";
|
||||
let cookies = self.cookies.read().unwrap();
|
||||
|
||||
let mut matching: Vec<String> = Vec::new();
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
for (domain, domain_cookies) in cookies.iter() {
|
||||
if !domain_matches(host, domain) {
|
||||
continue;
|
||||
}
|
||||
for entry in domain_cookies.values() {
|
||||
if let Some(exp) = entry.expires {
|
||||
if exp < now {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if entry.secure && !is_secure {
|
||||
continue;
|
||||
}
|
||||
if !path.starts_with(&entry.path) {
|
||||
continue;
|
||||
}
|
||||
matching.push(format!("{}={}", entry.name, entry.value));
|
||||
}
|
||||
}
|
||||
|
||||
matching.join("; ")
|
||||
}
|
||||
|
||||
pub fn get_all_cookies(&self) -> Vec<CookieInfo> {
|
||||
let cookies = self.cookies.read().unwrap();
|
||||
let mut result = Vec::new();
|
||||
for domain_cookies in cookies.values() {
|
||||
for entry in domain_cookies.values() {
|
||||
result.push(CookieInfo {
|
||||
name: entry.name.clone(),
|
||||
value: entry.value.clone(),
|
||||
domain: entry.domain.clone(),
|
||||
path: entry.path.clone(),
|
||||
secure: entry.secure,
|
||||
http_only: entry.http_only,
|
||||
});
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn set_cookies_from_cdp(&self, cookies: Vec<CookieInfo>) {
|
||||
let mut jar = self.cookies.write().unwrap();
|
||||
for cookie in cookies {
|
||||
let entry = CookieEntry {
|
||||
name: cookie.name.clone(),
|
||||
value: cookie.value,
|
||||
path: cookie.path,
|
||||
domain: cookie.domain.clone(),
|
||||
secure: cookie.secure,
|
||||
http_only: cookie.http_only,
|
||||
expires: None,
|
||||
same_site: "Lax".to_string(),
|
||||
};
|
||||
jar.entry(cookie.domain).or_default().insert(cookie.name, entry);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_js_visible_cookies(&self, url: &Url) -> String {
|
||||
let host = url.host_str().unwrap_or("");
|
||||
let path = url.path();
|
||||
let is_secure = url.scheme() == "https";
|
||||
let cookies = self.cookies.read().unwrap();
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
let mut matching: Vec<String> = Vec::new();
|
||||
|
||||
for (domain, domain_cookies) in cookies.iter() {
|
||||
if !domain_matches(host, domain) {
|
||||
continue;
|
||||
}
|
||||
for entry in domain_cookies.values() {
|
||||
if entry.http_only {
|
||||
continue;
|
||||
}
|
||||
if let Some(exp) = entry.expires {
|
||||
if exp < now {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if entry.secure && !is_secure {
|
||||
continue;
|
||||
}
|
||||
if !path.starts_with(&entry.path) {
|
||||
continue;
|
||||
}
|
||||
matching.push(format!("{}={}", entry.name, entry.value));
|
||||
}
|
||||
}
|
||||
|
||||
matching.join("; ")
|
||||
}
|
||||
|
||||
pub fn set_cookie_from_js(&self, cookie_str: &str, url: &Url) {
|
||||
let parts: Vec<&str> = cookie_str.splitn(2, ';').collect();
|
||||
let name_value = parts[0].trim();
|
||||
let (name, value) = match name_value.split_once('=') {
|
||||
Some((n, v)) => (n.trim().to_string(), v.trim().to_string()),
|
||||
None => return,
|
||||
};
|
||||
|
||||
let mut domain = url.host_str().unwrap_or("").to_lowercase();
|
||||
let mut path = url.path().to_string();
|
||||
let mut secure = false;
|
||||
let mut expires: Option<u64> = None;
|
||||
let mut same_site = "Lax".to_string();
|
||||
|
||||
if parts.len() > 1 {
|
||||
for attr in parts[1].split(';') {
|
||||
let attr = attr.trim();
|
||||
if let Some((key, val)) = attr.split_once('=') {
|
||||
match key.trim().to_lowercase().as_str() {
|
||||
"domain" => {
|
||||
domain = val.trim().trim_start_matches('.').to_lowercase();
|
||||
}
|
||||
"path" => {
|
||||
path = val.trim().to_string();
|
||||
}
|
||||
"expires" => {
|
||||
if let Ok(ts) = parse_http_date(val.trim()) {
|
||||
expires = Some(ts);
|
||||
}
|
||||
}
|
||||
"max-age" => {
|
||||
if let Ok(secs) = val.trim().parse::<i64>() {
|
||||
if secs <= 0 {
|
||||
expires = Some(0);
|
||||
} else {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
expires = Some(now + secs as u64);
|
||||
}
|
||||
}
|
||||
}
|
||||
"samesite" => {
|
||||
same_site = val.trim().to_string();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
} else {
|
||||
match attr.to_lowercase().as_str() {
|
||||
"secure" => secure = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(exp) = expires {
|
||||
if exp == 0 {
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
if let Some(domain_cookies) = cookies.get_mut(&domain) {
|
||||
domain_cookies.remove(&name);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
if exp < now {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let entry = CookieEntry {
|
||||
name: name.clone(),
|
||||
value,
|
||||
path,
|
||||
domain: domain.clone(),
|
||||
secure,
|
||||
http_only: false,
|
||||
expires,
|
||||
same_site,
|
||||
};
|
||||
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
cookies.entry(domain).or_default().insert(name, entry);
|
||||
}
|
||||
|
||||
pub fn delete_cookie(&self, name: &str, domain: &str) {
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
if domain.is_empty() {
|
||||
for domain_cookies in cookies.values_mut() {
|
||||
domain_cookies.remove(name);
|
||||
}
|
||||
} else {
|
||||
let domains_to_try = [
|
||||
domain.to_string(),
|
||||
format!(".{}", domain.trim_start_matches('.')),
|
||||
domain.trim_start_matches('.').to_string(),
|
||||
];
|
||||
for d in &domains_to_try {
|
||||
if let Some(domain_cookies) = cookies.get_mut(d.as_str()) {
|
||||
domain_cookies.remove(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear(&self) {
|
||||
self.cookies.write().unwrap().clear();
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CookieJar {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct CookieInfo {
|
||||
pub name: String,
|
||||
pub value: String,
|
||||
pub domain: String,
|
||||
pub path: String,
|
||||
pub secure: bool,
|
||||
#[serde(rename = "httpOnly")]
|
||||
pub http_only: bool,
|
||||
}
|
||||
|
||||
fn parse_http_date(s: &str) -> Result<u64, ()> {
|
||||
let months = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"];
|
||||
|
||||
let s = s.replace('-', " ");
|
||||
let parts: Vec<&str> = s.split_whitespace().collect();
|
||||
|
||||
if parts.len() < 5 { return Err(()); }
|
||||
|
||||
let day: u64 = parts[1].parse().map_err(|_| ())?;
|
||||
let month = months.iter().position(|m| parts[2].to_lowercase().starts_with(m))
|
||||
.ok_or(())? as u64 + 1;
|
||||
let year: u64 = parts[3].parse().map_err(|_| ())?;
|
||||
|
||||
let time_parts: Vec<&str> = parts[4].split(':').collect();
|
||||
let hour: u64 = time_parts.first().and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||
let minute: u64 = time_parts.get(1).and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||
let second: u64 = time_parts.get(2).and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||
|
||||
let mut days_total: u64 = 0;
|
||||
for y in 1970..year {
|
||||
days_total += if y % 4 == 0 && (y % 100 != 0 || y % 400 == 0) { 366 } else { 365 };
|
||||
}
|
||||
let days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
||||
let is_leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0);
|
||||
for m in 1..month {
|
||||
days_total += days_in_month[m as usize] + if m == 2 && is_leap { 1 } else { 0 };
|
||||
}
|
||||
days_total += day - 1;
|
||||
|
||||
Ok(days_total * 86400 + hour * 3600 + minute * 60 + second)
|
||||
}
|
||||
|
||||
fn domain_matches(host: &str, domain: &str) -> bool {
|
||||
let host = host.to_lowercase();
|
||||
let domain = domain.to_lowercase();
|
||||
host == domain || host.ends_with(&format!(".{}", domain))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_set_and_get_cookie() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/path").unwrap();
|
||||
jar.set_cookie("session=abc123; Path=/; Secure; HttpOnly", &url);
|
||||
|
||||
let header = jar.get_cookie_header(&url);
|
||||
assert!(header.contains("session=abc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cookie_domain_matching() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://www.example.com/").unwrap();
|
||||
jar.set_cookie("token=xyz; Domain=example.com", &url);
|
||||
|
||||
let header = jar.get_cookie_header(&url);
|
||||
assert!(header.contains("token=xyz"));
|
||||
|
||||
let sub_url = Url::parse("https://api.example.com/").unwrap();
|
||||
let header2 = jar.get_cookie_header(&sub_url);
|
||||
assert!(header2.contains("token=xyz"));
|
||||
|
||||
let other_url = Url::parse("https://other.com/").unwrap();
|
||||
let header3 = jar.get_cookie_header(&other_url);
|
||||
assert!(header3.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_secure_cookie_not_sent_over_http() {
|
||||
let jar = CookieJar::new();
|
||||
let https_url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("secure_token=secret; Secure", &https_url);
|
||||
|
||||
let http_url = Url::parse("http://example.com/").unwrap();
|
||||
let header = jar.get_cookie_header(&http_url);
|
||||
assert!(header.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_age_zero_deletes_cookie() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("session=abc", &url);
|
||||
assert!(jar.get_cookie_header(&url).contains("session=abc"));
|
||||
|
||||
jar.set_cookie("session=abc; Max-Age=0", &url);
|
||||
assert!(jar.get_cookie_header(&url).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_age_sets_expiry() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("token=xyz; Max-Age=3600", &url);
|
||||
assert!(jar.get_cookie_header(&url).contains("token=xyz"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_expired_cookie_not_sent() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("old=gone; Expires=Thu, 01 Jan 2020 00:00:00 GMT", &url);
|
||||
assert!(jar.get_cookie_header(&url).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_samesite_parsed() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("strict_cookie=val; SameSite=Strict", &url);
|
||||
assert!(jar.get_cookie_header(&url).contains("strict_cookie=val"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clear_cookies() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("a=1", &url);
|
||||
assert!(!jar.get_cookie_header(&url).is_empty());
|
||||
|
||||
jar.clear();
|
||||
assert!(jar.get_cookie_header(&url).is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::client::{RequestInfo, Response};
|
||||
|
||||
pub enum InterceptAction {
|
||||
Continue,
|
||||
Block,
|
||||
Fulfill(Response),
|
||||
ModifyHeaders(HashMap<String, String>),
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait RequestInterceptor {
|
||||
async fn intercept(&self, request: &RequestInfo) -> InterceptAction;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
pub mod client;
|
||||
pub mod cookies;
|
||||
pub mod interceptor;
|
||||
pub mod robots;
|
||||
pub mod blocklist;
|
||||
#[cfg(feature = "stealth")]
|
||||
pub mod wreq_client;
|
||||
|
||||
pub use client::{ObscuraHttpClient, ObscuraNetError, RequestInfo, ResourceType, Response};
|
||||
pub use cookies::{CookieInfo, CookieJar};
|
||||
pub use robots::RobotsCache;
|
||||
pub use blocklist::is_blocked as is_tracker_blocked;
|
||||
#[cfg(feature = "stealth")]
|
||||
pub use wreq_client::{StealthHttpClient, STEALTH_USER_AGENT};
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,163 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::RwLock;
|
||||
|
||||
pub struct RobotsCache {
|
||||
cache: RwLock<HashMap<String, RobotsRules>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct RobotsRules {
|
||||
disallowed: Vec<String>,
|
||||
allowed: Vec<String>,
|
||||
}
|
||||
|
||||
impl RobotsCache {
|
||||
pub fn new() -> Self {
|
||||
RobotsCache {
|
||||
cache: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_and_store(&self, domain: &str, body: &str, our_agent: &str) {
|
||||
let rules = parse_robots_txt(body, our_agent);
|
||||
self.cache.write().unwrap().insert(domain.to_string(), rules);
|
||||
}
|
||||
|
||||
pub fn is_allowed(&self, domain: &str, path: &str) -> bool {
|
||||
let cache = self.cache.read().unwrap();
|
||||
let rules = match cache.get(domain) {
|
||||
Some(r) => r,
|
||||
None => return true,
|
||||
};
|
||||
|
||||
for pattern in &rules.allowed {
|
||||
if path_matches(path, pattern) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
for pattern in &rules.disallowed {
|
||||
if path_matches(path, pattern) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RobotsCache {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_robots_txt(body: &str, our_agent: &str) -> RobotsRules {
|
||||
let our_agent_lower = our_agent.to_lowercase();
|
||||
let mut disallowed = Vec::new();
|
||||
let mut allowed = Vec::new();
|
||||
let mut in_matching_section = false;
|
||||
let mut found_specific = false;
|
||||
|
||||
for line in body.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some((key, value)) = line.split_once(':') {
|
||||
let key = key.trim().to_lowercase();
|
||||
let value = value.trim();
|
||||
|
||||
match key.as_str() {
|
||||
"user-agent" => {
|
||||
let agent = value.to_lowercase();
|
||||
in_matching_section = agent == "*"
|
||||
|| our_agent_lower.contains(&agent)
|
||||
|| agent.contains(&our_agent_lower);
|
||||
if agent != "*" && in_matching_section {
|
||||
found_specific = true;
|
||||
}
|
||||
}
|
||||
"disallow" if in_matching_section && !value.is_empty() => {
|
||||
disallowed.push(value.to_string());
|
||||
}
|
||||
"allow" if in_matching_section && !value.is_empty() => {
|
||||
allowed.push(value.to_string());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !found_specific {
|
||||
disallowed.clear();
|
||||
allowed.clear();
|
||||
in_matching_section = false;
|
||||
|
||||
for line in body.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') { continue; }
|
||||
if let Some((key, value)) = line.split_once(':') {
|
||||
let key = key.trim().to_lowercase();
|
||||
let value = value.trim();
|
||||
match key.as_str() {
|
||||
"user-agent" => {
|
||||
in_matching_section = value.trim() == "*";
|
||||
}
|
||||
"disallow" if in_matching_section && !value.is_empty() => {
|
||||
disallowed.push(value.to_string());
|
||||
}
|
||||
"allow" if in_matching_section && !value.is_empty() => {
|
||||
allowed.push(value.to_string());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
RobotsRules { disallowed, allowed }
|
||||
}
|
||||
|
||||
fn path_matches(path: &str, pattern: &str) -> bool {
|
||||
if pattern.ends_with('*') {
|
||||
path.starts_with(&pattern[..pattern.len() - 1])
|
||||
} else if pattern.ends_with('$') {
|
||||
path == &pattern[..pattern.len() - 1]
|
||||
} else {
|
||||
path.starts_with(pattern)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_basic_robots() {
|
||||
let body = "User-agent: *\nDisallow: /private/\nDisallow: /admin\nAllow: /admin/public\n";
|
||||
let cache = RobotsCache::new();
|
||||
cache.parse_and_store("example.com", body, "Obscura");
|
||||
assert!(cache.is_allowed("example.com", "/"));
|
||||
assert!(cache.is_allowed("example.com", "/page"));
|
||||
assert!(!cache.is_allowed("example.com", "/private/secret"));
|
||||
assert!(!cache.is_allowed("example.com", "/admin"));
|
||||
assert!(cache.is_allowed("example.com", "/admin/public"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_rules_means_allowed() {
|
||||
let cache = RobotsCache::new();
|
||||
assert!(cache.is_allowed("unknown.com", "/anything"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_disallow_all() {
|
||||
let body = "User-agent: *\nDisallow: /\n";
|
||||
let cache = RobotsCache::new();
|
||||
cache.parse_and_store("blocked.com", body, "Obscura");
|
||||
assert!(!cache.is_allowed("blocked.com", "/"));
|
||||
assert!(!cache.is_allowed("blocked.com", "/page"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::collections::HashMap;
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::error::Error;
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::sync::Arc;
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::time::Duration;
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
use tokio::sync::RwLock;
|
||||
#[cfg(feature = "stealth")]
|
||||
use url::Url;
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
use crate::cookies::CookieJar;
|
||||
#[cfg(feature = "stealth")]
|
||||
use crate::client::{Response, ObscuraNetError};
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
pub const STEALTH_USER_AGENT: &str =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
pub struct StealthHttpClient {
|
||||
client: wreq::Client,
|
||||
pub cookie_jar: Arc<CookieJar>,
|
||||
pub extra_headers: RwLock<HashMap<String, String>>,
|
||||
pub in_flight: Arc<std::sync::atomic::AtomicU32>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
impl StealthHttpClient {
|
||||
pub fn new(cookie_jar: Arc<CookieJar>) -> Self {
|
||||
Self::with_proxy(cookie_jar, None)
|
||||
}
|
||||
|
||||
pub fn with_proxy(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
|
||||
let cert_store = wreq::tls::CertStore::builder()
|
||||
.set_default_paths()
|
||||
.build()
|
||||
.expect("Failed to load system CA certificates");
|
||||
|
||||
let emulation_opts = wreq_util::EmulationOption::builder()
|
||||
.emulation(wreq_util::Emulation::Chrome145)
|
||||
.emulation_os(wreq_util::EmulationOS::Linux)
|
||||
.build();
|
||||
|
||||
let mut builder = wreq::Client::builder()
|
||||
.emulation(emulation_opts)
|
||||
.cert_store(cert_store)
|
||||
.timeout(Duration::from_secs(30))
|
||||
.redirect(wreq::redirect::Policy::none());
|
||||
|
||||
if let Some(proxy) = proxy_url {
|
||||
if let Ok(p) = wreq::Proxy::all(proxy) {
|
||||
builder = builder.proxy(p);
|
||||
}
|
||||
}
|
||||
|
||||
let client = builder.build().expect("failed to build wreq stealth client");
|
||||
|
||||
StealthHttpClient {
|
||||
client,
|
||||
cookie_jar,
|
||||
extra_headers: RwLock::new(HashMap::new()),
|
||||
in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
||||
let mut current_url = url.clone();
|
||||
let mut redirects = Vec::new();
|
||||
|
||||
for _ in 0..20 {
|
||||
let mut req = self.client.get(current_url.as_str());
|
||||
|
||||
let cookie_header = self.cookie_jar.get_cookie_header(¤t_url);
|
||||
if !cookie_header.is_empty() {
|
||||
req = req.header("Cookie", &cookie_header);
|
||||
}
|
||||
|
||||
for (k, v) in self.extra_headers.read().await.iter() {
|
||||
req = req.header(k.as_str(), v.as_str());
|
||||
}
|
||||
|
||||
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let resp = req.send().await.map_err(|e| {
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
ObscuraNetError::Network(format!("{}: {} (source: {:?})", current_url, e, e.source()))
|
||||
})?;
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let status = resp.status();
|
||||
|
||||
for val in resp.headers().get_all("set-cookie") {
|
||||
if let Ok(s) = val.to_str() {
|
||||
self.cookie_jar.set_cookie(s, ¤t_url);
|
||||
}
|
||||
}
|
||||
|
||||
let response_headers: HashMap<String, String> = resp
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
|
||||
if status.is_redirection() {
|
||||
if let Some(location) = resp.headers().get("location") {
|
||||
let location_str = location.to_str().map_err(|_| {
|
||||
ObscuraNetError::Network("Invalid redirect Location".into())
|
||||
})?;
|
||||
let next_url = current_url.join(location_str).map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Invalid redirect URL: {}", e))
|
||||
})?;
|
||||
redirects.push(current_url.clone());
|
||||
current_url = next_url;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let body = resp.bytes().await.map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Failed to read body: {}", e))
|
||||
})?.to_vec();
|
||||
|
||||
return Ok(Response {
|
||||
url: current_url,
|
||||
status: status.as_u16(),
|
||||
headers: response_headers,
|
||||
body,
|
||||
redirected_from: redirects,
|
||||
});
|
||||
}
|
||||
|
||||
Err(ObscuraNetError::TooManyRedirects(url.to_string()))
|
||||
}
|
||||
|
||||
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
|
||||
*self.extra_headers.write().await = headers;
|
||||
}
|
||||
|
||||
pub fn active_requests(&self) -> u32 {
|
||||
self.in_flight.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn is_network_idle(&self) -> bool {
|
||||
self.active_requests() == 0
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user