346 lines
13 KiB
Rust
346 lines
13 KiB
Rust
#[cfg(feature = "stealth")]
|
|
use std::collections::HashMap;
|
|
#[cfg(feature = "stealth")]
|
|
use std::error::Error;
|
|
#[cfg(feature = "stealth")]
|
|
use std::sync::Arc;
|
|
#[cfg(feature = "stealth")]
|
|
use std::time::Duration;
|
|
|
|
#[cfg(feature = "stealth")]
|
|
use tokio::sync::RwLock;
|
|
#[cfg(feature = "stealth")]
|
|
use url::Url;
|
|
|
|
#[cfg(feature = "stealth")]
|
|
use crate::cookies::CookieJar;
|
|
#[cfg(feature = "stealth")]
|
|
use crate::client::{Response, ObscuraNetError};
|
|
|
|
#[cfg(feature = "stealth")]
|
|
pub const STEALTH_USER_AGENT: &str =
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
|
|
|
|
// The wreq emulation (Profile::Chrome145, Platform::Windows) sends this exact
|
|
// UA and sec-ch-ua-platform "Windows" on the wire. navigator has to report the
|
|
// same identity, otherwise the TLS/HTTP layer and the JS layer disagree and a
|
|
// site cross-checks the mismatch as a bot signal.
|
|
#[cfg(feature = "stealth")]
|
|
pub const STEALTH_NAVIGATOR_PLATFORM: &str = "Win32";
|
|
#[cfg(feature = "stealth")]
|
|
pub const STEALTH_UA_PLATFORM: &str = "Windows";
|
|
#[cfg(feature = "stealth")]
|
|
pub const STEALTH_UA_PLATFORM_VERSION: &str = "15.0.0";
|
|
|
|
#[cfg(feature = "stealth")]
|
|
pub struct StealthHttpClient {
|
|
client: wreq::Client,
|
|
pub cookie_jar: Arc<CookieJar>,
|
|
pub extra_headers: RwLock<HashMap<String, String>>,
|
|
pub in_flight: Arc<std::sync::atomic::AtomicU32>,
|
|
}
|
|
|
|
#[cfg(feature = "stealth")]
|
|
impl StealthHttpClient {
|
|
pub fn new(cookie_jar: Arc<CookieJar>) -> Self {
|
|
Self::with_proxy(cookie_jar, None)
|
|
}
|
|
|
|
pub fn with_proxy(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
|
|
let emulation_opts = wreq_util::Emulation::builder()
|
|
.profile(wreq_util::Profile::Chrome145)
|
|
.platform(wreq_util::Platform::Windows)
|
|
.build();
|
|
|
|
let mut builder = wreq::Client::builder()
|
|
.emulation(emulation_opts)
|
|
.timeout(Duration::from_secs(30))
|
|
.redirect(wreq::redirect::Policy::none());
|
|
|
|
// Honor SSL_CERT_FILE / SSL_CERT_DIR in the stealth client too.
|
|
//
|
|
// `client.rs` (the reqwest path) already reads these via `configured_root_paths()` and
|
|
// feeds `add_root_certificate`, so a private CA works there. This client did not, which
|
|
// made the *better-fingerprinted* transport the only one unable to reach hosts behind a
|
|
// private/national CA (measured against a Brazilian government portal whose leaf is
|
|
// issued by an ICP-Brasil intermediate: `--stealth` failed with CERTIFICATE_VERIFY_FAILED
|
|
// while the reqwest path, with SSL_CERT_FILE set, completed the handshake).
|
|
//
|
|
// Two deliberate constraints:
|
|
//
|
|
// 1. `tls_cert_store` is used, NOT `tls_options`. `emulation()` overwrites `tls_options`
|
|
// wholesale ("This will overwrite the existing configuration"), so setting TLS options
|
|
// here would silently discard the Chrome fingerprint — the whole point of this client.
|
|
// `tls_cert_store` is a separate field on the config and composes with emulation.
|
|
//
|
|
// 2. Opt-in only. Supplying a store REPLACES the webpki roots (see `set_cert_store` in
|
|
// `tls/conn/ext.rs`), it does not add to them. Applying it unconditionally would break
|
|
// every ordinary site whenever the bundle is incomplete. With neither variable set,
|
|
// behaviour is byte-for-byte what it was before.
|
|
if crate::client::custom_cert_store_requested(
|
|
std::env::var_os("SSL_CERT_FILE").as_deref(),
|
|
std::env::var_os("SSL_CERT_DIR").as_deref(),
|
|
) {
|
|
match wreq::tls::trust::CertStore::builder().set_default_paths().build() {
|
|
Ok(store) => builder = builder.tls_cert_store(store),
|
|
Err(error) => tracing::warn!(
|
|
%error,
|
|
"SSL_CERT_FILE/SSL_CERT_DIR set but the certificate store failed to build; \
|
|
continuing with the default roots"
|
|
),
|
|
}
|
|
}
|
|
|
|
if let Some(proxy) = proxy_url {
|
|
if let Ok(p) = wreq::Proxy::all(proxy) {
|
|
builder = builder.proxy(p);
|
|
}
|
|
}
|
|
|
|
let client = builder.build().expect("failed to build wreq stealth client");
|
|
|
|
StealthHttpClient {
|
|
client,
|
|
cookie_jar,
|
|
extra_headers: RwLock::new(HashMap::new()),
|
|
in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),
|
|
}
|
|
}
|
|
|
|
pub async fn fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
|
let mut current_url = url.clone();
|
|
|
|
if let Some(host) = current_url.host_str() {
|
|
if crate::blocklist::is_blocked(host) {
|
|
tracing::debug!("Blocked tracker: {}", current_url);
|
|
return Ok(Response {
|
|
status: 0,
|
|
url: current_url,
|
|
headers: HashMap::new(),
|
|
body: Vec::new(),
|
|
redirected_from: Vec::new(),
|
|
});
|
|
}
|
|
}
|
|
|
|
let mut redirects = Vec::new();
|
|
|
|
for _ in 0..20 {
|
|
let mut req = self.client.get(current_url.as_str());
|
|
|
|
let cookie_header = self.cookie_jar.get_cookie_header(¤t_url);
|
|
if !cookie_header.is_empty() {
|
|
req = req.header("Cookie", &cookie_header);
|
|
}
|
|
|
|
for (k, v) in self.extra_headers.read().await.iter() {
|
|
req = req.header(k.as_str(), v.as_str());
|
|
}
|
|
|
|
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
|
let resp = req.send().await.map_err(|e| {
|
|
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
|
ObscuraNetError::Network(format!("{}: {} (source: {:?})", current_url, e, e.source()))
|
|
})?;
|
|
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
|
|
|
let status = resp.status();
|
|
|
|
for val in resp.headers().get_all("set-cookie") {
|
|
if let Ok(s) = val.to_str() {
|
|
self.cookie_jar.set_cookie(s, ¤t_url);
|
|
}
|
|
}
|
|
|
|
let response_headers: HashMap<String, String> = resp
|
|
.headers()
|
|
.iter()
|
|
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
|
.collect();
|
|
|
|
if status.is_redirection() {
|
|
if let Some(location) = resp.headers().get("location") {
|
|
let location_str = location.to_str().map_err(|_| {
|
|
ObscuraNetError::Network("Invalid redirect Location".into())
|
|
})?;
|
|
let next_url = current_url.join(location_str).map_err(|e| {
|
|
ObscuraNetError::Network(format!("Invalid redirect URL: {}", e))
|
|
})?;
|
|
redirects.push(current_url.clone());
|
|
current_url = next_url;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
let body = resp.bytes().await.map_err(|e| {
|
|
ObscuraNetError::Network(format!("Failed to read body: {}", e))
|
|
})?.to_vec();
|
|
|
|
return Ok(Response {
|
|
url: current_url,
|
|
status: status.as_u16(),
|
|
headers: response_headers,
|
|
body,
|
|
redirected_from: redirects,
|
|
});
|
|
}
|
|
|
|
Err(ObscuraNetError::TooManyRedirects(url.to_string()))
|
|
}
|
|
|
|
/// One request with no redirect following, for scripted fetch()/XHR. The
|
|
/// caller supplies the Fetch credentials decision for this redirect hop,
|
|
/// while this method preserves the Chrome transport fingerprint.
|
|
pub async fn send_single(
|
|
&self,
|
|
method: &str,
|
|
url: &Url,
|
|
headers: &HashMap<String, String>,
|
|
body: &str,
|
|
send_cookies: bool,
|
|
store_cookies: bool,
|
|
) -> Result<Response, ObscuraNetError> {
|
|
if let Some(host) = url.host_str() {
|
|
if crate::blocklist::is_blocked(host) {
|
|
tracing::debug!("Blocked tracker: {}", url);
|
|
return Ok(Response {
|
|
status: 0,
|
|
url: url.clone(),
|
|
headers: HashMap::new(),
|
|
body: Vec::new(),
|
|
redirected_from: Vec::new(),
|
|
});
|
|
}
|
|
}
|
|
|
|
let req_method = method
|
|
.parse::<wreq::Method>()
|
|
.map_err(|e| ObscuraNetError::Network(format!("invalid method '{}': {}", method, e)))?;
|
|
let mut req = self.client.request(req_method, url.as_str());
|
|
|
|
if send_cookies {
|
|
let cookie_header = self.cookie_jar.get_cookie_header(url);
|
|
if !cookie_header.is_empty() {
|
|
req = req.header("cookie", &cookie_header);
|
|
}
|
|
}
|
|
for (k, v) in self.extra_headers.read().await.iter() {
|
|
req = req.header(k.as_str(), v.as_str());
|
|
}
|
|
for (k, v) in headers.iter() {
|
|
req = req.header(k.as_str(), v.as_str());
|
|
}
|
|
if !body.is_empty() {
|
|
req = req.body(body.to_string());
|
|
}
|
|
|
|
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
|
let resp = req.send().await.map_err(|e| {
|
|
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
|
ObscuraNetError::Network(format!("{}: {}", url, e))
|
|
})?;
|
|
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
|
|
|
let status = resp.status();
|
|
if store_cookies {
|
|
for val in resp.headers().get_all("set-cookie") {
|
|
if let Ok(s) = val.to_str() {
|
|
self.cookie_jar.set_cookie(s, url);
|
|
}
|
|
}
|
|
}
|
|
let response_headers: HashMap<String, String> = resp
|
|
.headers()
|
|
.iter()
|
|
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
|
.collect();
|
|
let resp_body = resp
|
|
.bytes()
|
|
.await
|
|
.map_err(|e| ObscuraNetError::Network(format!("Failed to read body: {}", e)))?
|
|
.to_vec();
|
|
|
|
Ok(Response {
|
|
url: url.clone(),
|
|
status: status.as_u16(),
|
|
headers: response_headers,
|
|
body: resp_body,
|
|
redirected_from: Vec::new(),
|
|
})
|
|
}
|
|
|
|
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
|
|
*self.extra_headers.write().await = headers;
|
|
}
|
|
|
|
pub fn active_requests(&self) -> u32 {
|
|
self.in_flight.load(std::sync::atomic::Ordering::Relaxed)
|
|
}
|
|
|
|
pub fn is_network_idle(&self) -> bool {
|
|
self.active_requests() == 0
|
|
}
|
|
}
|
|
|
|
#[cfg(all(test, feature = "stealth"))]
|
|
mod tests {
|
|
use std::sync::Arc;
|
|
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
use url::Url;
|
|
|
|
use super::StealthHttpClient;
|
|
use crate::cookies::CookieJar;
|
|
|
|
const PLAIN_BODY: &str = "<!DOCTYPE html><html><body><p id=\"mark\">gzip ok</p></body></html>";
|
|
|
|
// gzip (level 9) of PLAIN_BODY, hardcoded so the fixture needs no
|
|
// compression dependency. A wrong byte fails the assert below.
|
|
const GZIP_BODY: &[u8] = &[
|
|
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x03, 0xb3, 0x51,
|
|
0x74, 0xf1, 0x77, 0x0e, 0x89, 0x0c, 0x70, 0x55, 0xc8, 0x28, 0xc9, 0xcd,
|
|
0xb1, 0xb3, 0x81, 0x90, 0x49, 0xf9, 0x29, 0x95, 0x76, 0x36, 0x05, 0x0a,
|
|
0x99, 0x29, 0xb6, 0x4a, 0xb9, 0x89, 0x45, 0xd9, 0x4a, 0x76, 0xe9, 0x55,
|
|
0x99, 0x05, 0x0a, 0xf9, 0xd9, 0x36, 0xfa, 0x05, 0x76, 0x36, 0xfa, 0x10,
|
|
0x69, 0x7d, 0xb0, 0x5a, 0x00, 0x80, 0x3d, 0x1c, 0x5f, 0x41, 0x00, 0x00,
|
|
0x00,
|
|
];
|
|
|
|
/// Serve one `Content-Encoding: gzip` response on an ephemeral port.
|
|
async fn gzip_fixture() -> u16 {
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let port = listener.local_addr().unwrap().port();
|
|
|
|
tokio::spawn(async move {
|
|
while let Ok((mut stream, _)) = listener.accept().await {
|
|
tokio::spawn(async move {
|
|
let mut buf = [0u8; 1024];
|
|
let _ = stream.read(&mut buf).await;
|
|
let head = format!(
|
|
"HTTP/1.1 200 OK\r\ncontent-type: text/html; charset=utf-8\r\ncontent-encoding: gzip\r\ncontent-length: {}\r\nconnection: close\r\n\r\n",
|
|
GZIP_BODY.len()
|
|
);
|
|
let _ = stream.write_all(head.as_bytes()).await;
|
|
let _ = stream.write_all(GZIP_BODY).await;
|
|
let _ = stream.shutdown().await;
|
|
});
|
|
}
|
|
});
|
|
|
|
port
|
|
}
|
|
|
|
// The emulation profile advertises gzip, so origins compress. Without the
|
|
// decoder the raw gzip bytes reach the HTML parser as document text.
|
|
#[tokio::test]
|
|
async fn stealth_client_decodes_gzip_response() {
|
|
let port = gzip_fixture().await;
|
|
let client = StealthHttpClient::new(Arc::new(CookieJar::new()));
|
|
let url = Url::parse(&format!("http://127.0.0.1:{port}/")).unwrap();
|
|
|
|
let resp = client.fetch(&url).await.expect("fixture must be reachable");
|
|
assert_eq!(resp.status, 200);
|
|
assert_eq!(resp.text(), PLAIN_BODY, "gzip body must be decompressed");
|
|
}
|
|
}
|