fix(net): decode Content-Encoding on the stealth client
The wreq dependency was declared without the gzip, brotli, deflate and zstd features, so the stealth client returned bodies still compressed while the reqwest path decoded them. The Chrome145 emulation advertises those encodings, so origins do compress, and the raw gzip bytes reached the HTML parser as document text. Add a regression test that serves one gzip response and asserts the stealth client decodes it. Fixes #563.
This commit is contained in:
3 files changed
+67
-2
No files matched your search
Generated
+3
@@ -496,6 +496,8 @@ dependencies = [
|
||||
"compression-core",
|
||||
"flate2",
|
||||
"memchr",
|
||||
"zstd",
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3680,6 +3682,7 @@ dependencies = [
|
||||
"tokio-btls",
|
||||
"tokio-socks",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"url",
|
||||
"webpki-root-certs",
|
||||
"wreq-proto",
|
||||
|
||||
@@ -39,7 +39,7 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["ring",
|
||||
# Enabling it on other platforms produces unresolved `build_script_main_*`
|
||||
# symbols at link time (see issue #39).
|
||||
[target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies]
|
||||
wreq = { version = "=6.0.0-rc.29", optional = true, features = ["prefix-symbols", "socks"] }
|
||||
wreq = { version = "=6.0.0-rc.29", optional = true, features = ["prefix-symbols", "socks", "gzip", "brotli", "deflate", "zstd"] }
|
||||
|
||||
[target.'cfg(not(any(target_os = "linux", target_os = "android")))'.dependencies]
|
||||
wreq = { version = "=6.0.0-rc.29", optional = true, features = ["socks"] }
|
||||
wreq = { version = "=6.0.0-rc.29", optional = true, features = ["socks", "gzip", "brotli", "deflate", "zstd"] }
|
||||
@@ -278,3 +278,65 @@ impl StealthHttpClient {
|
||||
self.active_requests() == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "stealth"))]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use url::Url;
|
||||
|
||||
use super::StealthHttpClient;
|
||||
use crate::cookies::CookieJar;
|
||||
|
||||
const PLAIN_BODY: &str = "<!DOCTYPE html><html><body><p id=\"mark\">gzip ok</p></body></html>";
|
||||
|
||||
// gzip (level 9) of PLAIN_BODY, hardcoded so the fixture needs no
|
||||
// compression dependency. A wrong byte fails the assert below.
|
||||
const GZIP_BODY: &[u8] = &[
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x03, 0xb3, 0x51,
|
||||
0x74, 0xf1, 0x77, 0x0e, 0x89, 0x0c, 0x70, 0x55, 0xc8, 0x28, 0xc9, 0xcd,
|
||||
0xb1, 0xb3, 0x81, 0x90, 0x49, 0xf9, 0x29, 0x95, 0x76, 0x36, 0x05, 0x0a,
|
||||
0x99, 0x29, 0xb6, 0x4a, 0xb9, 0x89, 0x45, 0xd9, 0x4a, 0x76, 0xe9, 0x55,
|
||||
0x99, 0x05, 0x0a, 0xf9, 0xd9, 0x36, 0xfa, 0x05, 0x76, 0x36, 0xfa, 0x10,
|
||||
0x69, 0x7d, 0xb0, 0x5a, 0x00, 0x80, 0x3d, 0x1c, 0x5f, 0x41, 0x00, 0x00,
|
||||
0x00,
|
||||
];
|
||||
|
||||
/// Serve one `Content-Encoding: gzip` response on an ephemeral port.
|
||||
async fn gzip_fixture() -> u16 {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
|
||||
tokio::spawn(async move {
|
||||
while let Ok((mut stream, _)) = listener.accept().await {
|
||||
tokio::spawn(async move {
|
||||
let mut buf = [0u8; 1024];
|
||||
let _ = stream.read(&mut buf).await;
|
||||
let head = format!(
|
||||
"HTTP/1.1 200 OK\r\ncontent-type: text/html; charset=utf-8\r\ncontent-encoding: gzip\r\ncontent-length: {}\r\nconnection: close\r\n\r\n",
|
||||
GZIP_BODY.len()
|
||||
);
|
||||
let _ = stream.write_all(head.as_bytes()).await;
|
||||
let _ = stream.write_all(GZIP_BODY).await;
|
||||
let _ = stream.shutdown().await;
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
port
|
||||
}
|
||||
|
||||
// The emulation profile advertises gzip, so origins compress. Without the
|
||||
// decoder the raw gzip bytes reach the HTML parser as document text.
|
||||
#[tokio::test]
|
||||
async fn stealth_client_decodes_gzip_response() {
|
||||
let port = gzip_fixture().await;
|
||||
let client = StealthHttpClient::new(Arc::new(CookieJar::new()));
|
||||
let url = Url::parse(&format!("http://127.0.0.1:{port}/")).unwrap();
|
||||
|
||||
let resp = client.fetch(&url).await.expect("fixture must be reachable");
|
||||
assert_eq!(resp.status, 200);
|
||||
assert_eq!(resp.text(), PLAIN_BODY, "gzip body must be decompressed");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user