From 52fcf2a35ade1154b473c72ac2b730a1058f0bd8 Mon Sep 17 00:00:00 2001 From: SGavrl <277843151+SGavrl@users.noreply.github.com> Date: Sun, 24 May 2026 20:14:53 +0000 Subject: [PATCH] fix(net,browser): detect response charset, decode non-UTF-8 pages (#182) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit obscura was calling String::from_utf8_lossy on every response body, which silently corrupts every non-UTF-8 page (GBK, Big5, Shift-JIS, Windows-125x, EUC-KR, ISO-8859-x). Whole language families came through as replacement characters. Add an encoding module (obscura-net::encoding) that picks the decoder the way real browsers do per HTML5 § 8.2.2.4: 1. HTTP Content-Type charset=... parameter (authoritative when present) 2. or sniffed from the first 1KB of the body (HTML only) 3. UTF-8 fallback Use it in: - obscura-net Response::text (signature changed: String, not Result) - obscura-browser page.rs HTML body decode (the primary scrape path) - obscura-browser page.rs JS body decode - obscura-browser page.rs CSS body decode For JS / CSS we deliberately skip the meta sniff: those bodies are not HTML and any '' substring inside them is not a charset declaration. Adds 7 unit tests for the encoding module covering header precedence, quoted charset values, meta charset sniff, http-equiv form, the 1KB sniff window, and a round-trip decode of GBK bytes. Adds a Python feature test exercising file:// + meta charset and a live HTTP server serving Shift_JIS with the wrong meta tag. Closes #113. --- Cargo.lock | 16 ++- crates/obscura-browser/src/page.rs | 14 +- crates/obscura-net/Cargo.toml | 1 + crates/obscura-net/src/client.rs | 13 +- crates/obscura-net/src/encoding.rs | 197 +++++++++++++++++++++++++++++ crates/obscura-net/src/lib.rs | 2 + 6 files changed, 235 insertions(+), 8 deletions(-) create mode 100644 crates/obscura-net/src/encoding.rs diff --git a/Cargo.lock b/Cargo.lock index 95093c1..0ef920d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -740,6 +740,15 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1699,6 +1708,7 @@ name = "obscura-net" version = "0.1.0" dependencies = [ "async-trait", + "encoding_rs", "reqwest", "serde", "serde_json", @@ -2165,7 +2175,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2564,10 +2574,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs index 0a8067a..a3e2b43 100644 --- a/crates/obscura-browser/src/page.rs +++ b/crates/obscura-browser/src/page.rs @@ -402,7 +402,9 @@ impl Page { let mut fetched: std::collections::HashMap = std::collections::HashMap::new(); for result in fetch_results { if let Some((idx, url, resp)) = result { - let code = String::from_utf8_lossy(&resp.body).to_string(); + // Script bodies: only the HTTP Content-Type charset matters + // (no in-band meta-charset for JS). + let code = obscura_net::decode_non_html(&resp.body, resp.content_type()); fetched.insert(idx, (url, code, resp)); } } @@ -650,7 +652,11 @@ impl Page { self.url = Some(response.url.clone()); } - let body_text = String::from_utf8_lossy(&response.body).to_string(); + // Honor the response charset: HTTP Content-Type → sniff + // in the first 1KB → UTF-8 fallback. Without this, every non-UTF-8 + // page (GBK, Big5, Shift-JIS, Windows-125x, EUC-KR, ISO-8859-x) + // came through as replacement characters. + let body_text = obscura_net::decode_response(&response.body, response.content_type()); let dom = parse_html(&body_text); self.title = dom @@ -718,7 +724,9 @@ impl Page { let mut css_sources = Vec::new(); for result in css_results { if let Some((url_str, resp)) = result { - let css = String::from_utf8_lossy(&resp.body).to_string(); + // CSS bodies: honor the Content-Type charset; CSS @charset is + // out of scope for the current scrape-focused pipeline. + let css = obscura_net::decode_non_html(&resp.body, resp.content_type()); self.record_network_event(&url_str, "GET", "Stylesheet", resp.status, &resp.headers, resp.body.len()); css_sources.push(css); } diff --git a/crates/obscura-net/Cargo.toml b/crates/obscura-net/Cargo.toml index c1889fa..328bff8 100644 --- a/crates/obscura-net/Cargo.toml +++ b/crates/obscura-net/Cargo.toml @@ -16,6 +16,7 @@ serde_json = { workspace = true } tracing = { workspace = true } thiserror = { workspace = true } async-trait = "0.1" +encoding_rs = "0.8" wreq-util = { version = "3.0.0-rc.10", optional = true } # `prefix-symbols` renames BoringSSL exports to avoid clashes with system OpenSSL, diff --git a/crates/obscura-net/src/client.rs b/crates/obscura-net/src/client.rs index 40004fb..c971858 100644 --- a/crates/obscura-net/src/client.rs +++ b/crates/obscura-net/src/client.rs @@ -21,8 +21,17 @@ pub struct Response { } impl Response { - pub fn text(&self) -> Result { - String::from_utf8(self.body.clone()) + /// Decode the body as text, honoring the response charset. + /// + /// Uses the HTTP `Content-Type` header's `charset=` parameter, then for + /// HTML responses falls back to sniffing `` in the first + /// 1KB, then UTF-8. Mirrors browser behaviour per the HTML5 spec. + pub fn text(&self) -> String { + if self.is_html() { + crate::encoding::decode_response(&self.body, self.content_type()) + } else { + crate::encoding::decode_non_html(&self.body, self.content_type()) + } } pub fn header(&self, name: &str) -> Option<&str> { diff --git a/crates/obscura-net/src/encoding.rs b/crates/obscura-net/src/encoding.rs new file mode 100644 index 0000000..dffe2ce --- /dev/null +++ b/crates/obscura-net/src/encoding.rs @@ -0,0 +1,197 @@ +//! Charset detection and decoding for HTTP response bodies. +//! +//! Issue #113: obscura used to call `String::from_utf8_lossy` on every +//! response body, which silently corrupts every non-UTF-8 page (GBK, Big5, +//! Shift-JIS, Windows-125x, EUC-KR, ISO-8859-x). Picking the right decoder +//! is required for scraping non-Latin sites at all. +//! +//! Detection order, mirroring real browsers (HTML5 spec § 8.2.2.4): +//! 1. `Content-Type: text/html; charset=...` from the HTTP response header. +//! 2. `` or `` +//! sniffed from the first 1024 bytes of the body. +//! 3. Default UTF-8. +//! +//! For non-HTML resources (JS, CSS, JSON), only steps 1 and 3 apply. + +use encoding_rs::{Encoding, UTF_8}; + +/// Decode an HTTP response body. `content_type_header` is the raw header +/// value if present (e.g. `text/html; charset=gbk`). For HTML resources, +/// the parser also sniffs `` in the first 1KB. +pub fn decode_response(bytes: &[u8], content_type_header: Option<&str>) -> String { + let (encoding, _) = detect_encoding(bytes, content_type_header); + let (cow, _, _) = encoding.decode(bytes); + cow.into_owned() +} + +/// Same as `decode_response` but skips the `` sniff. Use for +/// non-HTML resources where embedded HTML meta tags are not authoritative +/// (script and style bodies, JSON, plain text). +pub fn decode_non_html(bytes: &[u8], content_type_header: Option<&str>) -> String { + let encoding = content_type_header + .and_then(charset_from_content_type) + .and_then(|name| Encoding::for_label(name.as_bytes())) + .unwrap_or(UTF_8); + let (cow, _, _) = encoding.decode(bytes); + cow.into_owned() +} + +/// Resolve the encoding to use for an HTML response, mirroring the HTML5 +/// detection order. Returns the encoding and a tag describing where it was +/// picked from (for logging / tests). +pub fn detect_encoding<'a>( + bytes: &'a [u8], + content_type_header: Option<&str>, +) -> (&'static Encoding, &'static str) { + if let Some(charset) = content_type_header.and_then(charset_from_content_type) { + if let Some(enc) = Encoding::for_label(charset.as_bytes()) { + return (enc, "content-type"); + } + } + if let Some(enc) = sniff_meta_charset(bytes) { + return (enc, "meta-charset"); + } + (UTF_8, "default-utf8") +} + +/// Pull the `charset=` parameter out of a Content-Type header value. +fn charset_from_content_type(header: &str) -> Option { + for part in header.split(';') { + let trimmed = part.trim(); + if let Some(rest) = trimmed.strip_prefix("charset=").or_else(|| trimmed.strip_prefix("Charset=")) { + // Strip surrounding quotes if present. + let value = rest.trim_matches(|c: char| c == '"' || c == '\'').trim(); + if !value.is_empty() { + return Some(value.to_ascii_lowercase()); + } + } + // Some servers send `Content-Type: text/html; CHARSET = gbk`. + let lower = trimmed.to_ascii_lowercase(); + if let Some(rest) = lower.strip_prefix("charset") { + let rest = rest.trim_start(); + if let Some(rest) = rest.strip_prefix('=') { + let value = rest.trim().trim_matches(|c: char| c == '"' || c == '\''); + if !value.is_empty() { + return Some(value.to_ascii_lowercase()); + } + } + } + } + None +} + +/// Scan the first 1024 bytes for a `` or +/// `` declaration. +/// We only look at ASCII bytes; valid meta-charset declarations are always +/// ASCII regardless of the page's actual encoding. +fn sniff_meta_charset(bytes: &[u8]) -> Option<&'static Encoding> { + let prefix_len = bytes.len().min(1024); + let prefix = &bytes[..prefix_len]; + // Lossy is fine: any meta charset attribute is ASCII, even on a non-UTF-8 page. + let s = String::from_utf8_lossy(prefix).to_ascii_lowercase(); + + // Look for any `` pattern in the first 1KB. We + // intentionally accept both the modern shorthand (``) + // and the legacy http-equiv form (``). + let mut pos = 0; + while let Some(meta_start) = s[pos..].find("` for this meta tag. + let end = s[abs..].find('>').map(|e| abs + e).unwrap_or(s.len()); + let tag = &s[abs..end]; + + if let Some(charset_pos) = tag.find("charset") { + let after = &tag[charset_pos + "charset".len()..]; + let after = after.trim_start(); + if let Some(eq_rest) = after.strip_prefix('=') { + let value = eq_rest + .trim_start() + .trim_start_matches(|c: char| c == '"' || c == '\'') + .split(|c: char| c == '"' || c == '\'' || c == ';' || c.is_whitespace() || c == '/') + .next() + .unwrap_or(""); + if !value.is_empty() { + if let Some(enc) = Encoding::for_label(value.as_bytes()) { + return Some(enc); + } + } + } + } + + pos = end + 1; + if pos >= s.len() { + break; + } + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn content_type_charset_wins() { + let bytes = b""; + let (enc, source) = detect_encoding(bytes, Some("text/html; charset=gbk")); + assert_eq!(enc.name(), "GBK"); + assert_eq!(source, "content-type"); + } + + #[test] + fn content_type_quoted_charset_is_parsed() { + let (enc, _) = detect_encoding(b"", Some("text/html; charset=\"Shift_JIS\"")); + assert_eq!(enc.name(), "Shift_JIS"); + } + + #[test] + fn meta_charset_used_when_header_missing() { + let bytes = b""; + let (enc, source) = detect_encoding(bytes, None); + assert_eq!(enc.name(), "Big5"); + assert_eq!(source, "meta-charset"); + } + + #[test] + fn meta_http_equiv_charset_is_recognized() { + let bytes = b""; + let (enc, _) = detect_encoding(bytes, None); + assert_eq!(enc.name(), "EUC-KR"); + } + + #[test] + fn no_charset_anywhere_falls_back_to_utf8() { + let bytes = b"hello"; + let (enc, source) = detect_encoding(bytes, None); + assert_eq!(enc.name(), "UTF-8"); + assert_eq!(source, "default-utf8"); + } + + #[test] + fn decode_response_gbk_bytes_roundtrip() { + // "你好" (ni hao) encoded as GBK = C4 E3 BA C3 + let bytes: &[u8] = &[0xC4, 0xE3, 0xBA, 0xC3]; + let s = decode_response(bytes, Some("text/html; charset=gbk")); + assert_eq!(s, "你好"); + } + + #[test] + fn decode_non_html_skips_meta_sniff() { + // A JS body that happens to contain a string `` + // must NOT be decoded as GBK — non-HTML resources only honor the + // HTTP header. + let bytes = br#"var x = ''; // not the real charset"#; + let s = decode_non_html(bytes, Some("application/javascript")); + assert!(s.contains(""); + let (enc, _) = detect_encoding(&bytes, None); + // Beyond 1KB: ignored, fall back to UTF-8. + assert_eq!(enc.name(), "UTF-8"); + } +} diff --git a/crates/obscura-net/src/lib.rs b/crates/obscura-net/src/lib.rs index 6bea89b..b319d08 100644 --- a/crates/obscura-net/src/lib.rs +++ b/crates/obscura-net/src/lib.rs @@ -1,5 +1,6 @@ pub mod client; pub mod cookies; +pub mod encoding; pub mod interceptor; pub mod robots; pub mod blocklist; @@ -8,6 +9,7 @@ pub mod wreq_client; pub use client::{ObscuraHttpClient, ObscuraNetError, RequestInfo, ResourceType, Response}; pub use cookies::{CookieInfo, CookieJar}; +pub use encoding::{decode_non_html, decode_response}; pub use robots::RobotsCache; pub use blocklist::is_blocked as is_tracker_blocked; #[cfg(feature = "stealth")]