From edb1785313e3b404120cefda27e70cd9e9e7287c Mon Sep 17 00:00:00 2001 From: SGavrl <277843151+SGavrl@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:46:11 +0000 Subject: [PATCH] Implement document title and referrer semantics --- crates/obscura-browser/src/page.rs | 179 +++++++++++++++++++++++++++-- crates/obscura-js/js/bootstrap.js | 34 +++++- crates/obscura-js/src/ops.rs | 25 +++- crates/obscura-js/src/runtime.rs | 96 +++++++++++++++- 4 files changed, 320 insertions(+), 14 deletions(-) diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs index 04e7849..b95ab64 100644 --- a/crates/obscura-browser/src/page.rs +++ b/crates/obscura-browser/src/page.rs @@ -117,6 +117,31 @@ fn subresource_allowed(page_url: Option<&Url>, resource: &str) -> bool { } } +/// Compute the default `strict-origin-when-cross-origin` referrer value used +/// for a document-initiated navigation. Direct navigations bypass this helper +/// and use an empty referrer. Referrer-Policy overrides are not yet plumbed +/// through the navigation request. +fn navigation_referrer(source: &Url, target: &Url) -> String { + if !matches!(source.scheme(), "http" | "https") + || !matches!(target.scheme(), "http" | "https") + || (source.scheme() == "https" && target.scheme() == "http") + { + return String::new(); + } + + if source.origin() == target.origin() { + let mut sanitized = source.clone(); + sanitized.set_fragment(None); + let _ = sanitized.set_username(""); + let _ = sanitized.set_password(None); + return sanitized.to_string(); + } + + let mut origin = source.origin().ascii_serialization(); + origin.push('/'); + origin +} + /// Escape a value for safe inclusion inside a JavaScript template /// literal. The previous implementation only escaped `\`, `` ` `` and /// `${`; that left U+2028 / U+2029 (the JS-specific line terminators) @@ -171,6 +196,10 @@ pub struct Page { pub http_client: Arc, pub context: Arc, pub title: String, + /// Source document URL for the current document. This is deliberately + /// separate from `url`: direct automation navigations have no referrer, + /// while a navigation requested by page script uses the previous document. + pub referrer: String, /// CSS viewport used by responsive page JavaScript and CDP screenshots. /// The physical `screen` fingerprint remains independent. pub viewport: (f32, f32), @@ -547,6 +576,7 @@ impl Page { http_client, context, title: String::new(), + referrer: String::new(), viewport: (1280.0, 720.0), encoding: "UTF-8".to_string(), history: Vec::new(), @@ -637,6 +667,7 @@ impl Page { rt.set_url(&self.url_string()); rt.set_encoding(&self.encoding); rt.set_title(&self.title); + rt.set_referrer(&self.referrer); #[cfg(feature = "stealth")] if self.stealth_client.is_some() { @@ -1400,7 +1431,7 @@ impl Page { let result = match tokio::time::timeout( nav_timeout, - self.navigate_with_wait_post_inner(url_str, wait_until, method, body), + self.navigate_with_wait_post_inner(url_str, wait_until, method, body, ""), ) .await { @@ -1481,13 +1512,22 @@ impl Page { wait_until: crate::lifecycle::WaitUntil, method: &str, body: &str, + initial_referrer: &str, ) -> Result<(), PageError> { let mut current_url = url_str.to_string(); let mut current_method = method.to_string(); let mut current_body = body.to_string(); + let mut document_referrer = initial_referrer.to_string(); const REDIRECT_LIMIT: usize = 10; for chain in 0..REDIRECT_LIMIT { - self.navigate_single(¤t_url, wait_until, ¤t_method, ¤t_body).await?; + self.navigate_single( + ¤t_url, + wait_until, + ¤t_method, + ¤t_body, + &document_referrer, + ) + .await?; if let Some((next_url, next_method, next_body)) = self.take_pending_navigation() { if cross_scheme_to_file(¤t_url, &next_url) { // SOP gate. A web page must not be able to drive @@ -1504,6 +1544,15 @@ impl Page { break; } tracing::info!("JS-triggered navigation chain: {} {} -> {}", current_method, current_url, next_url); + document_referrer = self + .url + .as_ref() + .and_then(|source| { + Url::parse(&next_url) + .ok() + .map(|target| navigation_referrer(source, &target)) + }) + .unwrap_or_default(); current_url = next_url; current_method = next_method; current_body = next_body; @@ -1527,10 +1576,12 @@ impl Page { wait_until: crate::lifecycle::WaitUntil, method: &str, body: &str, + referrer: &str, ) -> Result<(), PageError> { let url = Url::parse(url_str).map_err(|e| PageError::InvalidUrl(e.to_string()))?; self.lifecycle = LifecycleState::Loading; + self.referrer = referrer.to_string(); self.url = Some(url.clone()); self.network_events.clear(); @@ -2322,13 +2373,38 @@ impl Page { pub async fn process_pending_navigation(&mut self) -> Result { if let Some((url, method, body)) = self.take_pending_navigation() { - self.navigate_with_wait_post( - &url, - crate::lifecycle::WaitUntil::Load, - &method, - &body, + let source_url = self + .url + .as_ref() + .and_then(|source| { + Url::parse(&url) + .ok() + .map(|target| navigation_referrer(source, &target)) + }) + .unwrap_or_default(); + let nav_timeout_ms: u64 = std::env::var("OBSCURA_NAV_TIMEOUT_MS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(30_000); + let result = tokio::time::timeout( + tokio::time::Duration::from_millis(nav_timeout_ms), + self.navigate_with_wait_post_inner( + &url, + crate::lifecycle::WaitUntil::Load, + &method, + &body, + &source_url, + ), ) - .await?; + .await + .map_err(|_| { + self.lifecycle = crate::lifecycle::LifecycleState::Failed; + PageError::NetworkError(format!( + "navigation exceeded {nav_timeout_ms}ms deadline" + )) + })?; + result?; + self.push_history(self.url_string()); Ok(true) } else { Ok(false) @@ -2385,12 +2461,97 @@ fn url_matches_cdp_pattern(pattern: &str, url: &str) -> bool { mod tests { use super::{ linked_stylesheet_requests, materialize_linked_stylesheet_script, parse_import_url, - rebase_css_urls, split_css_imports, + navigation_referrer, rebase_css_urls, split_css_imports, script_response_is_executable, truncate_on_char_boundary, url_matches_cdp_pattern, }; use base64::Engine as _; use obscura_dom::parse_html; + #[test] + fn default_navigation_referrer_matches_strict_origin_when_cross_origin() { + let source = url::Url::parse("https://user:pass@source.example/path?q=1#fragment") + .unwrap(); + let same_origin = url::Url::parse("https://source.example/next").unwrap(); + let cross_origin = url::Url::parse("https://target.example/next").unwrap(); + let downgrade = url::Url::parse("http://source.example/next").unwrap(); + + assert_eq!( + navigation_referrer(&source, &same_origin), + "https://source.example/path?q=1" + ); + assert_eq!( + navigation_referrer(&source, &cross_origin), + "https://source.example/" + ); + assert_eq!(navigation_referrer(&source, &downgrade), ""); + + let data_source = url::Url::parse("data:text/html,source").unwrap(); + assert_eq!(navigation_referrer(&data_source, &cross_origin), ""); + } + + #[tokio::test(flavor = "current_thread")] + async fn document_navigation_referrer_survives_http_redirects() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + std::thread::spawn(move || { + use std::io::{Read as _, Write as _}; + + for _ in 0..3 { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 2048]; + let length = stream.read(&mut request).unwrap(); + let request_text = String::from_utf8_lossy(&request[..length]); + let path = request_text + .lines() + .next() + .and_then(|line| line.split_ascii_whitespace().nth(1)) + .unwrap_or("/"); + let response = match path { + "/source" => { + let body = ""; + format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len(), + ) + } + "/redirect" => "HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(), + "/final" => { + let body = "final"; + format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len(), + ) + } + _ => "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(), + }; + stream.write_all(response.as_bytes()).unwrap(); + } + }); + + let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network( + "referrer-redirect".to_string(), + None, + false, + None, + None, + true, + )); + let mut page = super::Page::new("referrer-redirect".to_string(), context); + let source = format!("http://{address}/source"); + page.navigate(&source).await.unwrap(); + + let observed = page + .js + .as_mut() + .unwrap() + .evaluate("[document.URL, document.referrer]") + .unwrap(); + assert_eq!( + observed, + serde_json::json!([format!("http://{address}/final"), source]) + ); + } + fn client_replacement_page(name: &str, deferred: bool) -> super::Page { let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network( name.to_string(), None, false, None, None, true, diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js index 840a89a..1570f5b 100644 --- a/crates/obscura-js/js/bootstrap.js +++ b/crates/obscura-js/js/bootstrap.js @@ -3456,9 +3456,25 @@ class Document extends Node { return this._doctype; } get title() { return _domParse("document_title") ?? ""; } - set title(v) {} + set title(v) { + const value = String(v); + let title = this.querySelector("title"); + if (!title) { + let head = this.head; + const root = this.documentElement; + if (!head && root) { + head = this.createElement("head"); + root.insertBefore(head, this.body); + } + if (!head) return; + title = this.createElement("title"); + head.appendChild(title); + } + title.textContent = value; + } get URL() { return _domParse("document_url") ?? ""; } get documentURI() { return this.URL; } + get referrer() { return _domParse("document_referrer") ?? ""; } get location() { return globalThis.location; } set location(url) { Deno.core.ops.op_navigate(_resolveUrl(String(url)), 'GET', ''); } get defaultView() { return globalThis; } @@ -7183,7 +7199,20 @@ globalThis.DOMParser = class DOMParser { get head() { return findByTagName("HEAD"); }, get title() { const t = findByTagName("TITLE"); - return t ? (t.textContent || "") : ""; + return t ? (t.textContent || "").replace(/[\t\n\f\r ]+/g, " ").trim() : ""; + }, + set title(value) { + let t = findByTagName("TITLE"); + if (!t) { + let head = findByTagName("HEAD"); + if (!head) { + head = document.createElement("head"); + root.insertBefore(head, findByTagName("BODY")); + } + t = document.createElement("title"); + head.appendChild(t); + } + t.textContent = String(value); }, get firstChild() { return root; }, get lastChild() { return root; }, @@ -7193,6 +7222,7 @@ globalThis.DOMParser = class DOMParser { // URL about:blank, are already fully parsed, and carry no stylesheets. get URL() { return "about:blank"; }, get documentURI() { return "about:blank"; }, + get referrer() { return ""; }, get baseURI() { return "about:blank"; }, get compatMode() { return "CSS1Compat"; }, get characterSet() { return "UTF-8"; }, diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs index 59a95c7..d8f99c4 100644 --- a/crates/obscura-js/src/ops.rs +++ b/crates/obscura-js/src/ops.rs @@ -74,6 +74,10 @@ pub struct ObscuraState { /// encoding override for ``/`` hrefs in legacy-charset documents. pub encoding: String, pub title: String, + /// URL of the document that initiated this document's navigation. Direct + /// browser/API navigations leave this empty; document-initiated + /// navigations set it to the source document URL. + pub referrer: String, pub blocked_urls: Vec, pub cookie_jar: Option>, pub http_client: Option>, @@ -141,6 +145,7 @@ impl ObscuraState { url: "about:blank".to_string(), encoding: "UTF-8".to_string(), title: String::new(), + referrer: String::new(), blocked_urls: Vec::new(), cookie_jar: None, http_client: None, @@ -350,8 +355,26 @@ fn op_dom_inner(state: &OpState, cmd: String, arg1: String, arg2: String) -> Str match cmd.as_str() { "document_node_id" => dom.document().index().to_string(), - "document_title" => serde_json::to_string(&gs.title).unwrap_or("\"\"".into()), + "document_title" => { + // The DOM is authoritative after parsing. In particular, script + // changes through title.textContent must be reflected by + // document.title, not hidden behind the navigation-time snapshot. + let title = dom + .query_selector("title") + .ok() + .flatten() + .map(|title_id| { + dom.text_content(title_id) + .split(|ch| matches!(ch, '\t' | '\n' | '\u{000C}' | '\r' | ' ')) + .filter(|part| !part.is_empty()) + .collect::>() + .join(" ") + }) + .unwrap_or_default(); + serde_json::to_string(&title).unwrap_or("\"\"".into()) + } "document_url" => serde_json::to_string(&gs.url).unwrap_or("\"\"".into()), + "document_referrer" => serde_json::to_string(&gs.referrer).unwrap_or("\"\"".into()), "document_encoding" => serde_json::to_string(&gs.encoding).unwrap_or("\"UTF-8\"".into()), "document_element" => { for cid in dom.children(dom.document()) { diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs index 4cbd4ac..8b2b09e 100644 --- a/crates/obscura-js/src/runtime.rs +++ b/crates/obscura-js/src/runtime.rs @@ -255,6 +255,14 @@ impl ObscuraJsRuntime { self.state.borrow_mut().title = title.to_string(); } + /// Set the source document URL exposed as `document.referrer`. Navigation + /// owns this value; it is not derived from the current URL because direct + /// navigations and document-initiated navigations have different + /// referrer semantics. + pub fn set_referrer(&self, referrer: &str) { + self.state.borrow_mut().referrer = referrer.to_string(); + } + pub fn set_blocked_urls(&self, patterns: Vec) { self.state.borrow_mut().blocked_urls = patterns; } @@ -2244,7 +2252,91 @@ mod tests { fn test_document_title() { let mut rt = setup_runtime("Test"); let title = rt.evaluate("document.title").unwrap(); - assert_eq!(title, serde_json::json!("Test Page")); + assert_eq!(title, serde_json::json!("Test")); + + let result = rt + .evaluate( + r#" + (function() { + document.title = "A title"; + return [ + document.title, + document.querySelector("head > title").textContent, + document.querySelectorAll("title").length + ]; + })() + "#, + ) + .unwrap(); + assert_eq!( + result, + serde_json::json!(["A title", "A title", 1]) + ); + + let normalized = rt + .evaluate( + r#" + (function() { + document.querySelector("title").textContent = " live\n\tDOM title "; + return document.title; + })() + "#, + ) + .unwrap(); + assert_eq!(normalized, serde_json::json!("live DOM title")); + } + + #[test] + fn document_title_setter_creates_missing_title_element() { + let mut rt = setup_runtime("
content
"); + let result = rt + .evaluate( + r#" + (function() { + document.title = "Created"; + return [ + document.title, + document.head.tagName, + document.head.firstElementChild.tagName, + document.head.firstElementChild.textContent, + document.documentElement.firstElementChild === document.head + ]; + })() + "#, + ) + .unwrap(); + assert_eq!( + result, + serde_json::json!(["Created", "HEAD", "TITLE", "Created", true]) + ); + + let detached = rt + .evaluate( + r#" + (function() { + const doc = document.implementation.createHTMLDocument(); + doc.title = " Detached title "; + return [doc.title, doc.querySelector("title").textContent, doc.referrer]; + })() + "#, + ) + .unwrap(); + assert_eq!( + detached, + serde_json::json!(["Detached title", " Detached title ", ""]) + ); + } + + #[test] + fn document_referrer_has_explicit_navigation_state() { + let mut rt = setup_runtime(""); + assert_eq!(rt.evaluate("document.referrer").unwrap(), serde_json::json!("")); + + rt.set_referrer("https://source.example/path?q=1"); + assert_eq!( + rt.evaluate("document.referrer").unwrap(), + serde_json::json!("https://source.example/path?q=1") + ); } #[test] @@ -6052,7 +6144,7 @@ mod tests { let result = rt .call_function_on("() => document.title", None, &[], true) .await.unwrap(); - assert_eq!(result.value.unwrap(), serde_json::json!("Test Page")); + assert_eq!(result.value.unwrap(), serde_json::json!("Test")); } #[tokio::test(flavor = "current_thread")]