Implement document title and referrer semantics

crates/obscura-browser/src/page.rs

@@ -117,6 +117,31 @@ fn subresource_allowed(page_url: Option<&Url>, resource: &str) -> bool {
     }
 }
 
+/// Compute the default `strict-origin-when-cross-origin` referrer value used
+/// for a document-initiated navigation. Direct navigations bypass this helper
+/// and use an empty referrer. Referrer-Policy overrides are not yet plumbed
+/// through the navigation request.
+fn navigation_referrer(source: &Url, target: &Url) -> String {
+    if !matches!(source.scheme(), "http" | "https")
+        || !matches!(target.scheme(), "http" | "https")
+        || (source.scheme() == "https" && target.scheme() == "http")
+    {
+        return String::new();
+    }
+
+    if source.origin() == target.origin() {
+        let mut sanitized = source.clone();
+        sanitized.set_fragment(None);
+        let _ = sanitized.set_username("");
+        let _ = sanitized.set_password(None);
+        return sanitized.to_string();
+    }
+
+    let mut origin = source.origin().ascii_serialization();
+    origin.push('/');
+    origin
+}
+
 /// Escape a value for safe inclusion inside a JavaScript template
 /// literal. The previous implementation only escaped `\`, `` ` `` and
 /// `${`; that left U+2028 / U+2029 (the JS-specific line terminators)
@@ -171,6 +196,10 @@ pub struct Page {
     pub http_client: Arc<ObscuraHttpClient>,
     pub context: Arc<BrowserContext>,
     pub title: String,
+    /// Source document URL for the current document. This is deliberately
+    /// separate from `url`: direct automation navigations have no referrer,
+    /// while a navigation requested by page script uses the previous document.
+    pub referrer: String,
     /// CSS viewport used by responsive page JavaScript and CDP screenshots.
     /// The physical `screen` fingerprint remains independent.
     pub viewport: (f32, f32),
@@ -547,6 +576,7 @@ impl Page {
             http_client,
             context,
             title: String::new(),
+            referrer: String::new(),
             viewport: (1280.0, 720.0),
             encoding: "UTF-8".to_string(),
             history: Vec::new(),
@@ -637,6 +667,7 @@ impl Page {
         rt.set_url(&self.url_string());
         rt.set_encoding(&self.encoding);
         rt.set_title(&self.title);
+        rt.set_referrer(&self.referrer);
 
         #[cfg(feature = "stealth")]
         if self.stealth_client.is_some() {
@@ -1400,7 +1431,7 @@ impl Page {
 
         let result = match tokio::time::timeout(
             nav_timeout,
-            self.navigate_with_wait_post_inner(url_str, wait_until, method, body),
+            self.navigate_with_wait_post_inner(url_str, wait_until, method, body, ""),
         )
         .await
         {
@@ -1481,13 +1512,22 @@ impl Page {
         wait_until: crate::lifecycle::WaitUntil,
         method: &str,
         body: &str,
+        initial_referrer: &str,
     ) -> Result<(), PageError> {
         let mut current_url = url_str.to_string();
         let mut current_method = method.to_string();
         let mut current_body = body.to_string();
+        let mut document_referrer = initial_referrer.to_string();
         const REDIRECT_LIMIT: usize = 10;
         for chain in 0..REDIRECT_LIMIT {
-            self.navigate_single(&current_url, wait_until, &current_method, &current_body).await?;
+            self.navigate_single(
+                &current_url,
+                wait_until,
+                &current_method,
+                &current_body,
+                &document_referrer,
+            )
+            .await?;
             if let Some((next_url, next_method, next_body)) = self.take_pending_navigation() {
                 if cross_scheme_to_file(&current_url, &next_url) {
                     // SOP gate. A web page must not be able to drive
@@ -1504,6 +1544,15 @@ impl Page {
                     break;
                 }
                 tracing::info!("JS-triggered navigation chain: {} {} -> {}", current_method, current_url, next_url);
+                document_referrer = self
+                    .url
+                    .as_ref()
+                    .and_then(|source| {
+                        Url::parse(&next_url)
+                            .ok()
+                            .map(|target| navigation_referrer(source, &target))
+                    })
+                    .unwrap_or_default();
                 current_url = next_url;
                 current_method = next_method;
                 current_body = next_body;
@@ -1527,10 +1576,12 @@ impl Page {
         wait_until: crate::lifecycle::WaitUntil,
         method: &str,
         body: &str,
+        referrer: &str,
     ) -> Result<(), PageError> {
         let url = Url::parse(url_str).map_err(|e| PageError::InvalidUrl(e.to_string()))?;
 
         self.lifecycle = LifecycleState::Loading;
+        self.referrer = referrer.to_string();
         self.url = Some(url.clone());
         self.network_events.clear();
 
@@ -2322,13 +2373,38 @@ impl Page {
 
     pub async fn process_pending_navigation(&mut self) -> Result<bool, PageError> {
         if let Some((url, method, body)) = self.take_pending_navigation() {
-            self.navigate_with_wait_post(
+            let source_url = self
-                &url,
+                .url
-                crate::lifecycle::WaitUntil::Load,
+                .as_ref()
-                &method,
+                .and_then(|source| {
-                &body,
+                    Url::parse(&url)
+                        .ok()
+                        .map(|target| navigation_referrer(source, &target))
+                })
+                .unwrap_or_default();
+            let nav_timeout_ms: u64 = std::env::var("OBSCURA_NAV_TIMEOUT_MS")
+                .ok()
+                .and_then(|s| s.parse().ok())
+                .unwrap_or(30_000);
+            let result = tokio::time::timeout(
+                tokio::time::Duration::from_millis(nav_timeout_ms),
+                self.navigate_with_wait_post_inner(
+                    &url,
+                    crate::lifecycle::WaitUntil::Load,
+                    &method,
+                    &body,
+                    &source_url,
+                ),
             )
-            .await?;
+            .await
+            .map_err(|_| {
+                self.lifecycle = crate::lifecycle::LifecycleState::Failed;
+                PageError::NetworkError(format!(
+                    "navigation exceeded {nav_timeout_ms}ms deadline"
+                ))
+            })?;
+            result?;
+            self.push_history(self.url_string());
             Ok(true)
         } else {
             Ok(false)
@@ -2385,12 +2461,97 @@ fn url_matches_cdp_pattern(pattern: &str, url: &str) -> bool {
 mod tests {
     use super::{
         linked_stylesheet_requests, materialize_linked_stylesheet_script, parse_import_url,
-        rebase_css_urls, split_css_imports,
+        navigation_referrer, rebase_css_urls, split_css_imports,
         script_response_is_executable, truncate_on_char_boundary, url_matches_cdp_pattern,
     };
     use base64::Engine as _;
     use obscura_dom::parse_html;
 
+    #[test]
+    fn default_navigation_referrer_matches_strict_origin_when_cross_origin() {
+        let source = url::Url::parse("https://user:[email protected]/path?q=1#fragment")
+            .unwrap();
+        let same_origin = url::Url::parse("https://source.example/next").unwrap();
+        let cross_origin = url::Url::parse("https://target.example/next").unwrap();
+        let downgrade = url::Url::parse("http://source.example/next").unwrap();
+
+        assert_eq!(
+            navigation_referrer(&source, &same_origin),
+            "https://source.example/path?q=1"
+        );
+        assert_eq!(
+            navigation_referrer(&source, &cross_origin),
+            "https://source.example/"
+        );
+        assert_eq!(navigation_referrer(&source, &downgrade), "");
+
+        let data_source = url::Url::parse("data:text/html,source").unwrap();
+        assert_eq!(navigation_referrer(&data_source, &cross_origin), "");
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn document_navigation_referrer_survives_http_redirects() {
+        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap();
+        std::thread::spawn(move || {
+            use std::io::{Read as _, Write as _};
+
+            for _ in 0..3 {
+                let (mut stream, _) = listener.accept().unwrap();
+                let mut request = [0u8; 2048];
+                let length = stream.read(&mut request).unwrap();
+                let request_text = String::from_utf8_lossy(&request[..length]);
+                let path = request_text
+                    .lines()
+                    .next()
+                    .and_then(|line| line.split_ascii_whitespace().nth(1))
+                    .unwrap_or("/");
+                let response = match path {
+                    "/source" => {
+                        let body = "<script>location.href='/redirect'</script>";
+                        format!(
+                            "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+                            body.len(),
+                        )
+                    }
+                    "/redirect" => "HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(),
+                    "/final" => {
+                        let body = "<!doctype html><title>final</title>";
+                        format!(
+                            "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+                            body.len(),
+                        )
+                    }
+                    _ => "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(),
+                };
+                stream.write_all(response.as_bytes()).unwrap();
+            }
+        });
+
+        let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network(
+            "referrer-redirect".to_string(),
+            None,
+            false,
+            None,
+            None,
+            true,
+        ));
+        let mut page = super::Page::new("referrer-redirect".to_string(), context);
+        let source = format!("http://{address}/source");
+        page.navigate(&source).await.unwrap();
+
+        let observed = page
+            .js
+            .as_mut()
+            .unwrap()
+            .evaluate("[document.URL, document.referrer]")
+            .unwrap();
+        assert_eq!(
+            observed,
+            serde_json::json!([format!("http://{address}/final"), source])
+        );
+    }
+
     fn client_replacement_page(name: &str, deferred: bool) -> super::Page {
         let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network(
             name.to_string(), None, false, None, None, true,

crates/obscura-js/js/bootstrap.js

@@ -3456,9 +3456,25 @@ class Document extends Node {
     return this._doctype;
   }
   get title() { return _domParse("document_title") ?? ""; }
-  set title(v) {}
+  set title(v) {
+    const value = String(v);
+    let title = this.querySelector("title");
+    if (!title) {
+      let head = this.head;
+      const root = this.documentElement;
+      if (!head && root) {
+        head = this.createElement("head");
+        root.insertBefore(head, this.body);
+      }
+      if (!head) return;
+      title = this.createElement("title");
+      head.appendChild(title);
+    }
+    title.textContent = value;
+  }
   get URL() { return _domParse("document_url") ?? ""; }
   get documentURI() { return this.URL; }
+  get referrer() { return _domParse("document_referrer") ?? ""; }
   get location() { return globalThis.location; }
   set location(url) { Deno.core.ops.op_navigate(_resolveUrl(String(url)), 'GET', ''); }
   get defaultView() { return globalThis; }
@@ -7183,7 +7199,20 @@ globalThis.DOMParser = class DOMParser {
       get head() { return findByTagName("HEAD"); },
       get title() {
         const t = findByTagName("TITLE");
-        return t ? (t.textContent || "") : "";
+        return t ? (t.textContent || "").replace(/[\t\n\f\r ]+/g, " ").trim() : "";
+      },
+      set title(value) {
+        let t = findByTagName("TITLE");
+        if (!t) {
+          let head = findByTagName("HEAD");
+          if (!head) {
+            head = document.createElement("head");
+            root.insertBefore(head, findByTagName("BODY"));
+          }
+          t = document.createElement("title");
+          head.appendChild(t);
+        }
+        t.textContent = String(value);
       },
       get firstChild() { return root; },
       get lastChild() { return root; },
@@ -7193,6 +7222,7 @@ globalThis.DOMParser = class DOMParser {
       // URL about:blank, are already fully parsed, and carry no stylesheets.
       get URL() { return "about:blank"; },
       get documentURI() { return "about:blank"; },
+      get referrer() { return ""; },
       get baseURI() { return "about:blank"; },
       get compatMode() { return "CSS1Compat"; },
       get characterSet() { return "UTF-8"; },

crates/obscura-js/src/ops.rs

@@ -74,6 +74,10 @@ pub struct ObscuraState {
     /// encoding override for `<a>`/`<area>` hrefs in legacy-charset documents.
     pub encoding: String,
     pub title: String,
+    /// URL of the document that initiated this document's navigation. Direct
+    /// browser/API navigations leave this empty; document-initiated
+    /// navigations set it to the source document URL.
+    pub referrer: String,
     pub blocked_urls: Vec<String>,
     pub cookie_jar: Option<Arc<CookieJar>>,
     pub http_client: Option<Arc<ObscuraHttpClient>>,
@@ -141,6 +145,7 @@ impl ObscuraState {
             url: "about:blank".to_string(),
             encoding: "UTF-8".to_string(),
             title: String::new(),
+            referrer: String::new(),
             blocked_urls: Vec::new(),
             cookie_jar: None,
             http_client: None,
@@ -350,8 +355,26 @@ fn op_dom_inner(state: &OpState, cmd: String, arg1: String, arg2: String) -> Str
 
     match cmd.as_str() {
         "document_node_id" => dom.document().index().to_string(),
-        "document_title" => serde_json::to_string(&gs.title).unwrap_or("\"\"".into()),
+        "document_title" => {
+            // The DOM is authoritative after parsing. In particular, script
+            // changes through title.textContent must be reflected by
+            // document.title, not hidden behind the navigation-time snapshot.
+            let title = dom
+                .query_selector("title")
+                .ok()
+                .flatten()
+                .map(|title_id| {
+                    dom.text_content(title_id)
+                        .split(|ch| matches!(ch, '\t' | '\n' | '\u{000C}' | '\r' | ' '))
+                        .filter(|part| !part.is_empty())
+                        .collect::<Vec<_>>()
+                        .join(" ")
+                })
+                .unwrap_or_default();
+            serde_json::to_string(&title).unwrap_or("\"\"".into())
+        }
         "document_url" => serde_json::to_string(&gs.url).unwrap_or("\"\"".into()),
+        "document_referrer" => serde_json::to_string(&gs.referrer).unwrap_or("\"\"".into()),
         "document_encoding" => serde_json::to_string(&gs.encoding).unwrap_or("\"UTF-8\"".into()),
         "document_element" => {
             for cid in dom.children(dom.document()) {

crates/obscura-js/src/runtime.rs

@@ -255,6 +255,14 @@ impl ObscuraJsRuntime {
         self.state.borrow_mut().title = title.to_string();
     }
 
+    /// Set the source document URL exposed as `document.referrer`. Navigation
+    /// owns this value; it is not derived from the current URL because direct
+    /// navigations and document-initiated navigations have different
+    /// referrer semantics.
+    pub fn set_referrer(&self, referrer: &str) {
+        self.state.borrow_mut().referrer = referrer.to_string();
+    }
+
     pub fn set_blocked_urls(&self, patterns: Vec<String>) {
         self.state.borrow_mut().blocked_urls = patterns;
     }
@@ -2244,7 +2252,91 @@ mod tests {
     fn test_document_title() {
         let mut rt = setup_runtime("<html><head><title>Test</title></head><body></body></html>");
         let title = rt.evaluate("document.title").unwrap();
-        assert_eq!(title, serde_json::json!("Test Page"));
+        assert_eq!(title, serde_json::json!("Test"));
+
+        let result = rt
+            .evaluate(
+                r#"
+                (function() {
+                  document.title = "A <new> title";
+                  return [
+                    document.title,
+                    document.querySelector("head > title").textContent,
+                    document.querySelectorAll("title").length
+                  ];
+                })()
+                "#,
+            )
+            .unwrap();
+        assert_eq!(
+            result,
+            serde_json::json!(["A <new> title", "A <new> title", 1])
+        );
+
+        let normalized = rt
+            .evaluate(
+                r#"
+                (function() {
+                  document.querySelector("title").textContent = "  live\n\tDOM   title  ";
+                  return document.title;
+                })()
+                "#,
+            )
+            .unwrap();
+        assert_eq!(normalized, serde_json::json!("live DOM title"));
+    }
+
+    #[test]
+    fn document_title_setter_creates_missing_title_element() {
+        let mut rt = setup_runtime("<html><body><main>content</main></body></html>");
+        let result = rt
+            .evaluate(
+                r#"
+                (function() {
+                  document.title = "Created";
+                  return [
+                    document.title,
+                    document.head.tagName,
+                    document.head.firstElementChild.tagName,
+                    document.head.firstElementChild.textContent,
+                    document.documentElement.firstElementChild === document.head
+                  ];
+                })()
+                "#,
+            )
+            .unwrap();
+        assert_eq!(
+            result,
+            serde_json::json!(["Created", "HEAD", "TITLE", "Created", true])
+        );
+
+        let detached = rt
+            .evaluate(
+                r#"
+                (function() {
+                  const doc = document.implementation.createHTMLDocument();
+                  doc.title = "  Detached   title  ";
+                  return [doc.title, doc.querySelector("title").textContent, doc.referrer];
+                })()
+                "#,
+            )
+            .unwrap();
+        assert_eq!(
+            detached,
+            serde_json::json!(["Detached title", "  Detached   title  ", ""])
+        );
+    }
+
+    #[test]
+    fn document_referrer_has_explicit_navigation_state() {
+        let mut rt = setup_runtime("<html><body></body></html>");
+        assert_eq!(rt.evaluate("document.referrer").unwrap(), serde_json::json!(""));
+
+        rt.set_referrer("https://source.example/path?q=1");
+        assert_eq!(
+            rt.evaluate("document.referrer").unwrap(),
+            serde_json::json!("https://source.example/path?q=1")
+        );
     }
 
     #[test]
@@ -6052,7 +6144,7 @@ mod tests {
         let result = rt
             .call_function_on("() => document.title", None, &[], true)
             .await.unwrap();
-        assert_eq!(result.value.unwrap(), serde_json::json!("Test Page"));
+        assert_eq!(result.value.unwrap(), serde_json::json!("Test"));
     }
 
     #[tokio::test(flavor = "current_thread")]