thebadking/obscura
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement document title and referrer semantics

Browse Source
This commit is contained in:
SGavrl committed 2026-08-03 13:46:11 +00:00
1 parent b21b194d09
commit edb1785313
4 files changed
+316 -10
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/obscura-browser/src/page.rs
+166 -5
View File
@@ -117,6 +117,31 @@ fn subresource_allowed(page_url: Option<&Url>, resource: &str) -> bool {
}
}
/// Compute the default `strict-origin-when-cross-origin` referrer value used
/// for a document-initiated navigation. Direct navigations bypass this helper
/// and use an empty referrer. Referrer-Policy overrides are not yet plumbed
/// through the navigation request.
fn navigation_referrer(source: &Url, target: &Url) -> String {
if !matches!(source.scheme(), "http" | "https")
|| !matches!(target.scheme(), "http" | "https")
|| (source.scheme() == "https" && target.scheme() == "http")
{
return String::new();
}
if source.origin() == target.origin() {
let mut sanitized = source.clone();
sanitized.set_fragment(None);
let _ = sanitized.set_username("");
let _ = sanitized.set_password(None);
return sanitized.to_string();
}
let mut origin = source.origin().ascii_serialization();
origin.push('/');
origin
}
/// Escape a value for safe inclusion inside a JavaScript template
/// literal. The previous implementation only escaped `\`, `` ` `` and
/// `${`; that left U+2028 / U+2029 (the JS-specific line terminators)
@@ -171,6 +196,10 @@ pub struct Page {
pub http_client: Arc<ObscuraHttpClient>,
pub context: Arc<BrowserContext>,
pub title: String,
/// Source document URL for the current document. This is deliberately
/// separate from `url`: direct automation navigations have no referrer,
/// while a navigation requested by page script uses the previous document.
pub referrer: String,
/// CSS viewport used by responsive page JavaScript and CDP screenshots.
/// The physical `screen` fingerprint remains independent.
pub viewport: (f32, f32),
@@ -547,6 +576,7 @@ impl Page {
http_client,
context,
title: String::new(),
referrer: String::new(),
viewport: (1280.0, 720.0),
encoding: "UTF-8".to_string(),
history: Vec::new(),
@@ -637,6 +667,7 @@ impl Page {
rt.set_url(&self.url_string());
rt.set_encoding(&self.encoding);
rt.set_title(&self.title);
rt.set_referrer(&self.referrer);
#[cfg(feature = "stealth")]
if self.stealth_client.is_some() {
@@ -1400,7 +1431,7 @@ impl Page {
let result = match tokio::time::timeout(
nav_timeout,
self.navigate_with_wait_post_inner(url_str, wait_until, method, body),
self.navigate_with_wait_post_inner(url_str, wait_until, method, body, ""),
)
.await
{
@@ -1481,13 +1512,22 @@ impl Page {
wait_until: crate::lifecycle::WaitUntil,
method: &str,
body: &str,
initial_referrer: &str,
) -> Result<(), PageError> {
let mut current_url = url_str.to_string();
let mut current_method = method.to_string();
let mut current_body = body.to_string();
let mut document_referrer = initial_referrer.to_string();
const REDIRECT_LIMIT: usize = 10;
for chain in 0..REDIRECT_LIMIT {
self.navigate_single(&current_url, wait_until, &current_method, &current_body).await?;
self.navigate_single(
&current_url,
wait_until,
&current_method,
&current_body,
&document_referrer,
)
.await?;
if let Some((next_url, next_method, next_body)) = self.take_pending_navigation() {
if cross_scheme_to_file(&current_url, &next_url) {
// SOP gate. A web page must not be able to drive
@@ -1504,6 +1544,15 @@ impl Page {
break;
}
tracing::info!("JS-triggered navigation chain: {} {} -> {}", current_method, current_url, next_url);
document_referrer = self
.url
.as_ref()
.and_then(|source| {
Url::parse(&next_url)
.ok()
.map(|target| navigation_referrer(source, &target))
})
.unwrap_or_default();
current_url = next_url;
current_method = next_method;
current_body = next_body;
@@ -1527,10 +1576,12 @@ impl Page {
wait_until: crate::lifecycle::WaitUntil,
method: &str,
body: &str,
referrer: &str,
) -> Result<(), PageError> {
let url = Url::parse(url_str).map_err(|e| PageError::InvalidUrl(e.to_string()))?;
self.lifecycle = LifecycleState::Loading;
self.referrer = referrer.to_string();
self.url = Some(url.clone());
self.network_events.clear();
@@ -2322,13 +2373,38 @@ impl Page {
pub async fn process_pending_navigation(&mut self) -> Result<bool, PageError> {
if let Some((url, method, body)) = self.take_pending_navigation() {
self.navigate_with_wait_post(
let source_url = self
.url
.as_ref()
.and_then(|source| {
Url::parse(&url)
.ok()
.map(|target| navigation_referrer(source, &target))
})
.unwrap_or_default();
let nav_timeout_ms: u64 = std::env::var("OBSCURA_NAV_TIMEOUT_MS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(30_000);
let result = tokio::time::timeout(
tokio::time::Duration::from_millis(nav_timeout_ms),
self.navigate_with_wait_post_inner(
&url,
crate::lifecycle::WaitUntil::Load,
&method,
&body,
&source_url,
),
)
.await?;
.await
.map_err(|_| {
self.lifecycle = crate::lifecycle::LifecycleState::Failed;
PageError::NetworkError(format!(
"navigation exceeded {nav_timeout_ms}ms deadline"
))
})?;
result?;
self.push_history(self.url_string());
Ok(true)
} else {
Ok(false)
@@ -2385,12 +2461,97 @@ fn url_matches_cdp_pattern(pattern: &str, url: &str) -> bool {
mod tests {
use super::{
linked_stylesheet_requests, materialize_linked_stylesheet_script, parse_import_url,
rebase_css_urls, split_css_imports,
navigation_referrer, rebase_css_urls, split_css_imports,
script_response_is_executable, truncate_on_char_boundary, url_matches_cdp_pattern,
};
use base64::Engine as _;
use obscura_dom::parse_html;
#[test]
fn default_navigation_referrer_matches_strict_origin_when_cross_origin() {
let source = url::Url::parse("https://user:[email protected]/path?q=1#fragment")
.unwrap();
let same_origin = url::Url::parse("https://source.example/next").unwrap();
let cross_origin = url::Url::parse("https://target.example/next").unwrap();
let downgrade = url::Url::parse("http://source.example/next").unwrap();
assert_eq!(
navigation_referrer(&source, &same_origin),
"https://source.example/path?q=1"
);
assert_eq!(
navigation_referrer(&source, &cross_origin),
"https://source.example/"
);
assert_eq!(navigation_referrer(&source, &downgrade), "");
let data_source = url::Url::parse("data:text/html,source").unwrap();
assert_eq!(navigation_referrer(&data_source, &cross_origin), "");
}
#[tokio::test(flavor = "current_thread")]
async fn document_navigation_referrer_survives_http_redirects() {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
std::thread::spawn(move || {
use std::io::{Read as _, Write as _};
for _ in 0..3 {
let (mut stream, _) = listener.accept().unwrap();
let mut request = [0u8; 2048];
let length = stream.read(&mut request).unwrap();
let request_text = String::from_utf8_lossy(&request[..length]);
let path = request_text
.lines()
.next()
.and_then(|line| line.split_ascii_whitespace().nth(1))
.unwrap_or("/");
let response = match path {
"/source" => {
let body = "<script>location.href='/redirect'</script>";
format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len(),
)
}
"/redirect" => "HTTP/1.1 302 Found\r\nLocation: /final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(),
"/final" => {
let body = "<!doctype html><title>final</title>";
format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len(),
)
}
_ => "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(),
};
stream.write_all(response.as_bytes()).unwrap();
}
});
let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network(
"referrer-redirect".to_string(),
None,
false,
None,
None,
true,
));
let mut page = super::Page::new("referrer-redirect".to_string(), context);
let source = format!("http://{address}/source");
page.navigate(&source).await.unwrap();
let observed = page
.js
.as_mut()
.unwrap()
.evaluate("[document.URL, document.referrer]")
.unwrap();
assert_eq!(
observed,
serde_json::json!([format!("http://{address}/final"), source])
);
}
fn client_replacement_page(name: &str, deferred: bool) -> super::Page {
let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network(
name.to_string(), None, false, None, None, true,
crates/obscura-js/js/bootstrap.js
Vendored
+32 -2
View File
@@ -3456,9 +3456,25 @@ class Document extends Node {
return this._doctype;
}
get title() { return _domParse("document_title") ?? ""; }
set title(v) {}
set title(v) {
const value = String(v);
let title = this.querySelector("title");
if (!title) {
let head = this.head;
const root = this.documentElement;
if (!head && root) {
head = this.createElement("head");
root.insertBefore(head, this.body);
}
if (!head) return;
title = this.createElement("title");
head.appendChild(title);
}
title.textContent = value;
}
get URL() { return _domParse("document_url") ?? ""; }
get documentURI() { return this.URL; }
get referrer() { return _domParse("document_referrer") ?? ""; }
get location() { return globalThis.location; }
set location(url) { Deno.core.ops.op_navigate(_resolveUrl(String(url)), 'GET', ''); }
get defaultView() { return globalThis; }
@@ -7183,7 +7199,20 @@ globalThis.DOMParser = class DOMParser {
get head() { return findByTagName("HEAD"); },
get title() {
const t = findByTagName("TITLE");
return t ? (t.textContent || "") : "";
return t ? (t.textContent || "").replace(/[\t\n\f\r ]+/g, " ").trim() : "";
},
set title(value) {
let t = findByTagName("TITLE");
if (!t) {
let head = findByTagName("HEAD");
if (!head) {
head = document.createElement("head");
root.insertBefore(head, findByTagName("BODY"));
}
t = document.createElement("title");
head.appendChild(t);
}
t.textContent = String(value);
},
get firstChild() { return root; },
get lastChild() { return root; },
@@ -7193,6 +7222,7 @@ globalThis.DOMParser = class DOMParser {
// URL about:blank, are already fully parsed, and carry no stylesheets.
get URL() { return "about:blank"; },
get documentURI() { return "about:blank"; },
get referrer() { return ""; },
get baseURI() { return "about:blank"; },
get compatMode() { return "CSS1Compat"; },
get characterSet() { return "UTF-8"; },
crates/obscura-js/src/ops.rs
+24 -1
View File
@@ -74,6 +74,10 @@ pub struct ObscuraState {
/// encoding override for `<a>`/`<area>` hrefs in legacy-charset documents.
pub encoding: String,
pub title: String,
/// URL of the document that initiated this document's navigation. Direct
/// browser/API navigations leave this empty; document-initiated
/// navigations set it to the source document URL.
pub referrer: String,
pub blocked_urls: Vec<String>,
pub cookie_jar: Option<Arc<CookieJar>>,
pub http_client: Option<Arc<ObscuraHttpClient>>,
@@ -141,6 +145,7 @@ impl ObscuraState {
url: "about:blank".to_string(),
encoding: "UTF-8".to_string(),
title: String::new(),
referrer: String::new(),
blocked_urls: Vec::new(),
cookie_jar: None,
http_client: None,
@@ -350,8 +355,26 @@ fn op_dom_inner(state: &OpState, cmd: String, arg1: String, arg2: String) -> Str
match cmd.as_str() {
"document_node_id" => dom.document().index().to_string(),
"document_title" => serde_json::to_string(&gs.title).unwrap_or("\"\"".into()),
"document_title" => {
// The DOM is authoritative after parsing. In particular, script
// changes through title.textContent must be reflected by
// document.title, not hidden behind the navigation-time snapshot.
let title = dom
.query_selector("title")
.ok()
.flatten()
.map(|title_id| {
dom.text_content(title_id)
.split(|ch| matches!(ch, '\t' | '\n' | '\u{000C}' | '\r' | ' '))
.filter(|part| !part.is_empty())
.collect::<Vec<_>>()
.join(" ")
})
.unwrap_or_default();
serde_json::to_string(&title).unwrap_or("\"\"".into())
}
"document_url" => serde_json::to_string(&gs.url).unwrap_or("\"\"".into()),
"document_referrer" => serde_json::to_string(&gs.referrer).unwrap_or("\"\"".into()),
"document_encoding" => serde_json::to_string(&gs.encoding).unwrap_or("\"UTF-8\"".into()),
"document_element" => {
for cid in dom.children(dom.document()) {
crates/obscura-js/src/runtime.rs
+94 -2
View File
@@ -255,6 +255,14 @@ impl ObscuraJsRuntime {
self.state.borrow_mut().title = title.to_string();
}
/// Set the source document URL exposed as `document.referrer`. Navigation
/// owns this value; it is not derived from the current URL because direct
/// navigations and document-initiated navigations have different
/// referrer semantics.
pub fn set_referrer(&self, referrer: &str) {
self.state.borrow_mut().referrer = referrer.to_string();
}
pub fn set_blocked_urls(&self, patterns: Vec<String>) {
self.state.borrow_mut().blocked_urls = patterns;
}
@@ -2244,7 +2252,91 @@ mod tests {
fn test_document_title() {
let mut rt = setup_runtime("<html><head><title>Test</title></head><body></body></html>");
let title = rt.evaluate("document.title").unwrap();
assert_eq!(title, serde_json::json!("Test Page"));
assert_eq!(title, serde_json::json!("Test"));
let result = rt
.evaluate(
r#"
(function() {
document.title = "A <new> title";
return [
document.title,
document.querySelector("head > title").textContent,
document.querySelectorAll("title").length
];
})()
"#,
)
.unwrap();
assert_eq!(
result,
serde_json::json!(["A <new> title", "A <new> title", 1])
);
let normalized = rt
.evaluate(
r#"
(function() {
document.querySelector("title").textContent = " live\n\tDOM title ";
return document.title;
})()
"#,
)
.unwrap();
assert_eq!(normalized, serde_json::json!("live DOM title"));
}
#[test]
fn document_title_setter_creates_missing_title_element() {
let mut rt = setup_runtime("<html><body><main>content</main></body></html>");
let result = rt
.evaluate(
r#"
(function() {
document.title = "Created";
return [
document.title,
document.head.tagName,
document.head.firstElementChild.tagName,
document.head.firstElementChild.textContent,
document.documentElement.firstElementChild === document.head
];
})()
"#,
)
.unwrap();
assert_eq!(
result,
serde_json::json!(["Created", "HEAD", "TITLE", "Created", true])
);
let detached = rt
.evaluate(
r#"
(function() {
const doc = document.implementation.createHTMLDocument();
doc.title = " Detached title ";
return [doc.title, doc.querySelector("title").textContent, doc.referrer];
})()
"#,
)
.unwrap();
assert_eq!(
detached,
serde_json::json!(["Detached title", " Detached title ", ""])
);
}
#[test]
fn document_referrer_has_explicit_navigation_state() {
let mut rt = setup_runtime("<html><body></body></html>");
assert_eq!(rt.evaluate("document.referrer").unwrap(), serde_json::json!(""));
rt.set_referrer("https://source.example/path?q=1");
assert_eq!(
rt.evaluate("document.referrer").unwrap(),
serde_json::json!("https://source.example/path?q=1")
);
}
#[test]
@@ -6052,7 +6144,7 @@ mod tests {
let result = rt
.call_function_on("() => document.title", None, &[], true)
.await.unwrap();
assert_eq!(result.value.unwrap(), serde_json::json!("Test Page"));
assert_eq!(result.value.unwrap(), serde_json::json!("Test"));
}
#[tokio::test(flavor = "current_thread")]