From 3bf0da2fae91f228ccda83bf5f47d0923c6a534c Mon Sep 17 00:00:00 2001 From: RohitMadhu Date: Mon, 22 Jun 2026 07:02:49 -0400 Subject: [PATCH] Add Network.setBlockedURLs support (#315) Added Network.setBlockedURLs dispatch handling. Stores blocked URL patterns on each page. Applies blocking to page subresources and JS fetch / XHR requests. Supports wildcard CDP-style patterns like *://*.gstatic.com/*.woff2. --- crates/obscura-browser/src/page.rs | 81 +++++++++++++--- crates/obscura-cdp/src/dispatch.rs | 3 +- crates/obscura-cdp/src/domains/network.rs | 109 ++++++++++++++++++++++ crates/obscura-js/src/ops.rs | 56 +++++++++-- 4 files changed, 228 insertions(+), 21 deletions(-) diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs index 51e75ea..3264905 100644 --- a/crates/obscura-browser/src/page.rs +++ b/crates/obscura-browser/src/page.rs @@ -172,6 +172,7 @@ pub struct Page { network_event_counter: u32, pub intercept_enabled: bool, pub intercept_block_patterns: Vec, + pub blocked_url_patterns: Vec, intercept_tx: Option>, // Scripts to execute in the page's JS context BEFORE any of the page's // own scripts run — the CDP `Page.addScriptToEvaluateOnNewDocument` @@ -226,6 +227,7 @@ impl Page { network_event_counter: 0, intercept_enabled: false, intercept_block_patterns: Vec::new(), + blocked_url_patterns: Vec::new(), intercept_tx: None, preload_scripts: Vec::new(), #[cfg(feature = "stealth")] @@ -234,21 +236,18 @@ impl Page { } fn should_block_url(&self, url: &str) -> bool { - if !self.intercept_enabled || self.intercept_block_patterns.is_empty() { - return false; - } - for pattern in &self.intercept_block_patterns { - if pattern == "*" { return true; } - if pattern.starts_with('*') && pattern.ends_with('*') { - if url.contains(&pattern[1..pattern.len()-1]) { return true; } - } else if pattern.starts_with('*') { - if url.ends_with(&pattern[1..]) { return true; } - } else if pattern.ends_with('*') { - if url.starts_with(&pattern[..pattern.len()-1]) { return true; } - } else if url.contains(pattern) { + for pattern in &self.blocked_url_patterns { + if url_matches_cdp_pattern(pattern, url) { return true; } } + if self.intercept_enabled { + for pattern in &self.intercept_block_patterns { + if url_matches_cdp_pattern(pattern, url) { + return true; + } + } + } false } @@ -317,6 +316,7 @@ impl Page { rt.set_cookie_jar(self.context.cookie_jar.clone()); rt.set_http_client(self.http_client.clone()); + rt.set_blocked_urls(self.blocked_url_patterns.clone()); if let Some(tx) = &self.intercept_tx { rt.set_intercept_tx(tx.clone()); @@ -1353,6 +1353,7 @@ impl Page { } pub fn set_blocked_urls(&mut self, patterns: Vec) { + self.blocked_url_patterns = patterns.clone(); if let Some(js) = &self.js { js.set_blocked_urls(patterns); } @@ -1549,6 +1550,62 @@ impl Page { } } +fn url_matches_cdp_pattern(pattern: &str, url: &str) -> bool { + if pattern == "*" { + return true; + } + + let mut remainder = url; + let mut first = true; + for part in pattern.split('*') { + if part.is_empty() { + continue; + } + + let Some(index) = remainder.find(part) else { + return false; + }; + + if first && !pattern.starts_with('*') && index != 0 { + return false; + } + + remainder = &remainder[index + part.len()..]; + first = false; + } + + pattern.ends_with('*') || remainder.is_empty() +} + +#[cfg(test)] +mod tests { + use super::url_matches_cdp_pattern; + + #[test] + fn url_matches_cdp_pattern_handles_wildcards_across_url_parts() { + assert!(url_matches_cdp_pattern( + "*://*.gstatic.com/*.woff2", + "https://fonts.gstatic.com/s/inter/v18/UcCO3FwrK3iLTcviYwYZ8UA3.woff2", + )); + assert!(url_matches_cdp_pattern( + "*://*.google.com/maps/vt/*", + "https://www.google.com/maps/vt/pb=!1m4!1m3", + )); + assert!(url_matches_cdp_pattern( + "https://example.com/assets/*", + "https://example.com/assets/app.js", + )); + assert!(!url_matches_cdp_pattern( + "https://example.com/assets/*", + "https://cdn.example.com/assets/app.js", + )); + assert!(!url_matches_cdp_pattern( + "*://*.gstatic.com/*.woff2", + "https://fonts.gstatic.com/s/inter/v18/font.woff", + )); + } +} + #[derive(Debug, thiserror::Error)] pub enum PageError { #[error("Invalid URL: {0}")] diff --git a/crates/obscura-cdp/src/dispatch.rs b/crates/obscura-cdp/src/dispatch.rs index 93f6b50..1fce87b 100644 --- a/crates/obscura-cdp/src/dispatch.rs +++ b/crates/obscura-cdp/src/dispatch.rs @@ -233,7 +233,8 @@ fn is_v8_free_method(method: &str) -> bool { | "Runtime.runIfWaitingForDebugger" | "Runtime.getExceptionDetails" | "Runtime.discardConsoleEntries" | "Network.enable" | "Network.disable" | "Network.setCacheDisabled" - | "Network.setRequestInterception" | "Network.setExtraHTTPHeaders" + | "Network.setRequestInterception" | "Network.setBlockedURLs" + | "Network.setExtraHTTPHeaders" | "Network.setUserAgentOverride" | "Network.getCookies" | "Network.getAllCookies" | "Network.setCookie" | "Network.setCookies" diff --git a/crates/obscura-cdp/src/domains/network.rs b/crates/obscura-cdp/src/domains/network.rs index ec50bf8..fc5c910 100644 --- a/crates/obscura-cdp/src/domains/network.rs +++ b/crates/obscura-cdp/src/domains/network.rs @@ -96,6 +96,27 @@ pub async fn handle( } "setCacheDisabled" => Ok(json!({})), "setRequestInterception" => Ok(json!({})), + "setBlockedURLs" => { + let patterns = params + .get("urls") + .and_then(|value| value.as_array()) + .map(|values| { + values + .iter() + .filter_map(|value| value.as_str().map(ToString::to_string)) + .collect::>() + }) + .unwrap_or_default(); + + if let Some(page) = ctx.get_session_page_mut(session_id) { + page.set_blocked_urls(patterns); + } else { + for page in &mut ctx.pages { + page.set_blocked_urls(patterns.clone()); + } + } + Ok(json!({})) + } "getResponseBody" => { let request_id = params .get("requestId") @@ -224,6 +245,94 @@ mod tests { assert!(ctx.default_context.cookie_jar.get_all_cookies().is_empty()); } + #[tokio::test] + async fn set_blocked_urls_targets_session_page_without_enabling_interception() { + let mut ctx = CdpContext::new(); + let page_id = ctx.create_page(); + let session_id = Some("session-1".to_string()); + ctx.sessions.insert(session_id.clone().unwrap(), page_id.clone()); + + handle( + "setBlockedURLs", + &json!({ + "urls": [ + "*://*.example.com/*.png", + "*://cdn.example.com/*" + ] + }), + &mut ctx, + &session_id, + ) + .await + .expect("setBlockedURLs must succeed for a session page"); + + let page = ctx.get_page(&page_id).unwrap(); + assert_eq!( + page.blocked_url_patterns, + vec![ + "*://*.example.com/*.png".to_string(), + "*://cdn.example.com/*".to_string(), + ] + ); + assert!(!page.intercept_enabled); + assert!(page.intercept_block_patterns.is_empty()); + } + + #[tokio::test] + async fn set_blocked_urls_without_session_updates_existing_pages() { + let mut ctx = CdpContext::new(); + let left = ctx.create_page(); + let right = ctx.create_page(); + + handle( + "setBlockedURLs", + &json!({ "urls": ["*://tiles.example.test/*"] }), + &mut ctx, + &None, + ) + .await + .expect("setBlockedURLs must succeed without a session"); + + assert_eq!( + ctx.get_page(&left).unwrap().blocked_url_patterns, + vec!["*://tiles.example.test/*".to_string()] + ); + assert_eq!( + ctx.get_page(&right).unwrap().blocked_url_patterns, + vec!["*://tiles.example.test/*".to_string()] + ); + } + + #[tokio::test] + async fn set_blocked_urls_replaces_existing_patterns() { + let mut ctx = CdpContext::new(); + let page_id = ctx.create_page(); + let session_id = Some("session-1".to_string()); + ctx.sessions.insert(session_id.clone().unwrap(), page_id.clone()); + + handle( + "setBlockedURLs", + &json!({ "urls": ["*://old.example.test/*"] }), + &mut ctx, + &session_id, + ) + .await + .unwrap(); + handle( + "setBlockedURLs", + &json!({ "urls": ["*://new.example.test/*"] }), + &mut ctx, + &session_id, + ) + .await + .unwrap(); + + assert_eq!( + ctx.get_page(&page_id).unwrap().blocked_url_patterns, + vec!["*://new.example.test/*".to_string()] + ); + } + #[tokio::test] async fn get_response_body_returns_stored_document_body() { let mut ctx = CdpContext::new(); diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs index 1975b72..bfbf1fd 100644 --- a/crates/obscura-js/src/ops.rs +++ b/crates/obscura-js/src/ops.rs @@ -950,16 +950,56 @@ fn glob_match(pattern: &str, url: &str) -> bool { if pattern == "*" { return true; } - if pattern.starts_with('*') && pattern.ends_with('*') { - return url.contains(&pattern[1..pattern.len() - 1]); + + let mut remainder = url; + let mut first = true; + for part in pattern.split('*') { + if part.is_empty() { + continue; + } + + let Some(index) = remainder.find(part) else { + return false; + }; + + if first && !pattern.starts_with('*') && index != 0 { + return false; + } + + remainder = &remainder[index + part.len()..]; + first = false; } - if pattern.starts_with('*') { - return url.ends_with(&pattern[1..]); + + pattern.ends_with('*') || remainder.is_empty() +} + +#[cfg(test)] +mod tests { + use super::glob_match; + + #[test] + fn glob_match_handles_cdp_blocked_url_patterns() { + assert!(glob_match( + "*://*.google.com/maps/vt/*", + "https://www.google.com/maps/vt/pb=!1m4!1m3", + )); + assert!(glob_match( + "*://*.gstatic.com/*.woff2", + "https://fonts.gstatic.com/s/inter/v18/font.woff2", + )); + assert!(glob_match( + "https://example.com/assets/*", + "https://example.com/assets/app.js", + )); + assert!(!glob_match( + "https://example.com/assets/*", + "https://cdn.example.com/assets/app.js", + )); + assert!(!glob_match( + "*://*.gstatic.com/*.woff2", + "https://fonts.gstatic.com/s/inter/v18/font.woff", + )); } - if pattern.ends_with('*') { - return url.starts_with(&pattern[..pattern.len() - 1]); - } - url == pattern } fn validate_fetch_url(url: &url::Url) -> Result<(), String> {