fix(net): scope passive callbacks to the registering page (#414)
The on_request/on_response Vec lived on the ObscuraHttpClient shared by every page in the context, so a callback registered on one page fired for all pages (#408 cross-page bleed) and outlived its page. Callbacks now live in a per-page CallbackRegistry (Arc shared with the page's JS runtime state). Page-driven fetch paths (navigation, POST, scripts, stylesheets, robots.txt, module loads, JS fetch/XHR in both plain and stealth modes) pass the registry through; the HTTP client itself is callback-free. Adds callbacks_do_not_bleed_across_pages: page A's callback stays silent for page B's requests and stops firing after page A is dropped.
This commit is contained in:
1 parent
ccfa5fb87b
commit
f5dc97b131
8 files changed
+249
-108
No files matched your search
@@ -3,7 +3,7 @@ use std::sync::Arc;
|
|||||||
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
|
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
|
||||||
use obscura_dom::{parse_html, DomTree};
|
use obscura_dom::{parse_html, DomTree};
|
||||||
use obscura_js::runtime::ObscuraJsRuntime;
|
use obscura_js::runtime::ObscuraJsRuntime;
|
||||||
use obscura_net::{ObscuraHttpClient, ObscuraNetError, RequestCallback, Response, ResponseCallback};
|
use obscura_net::{CallbackRegistry, ObscuraHttpClient, ObscuraNetError, RequestCallback, Response, ResponseCallback};
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use crate::context::BrowserContext;
|
use crate::context::BrowserContext;
|
||||||
@@ -194,6 +194,10 @@ pub struct Page {
|
|||||||
// contract. Includes `Runtime.addBinding` shims so puppeteer's
|
// contract. Includes `Runtime.addBinding` shims so puppeteer's
|
||||||
// `exposeFunction` bindings exist before inline `<script>` tags execute.
|
// `exposeFunction` bindings exist before inline `<script>` tags execute.
|
||||||
preload_scripts: Vec<String>,
|
preload_scripts: Vec<String>,
|
||||||
|
/// Passive on_request/on_response callbacks, scoped to this page (issue
|
||||||
|
/// #408): they fire only for requests this page drives and die with it.
|
||||||
|
/// Arc because the JS runtime state holds a second handle for fetch()/XHR.
|
||||||
|
callbacks: Arc<CallbackRegistry>,
|
||||||
#[cfg(feature = "stealth")]
|
#[cfg(feature = "stealth")]
|
||||||
pub stealth_client: Option<Arc<StealthHttpClient>>,
|
pub stealth_client: Option<Arc<StealthHttpClient>>,
|
||||||
}
|
}
|
||||||
@@ -245,6 +249,7 @@ impl Page {
|
|||||||
blocked_url_patterns: Vec::new(),
|
blocked_url_patterns: Vec::new(),
|
||||||
intercept_tx: None,
|
intercept_tx: None,
|
||||||
preload_scripts: Vec::new(),
|
preload_scripts: Vec::new(),
|
||||||
|
callbacks: Arc::new(CallbackRegistry::new()),
|
||||||
#[cfg(feature = "stealth")]
|
#[cfg(feature = "stealth")]
|
||||||
stealth_client,
|
stealth_client,
|
||||||
}
|
}
|
||||||
@@ -271,7 +276,9 @@ impl Page {
|
|||||||
if let Some(ref stealth) = self.stealth_client {
|
if let Some(ref stealth) = self.stealth_client {
|
||||||
return stealth.fetch(url).await;
|
return stealth.fetch(url).await;
|
||||||
}
|
}
|
||||||
self.http_client.fetch(url).await
|
self.http_client
|
||||||
|
.fetch_with_callbacks(url, Some(&self.callbacks))
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
fn init_js(&mut self) {
|
fn init_js(&mut self) {
|
||||||
// Drop any existing runtime so the JS realm starts clean on
|
// Drop any existing runtime so the JS realm starts clean on
|
||||||
@@ -333,6 +340,7 @@ impl Page {
|
|||||||
|
|
||||||
rt.set_cookie_jar(self.context.cookie_jar.clone());
|
rt.set_cookie_jar(self.context.cookie_jar.clone());
|
||||||
rt.set_http_client(self.http_client.clone());
|
rt.set_http_client(self.http_client.clone());
|
||||||
|
rt.set_callbacks(self.callbacks.clone());
|
||||||
rt.set_blocked_urls(self.blocked_url_patterns.clone());
|
rt.set_blocked_urls(self.blocked_url_patterns.clone());
|
||||||
#[cfg(feature = "stealth")]
|
#[cfg(feature = "stealth")]
|
||||||
if let Some(ref stealth) = self.stealth_client {
|
if let Some(ref stealth) = self.stealth_client {
|
||||||
@@ -533,8 +541,10 @@ impl Page {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let client = self.http_client.clone();
|
let client = self.http_client.clone();
|
||||||
|
let page_callbacks = self.callbacks.clone();
|
||||||
let fetch_futures: Vec<_> = fetch_tasks.iter().map(|(idx, url)| {
|
let fetch_futures: Vec<_> = fetch_tasks.iter().map(|(idx, url)| {
|
||||||
let client = client.clone();
|
let client = client.clone();
|
||||||
|
let cbs = page_callbacks.clone();
|
||||||
let url = url.clone();
|
let url = url.clone();
|
||||||
let idx = *idx;
|
let idx = *idx;
|
||||||
async move {
|
async move {
|
||||||
@@ -563,7 +573,7 @@ impl Page {
|
|||||||
};
|
};
|
||||||
return Some((idx, url, resp));
|
return Some((idx, url, resp));
|
||||||
}
|
}
|
||||||
match client.fetch(&parsed).await {
|
match client.fetch_with_callbacks(&parsed, Some(&cbs)).await {
|
||||||
Ok(resp) => Some((idx, url, resp)),
|
Ok(resp) => Some((idx, url, resp)),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("Failed to fetch script {}: {}", url, e);
|
tracing::warn!("Failed to fetch script {}: {}", url, e);
|
||||||
@@ -947,7 +957,11 @@ impl Page {
|
|||||||
if self.context.robots_cache.is_allowed(domain, "/robots.txt") {
|
if self.context.robots_cache.is_allowed(domain, "/robots.txt") {
|
||||||
let robots_url = format!("{}://{}/robots.txt", url.scheme(), domain);
|
let robots_url = format!("{}://{}/robots.txt", url.scheme(), domain);
|
||||||
if let Ok(robots_url) = Url::parse(&robots_url) {
|
if let Ok(robots_url) = Url::parse(&robots_url) {
|
||||||
if let Ok(resp) = self.http_client.fetch(&robots_url).await {
|
if let Ok(resp) = self
|
||||||
|
.http_client
|
||||||
|
.fetch_with_callbacks(&robots_url, Some(&self.callbacks))
|
||||||
|
.await
|
||||||
|
{
|
||||||
if resp.status == 200 {
|
if resp.status == 200 {
|
||||||
let body = String::from_utf8_lossy(&resp.body);
|
let body = String::from_utf8_lossy(&resp.body);
|
||||||
self.context.robots_cache.parse_and_store(
|
self.context.robots_cache.parse_and_store(
|
||||||
@@ -1000,7 +1014,9 @@ impl Page {
|
|||||||
headers.insert("content-type".to_string(), content_type);
|
headers.insert("content-type".to_string(), content_type);
|
||||||
Ok(obscura_net::Response { url: url.clone(), status: 200, headers, body: body_bytes, redirected_from: Vec::new() })
|
Ok(obscura_net::Response { url: url.clone(), status: 200, headers, body: body_bytes, redirected_from: Vec::new() })
|
||||||
} else if method == "POST" {
|
} else if method == "POST" {
|
||||||
self.http_client.post_form(&url, body).await
|
self.http_client
|
||||||
|
.post_form_with_callbacks(&url, body, Some(&self.callbacks))
|
||||||
|
.await
|
||||||
} else {
|
} else {
|
||||||
self.do_fetch(&url).await
|
self.do_fetch(&url).await
|
||||||
}.map_err(|e| {
|
}.map_err(|e| {
|
||||||
@@ -1086,12 +1102,14 @@ impl Page {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let client = self.http_client.clone();
|
let client = self.http_client.clone();
|
||||||
|
let page_callbacks = self.callbacks.clone();
|
||||||
let css_futures: Vec<_> = css_fetch_urls.iter().map(|full_url| {
|
let css_futures: Vec<_> = css_fetch_urls.iter().map(|full_url| {
|
||||||
let client = client.clone();
|
let client = client.clone();
|
||||||
|
let cbs = page_callbacks.clone();
|
||||||
let url_str = full_url.clone();
|
let url_str = full_url.clone();
|
||||||
async move {
|
async move {
|
||||||
let parsed = Url::parse(&url_str).unwrap_or_else(|_| Url::parse("about:blank").unwrap());
|
let parsed = Url::parse(&url_str).unwrap_or_else(|_| Url::parse("about:blank").unwrap());
|
||||||
match client.fetch(&parsed).await {
|
match client.fetch_with_callbacks(&parsed, Some(&cbs)).await {
|
||||||
Ok(resp) => Some((url_str, resp)),
|
Ok(resp) => Some((url_str, resp)),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::debug!("Failed to fetch stylesheet {}: {}", url_str, e);
|
tracing::debug!("Failed to fetch stylesheet {}: {}", url_str, e);
|
||||||
@@ -1650,34 +1668,34 @@ impl Page {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Register a passive callback fired for every JS `fetch()`/XHR (and
|
/// Register a passive callback fired for every JS `fetch()`/XHR (and
|
||||||
/// navigation) request, once the method/headers/body are known and before it
|
/// navigation) request this page makes, once the method/headers/body are
|
||||||
/// is sent. Non-blocking; use `enable_interception` to mutate or block.
|
/// known and before it is sent. Non-blocking; use `enable_interception` to
|
||||||
/// Register a passive request observer, returning a stable id. Pass the id
|
/// mutate or block. Returns a stable id; pass it to `off_request` to
|
||||||
/// to `off_request` to detach it (issue #408). Note the observer lives on
|
/// detach (issue #408). Scoped to this page: it never sees sibling pages'
|
||||||
/// the context's shared http client, so it also sees requests from sibling
|
/// requests and dies with the page.
|
||||||
/// pages in the same context; detach it when the capturing phase ends.
|
|
||||||
pub fn on_request(&mut self, cb: RequestCallback) -> u64 {
|
pub fn on_request(&mut self, cb: RequestCallback) -> u64 {
|
||||||
self.http_client.register_on_request(cb)
|
self.callbacks.add_request(cb)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Register a passive callback fired with every JS `fetch()`/XHR (and
|
/// Register a passive callback fired with every JS `fetch()`/XHR (and
|
||||||
/// navigation) response, including its body. Non-blocking. Returns a stable
|
/// navigation) response this page receives, including its body.
|
||||||
/// id for `off_response`. The main path for crawlers that need to capture
|
/// Non-blocking. The main path for crawlers that need to capture API
|
||||||
/// API response payloads. Shared across the context like `on_request`.
|
/// response payloads. Returns a stable id for `off_response`. Page-scoped
|
||||||
|
/// like `on_request`.
|
||||||
pub fn on_response(&mut self, cb: ResponseCallback) -> u64 {
|
pub fn on_response(&mut self, cb: ResponseCallback) -> u64 {
|
||||||
self.http_client.register_on_response(cb)
|
self.callbacks.add_response(cb)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Detach a request observer registered with `on_request`. Returns true if
|
/// Detach a request observer registered with `on_request`. Returns true if
|
||||||
/// one was removed.
|
/// one was removed.
|
||||||
pub fn off_request(&mut self, id: u64) -> bool {
|
pub fn off_request(&mut self, id: u64) -> bool {
|
||||||
self.http_client.remove_on_request(id)
|
self.callbacks.remove_request(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Detach a response observer registered with `on_response`. Returns true if
|
/// Detach a response observer registered with `on_response`. Returns true if
|
||||||
/// one was removed.
|
/// one was removed.
|
||||||
pub fn off_response(&mut self, id: u64) -> bool {
|
pub fn off_response(&mut self, id: u64) -> bool {
|
||||||
self.http_client.remove_on_response(id)
|
self.callbacks.remove_response(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn process_pending_navigation(&mut self) -> Result<bool, PageError> {
|
pub async fn process_pending_navigation(&mut self) -> Result<bool, PageError> {
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ use deno_core::OpState;
|
|||||||
use deno_core::Extension;
|
use deno_core::Extension;
|
||||||
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
|
use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
|
||||||
use obscura_dom::{DomTree, NodeData, NodeId};
|
use obscura_dom::{DomTree, NodeData, NodeId};
|
||||||
use obscura_net::{CookieJar, ObscuraHttpClient, RequestInfo, ResourceType, Response};
|
use obscura_net::{CallbackRegistry, CookieJar, ObscuraHttpClient, RequestInfo, ResourceType, Response};
|
||||||
#[cfg(feature = "stealth")]
|
#[cfg(feature = "stealth")]
|
||||||
use obscura_net::StealthHttpClient;
|
use obscura_net::StealthHttpClient;
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
@@ -75,6 +75,10 @@ pub struct ObscuraState {
|
|||||||
pub blocked_urls: Vec<String>,
|
pub blocked_urls: Vec<String>,
|
||||||
pub cookie_jar: Option<Arc<CookieJar>>,
|
pub cookie_jar: Option<Arc<CookieJar>>,
|
||||||
pub http_client: Option<Arc<ObscuraHttpClient>>,
|
pub http_client: Option<Arc<ObscuraHttpClient>>,
|
||||||
|
/// The owning page's passive on_request/on_response callbacks (issue
|
||||||
|
/// #408). Page-scoped, so scripted fetch()/XHR observation stays local to
|
||||||
|
/// the page that registered it.
|
||||||
|
pub callbacks: Option<Arc<CallbackRegistry>>,
|
||||||
/// When set (stealth mode), scripted fetch()/XHR is routed through the wreq
|
/// When set (stealth mode), scripted fetch()/XHR is routed through the wreq
|
||||||
/// client so the request carries the Chrome TLS fingerprint and client
|
/// client so the request carries the Chrome TLS fingerprint and client
|
||||||
/// hints instead of the rustls ClientHello op_fetch_url would otherwise send.
|
/// hints instead of the rustls ClientHello op_fetch_url would otherwise send.
|
||||||
@@ -111,6 +115,7 @@ impl ObscuraState {
|
|||||||
blocked_urls: Vec::new(),
|
blocked_urls: Vec::new(),
|
||||||
cookie_jar: None,
|
cookie_jar: None,
|
||||||
http_client: None,
|
http_client: None,
|
||||||
|
callbacks: None,
|
||||||
#[cfg(feature = "stealth")]
|
#[cfg(feature = "stealth")]
|
||||||
stealth_client: None,
|
stealth_client: None,
|
||||||
pending_navigation: None,
|
pending_navigation: None,
|
||||||
@@ -642,7 +647,7 @@ async fn op_fetch_url(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let (cookie_jar, in_flight, intercept_tx, proxy_url, http_client) = {
|
let (cookie_jar, in_flight, intercept_tx, proxy_url, callbacks) = {
|
||||||
let state_borrow = state.borrow();
|
let state_borrow = state.borrow();
|
||||||
let gs = state_borrow.borrow::<SharedState>().clone();
|
let gs = state_borrow.borrow::<SharedState>().clone();
|
||||||
let mut gs = gs.borrow_mut();
|
let mut gs = gs.borrow_mut();
|
||||||
@@ -674,7 +679,7 @@ async fn op_fetch_url(
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
(jar, in_flight, itx, proxy_url, gs.http_client.clone())
|
(jar, in_flight, itx, proxy_url, gs.callbacks.clone())
|
||||||
};
|
};
|
||||||
|
|
||||||
// Slots the interception channel can override via Continue so a consumer
|
// Slots the interception channel can override via Continue so a consumer
|
||||||
@@ -782,9 +787,8 @@ async fn op_fetch_url(
|
|||||||
// reaches the network (Fulfill/Fail from the interception channel short-
|
// reaches the network (Fulfill/Fail from the interception channel short-
|
||||||
// circuit earlier). on_request/on_response previously fired only for
|
// circuit earlier). on_request/on_response previously fired only for
|
||||||
// navigation; this wires them for JS fetch()/XHR too.
|
// navigation; this wires them for JS fetch()/XHR too.
|
||||||
if let Some(ref hc) = http_client {
|
if let Some(ref cbs) = callbacks {
|
||||||
let cbs = hc.on_request.read().await;
|
if cbs.has_request_callbacks().await {
|
||||||
if !cbs.is_empty() {
|
|
||||||
if let Ok(parsed) = url::Url::parse(&url) {
|
if let Ok(parsed) = url::Url::parse(&url) {
|
||||||
let info = RequestInfo {
|
let info = RequestInfo {
|
||||||
url: parsed,
|
url: parsed,
|
||||||
@@ -792,9 +796,7 @@ async fn op_fetch_url(
|
|||||||
headers: custom_headers.clone(),
|
headers: custom_headers.clone(),
|
||||||
resource_type: ResourceType::Fetch,
|
resource_type: ResourceType::Fetch,
|
||||||
};
|
};
|
||||||
for (_, cb) in cbs.iter() {
|
cbs.fire_request(&info).await;
|
||||||
cb(&info);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -822,7 +824,7 @@ async fn op_fetch_url(
|
|||||||
page_origin.clone(),
|
page_origin.clone(),
|
||||||
is_cross_origin,
|
is_cross_origin,
|
||||||
mode.clone(),
|
mode.clone(),
|
||||||
http_client.clone(),
|
callbacks.clone(),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
@@ -1028,9 +1030,8 @@ async fn op_fetch_url(
|
|||||||
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?;
|
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?;
|
||||||
let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
|
let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
|
||||||
let resp_body_base64 = BASE64.encode(&resp_bytes);
|
let resp_body_base64 = BASE64.encode(&resp_bytes);
|
||||||
if let Some(ref hc) = http_client {
|
if let Some(ref cbs) = callbacks {
|
||||||
let cbs = hc.on_response.read().await;
|
if cbs.has_response_callbacks().await {
|
||||||
if !cbs.is_empty() {
|
|
||||||
let resp = fetch_response(&url, status, resp_headers.clone(), resp_bytes.to_vec());
|
let resp = fetch_response(&url, status, resp_headers.clone(), resp_bytes.to_vec());
|
||||||
let info = RequestInfo {
|
let info = RequestInfo {
|
||||||
url: resp.url.clone(),
|
url: resp.url.clone(),
|
||||||
@@ -1038,9 +1039,7 @@ async fn op_fetch_url(
|
|||||||
headers: resp_headers.clone(),
|
headers: resp_headers.clone(),
|
||||||
resource_type: ResourceType::Fetch,
|
resource_type: ResourceType::Fetch,
|
||||||
};
|
};
|
||||||
for (_, cb) in cbs.iter() {
|
cbs.fire_response(&info, &resp).await;
|
||||||
cb(&info, &resp);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let response_request_id = {
|
let response_request_id = {
|
||||||
@@ -1133,7 +1132,7 @@ async fn stealth_fetch_all(
|
|||||||
page_origin: String,
|
page_origin: String,
|
||||||
is_cross_origin: bool,
|
is_cross_origin: bool,
|
||||||
mode: String,
|
mode: String,
|
||||||
http_client: Option<Arc<ObscuraHttpClient>>,
|
callbacks: Option<Arc<CallbackRegistry>>,
|
||||||
) -> Result<String, deno_error::JsErrorBox> {
|
) -> Result<String, deno_error::JsErrorBox> {
|
||||||
let mut current_url = url.clone();
|
let mut current_url = url.clone();
|
||||||
let mut current_method = method;
|
let mut current_method = method;
|
||||||
@@ -1221,9 +1220,8 @@ async fn stealth_fetch_all(
|
|||||||
|
|
||||||
let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
|
let resp_body = String::from_utf8_lossy(&resp_bytes).to_string();
|
||||||
let resp_body_base64 = BASE64.encode(&resp_bytes);
|
let resp_body_base64 = BASE64.encode(&resp_bytes);
|
||||||
if let Some(ref hc) = http_client {
|
if let Some(ref cbs) = callbacks {
|
||||||
let cbs = hc.on_response.read().await;
|
if cbs.has_response_callbacks().await {
|
||||||
if !cbs.is_empty() {
|
|
||||||
let resp = fetch_response(&url, status, resp_headers.clone(), resp_bytes.clone());
|
let resp = fetch_response(&url, status, resp_headers.clone(), resp_bytes.clone());
|
||||||
let info = RequestInfo {
|
let info = RequestInfo {
|
||||||
url: resp.url.clone(),
|
url: resp.url.clone(),
|
||||||
@@ -1231,9 +1229,7 @@ async fn stealth_fetch_all(
|
|||||||
headers: resp_headers.clone(),
|
headers: resp_headers.clone(),
|
||||||
resource_type: ResourceType::Fetch,
|
resource_type: ResourceType::Fetch,
|
||||||
};
|
};
|
||||||
for (_, cb) in cbs.iter() {
|
cbs.fire_response(&info, &resp).await;
|
||||||
cb(&info, &resp);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -153,6 +153,12 @@ impl ObscuraJsRuntime {
|
|||||||
self.state.borrow_mut().http_client = Some(client);
|
self.state.borrow_mut().http_client = Some(client);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Install the owning page's passive on_request/on_response callback
|
||||||
|
/// registry so scripted fetch()/XHR observation is page-scoped (issue #408).
|
||||||
|
pub fn set_callbacks(&self, callbacks: std::sync::Arc<obscura_net::CallbackRegistry>) {
|
||||||
|
self.state.borrow_mut().callbacks = Some(callbacks);
|
||||||
|
}
|
||||||
|
|
||||||
/// Install the stealth (wreq) HTTP client so scripted fetch()/XHR is routed
|
/// Install the stealth (wreq) HTTP client so scripted fetch()/XHR is routed
|
||||||
/// through it in stealth mode (see op_fetch_url / stealth_fetch_all).
|
/// through it in stealth mode (see op_fetch_url / stealth_fetch_all).
|
||||||
#[cfg(feature = "stealth")]
|
#[cfg(feature = "stealth")]
|
||||||
@@ -631,9 +637,12 @@ impl ObscuraJsRuntime {
|
|||||||
// Fetch the module source. The old impl registered an empty string
|
// Fetch the module source. The old impl registered an empty string
|
||||||
// and called it loaded, so every Vite / Next module bundle "loaded"
|
// and called it loaded, so every Vite / Next module bundle "loaded"
|
||||||
// in 1ms with zero code and the SPA never mounted (issue #205).
|
// in 1ms with zero code and the SPA never mounted (issue #205).
|
||||||
let client = self.state.borrow().http_client.clone();
|
let (client, callbacks) = {
|
||||||
|
let st = self.state.borrow();
|
||||||
|
(st.http_client.clone(), st.callbacks.clone())
|
||||||
|
};
|
||||||
let source_code = match client {
|
let source_code = match client {
|
||||||
Some(c) => match c.fetch(&specifier).await {
|
Some(c) => match c.fetch_with_callbacks(&specifier, callbacks.as_deref()).await {
|
||||||
Ok(resp) => obscura_net::decode_non_html(&resp.body, resp.content_type()),
|
Ok(resp) => obscura_net::decode_non_html(&resp.body, resp.content_type()),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("Module fetch failed ({}): {}", url, e);
|
tracing::warn!("Module fetch failed ({}): {}", url, e);
|
||||||
|
|||||||
@@ -75,6 +75,107 @@ pub enum ResourceType {
|
|||||||
pub type RequestCallback = Arc<dyn Fn(&RequestInfo) + Send + Sync>;
|
pub type RequestCallback = Arc<dyn Fn(&RequestInfo) + Send + Sync>;
|
||||||
pub type ResponseCallback = Arc<dyn Fn(&RequestInfo, &Response) + Send + Sync>;
|
pub type ResponseCallback = Arc<dyn Fn(&RequestInfo, &Response) + Send + Sync>;
|
||||||
|
|
||||||
|
/// Page-scoped store for the passive on_request/on_response callbacks (issue
|
||||||
|
/// #408). Each `Page` owns one, so a callback never fires for another page's
|
||||||
|
/// requests and dies with its page. The HTTP client itself stays
|
||||||
|
/// callback-free; page-driven fetches pass the page's registry in. Ids keep
|
||||||
|
/// the `u64` shape #416 established on `Page::on_request`/`on_response`.
|
||||||
|
pub struct CallbackRegistry {
|
||||||
|
on_request: RwLock<Vec<(u64, RequestCallback)>>,
|
||||||
|
on_response: RwLock<Vec<(u64, ResponseCallback)>>,
|
||||||
|
id_counter: std::sync::atomic::AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CallbackRegistry {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
CallbackRegistry {
|
||||||
|
on_request: RwLock::new(Vec::new()),
|
||||||
|
on_response: RwLock::new(Vec::new()),
|
||||||
|
id_counter: std::sync::atomic::AtomicU64::new(1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn next_id(&self) -> u64 {
|
||||||
|
self.id_counter
|
||||||
|
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Register a request callback; the returned id detaches it via
|
||||||
|
/// `remove_request`. Sync like the pre-registry push path: registration
|
||||||
|
/// happens from `Page` setup where no reader holds the lock, so
|
||||||
|
/// `try_write` cannot fail there.
|
||||||
|
pub fn add_request(&self, cb: RequestCallback) -> u64 {
|
||||||
|
let id = self.next_id();
|
||||||
|
if let Ok(mut v) = self.on_request.try_write() {
|
||||||
|
v.push((id, cb));
|
||||||
|
}
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Register a response callback; see `add_request`.
|
||||||
|
pub fn add_response(&self, cb: ResponseCallback) -> u64 {
|
||||||
|
let id = self.next_id();
|
||||||
|
if let Ok(mut v) = self.on_response.try_write() {
|
||||||
|
v.push((id, cb));
|
||||||
|
}
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Detach a request callback. Returns true when the id was found and
|
||||||
|
/// removed, so a double detach is a visible no-op.
|
||||||
|
pub fn remove_request(&self, id: u64) -> bool {
|
||||||
|
match self.on_request.try_write() {
|
||||||
|
Ok(mut v) => {
|
||||||
|
let before = v.len();
|
||||||
|
v.retain(|(cid, _)| *cid != id);
|
||||||
|
v.len() != before
|
||||||
|
}
|
||||||
|
Err(_) => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Detach a response callback; see `remove_request`.
|
||||||
|
pub fn remove_response(&self, id: u64) -> bool {
|
||||||
|
match self.on_response.try_write() {
|
||||||
|
Ok(mut v) => {
|
||||||
|
let before = v.len();
|
||||||
|
v.retain(|(cid, _)| *cid != id);
|
||||||
|
v.len() != before
|
||||||
|
}
|
||||||
|
Err(_) => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// True when at least one request callback is registered. Lets fire sites
|
||||||
|
/// skip building a `RequestInfo` when nobody listens.
|
||||||
|
pub async fn has_request_callbacks(&self) -> bool {
|
||||||
|
!self.on_request.read().await.is_empty()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// True when at least one response callback is registered.
|
||||||
|
pub async fn has_response_callbacks(&self) -> bool {
|
||||||
|
!self.on_response.read().await.is_empty()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn fire_request(&self, info: &RequestInfo) {
|
||||||
|
for (_, cb) in self.on_request.read().await.iter() {
|
||||||
|
cb(info);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn fire_response(&self, info: &RequestInfo, resp: &Response) {
|
||||||
|
for (_, cb) in self.on_response.read().await.iter() {
|
||||||
|
cb(info, resp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for CallbackRegistry {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Process-wide opt-in via env var. Older flow that issue #4 introduced. The
|
/// Process-wide opt-in via env var. Older flow that issue #4 introduced. The
|
||||||
/// new `--allow-private-network` CLI flag (issue #33) sets a per-client field
|
/// new `--allow-private-network` CLI flag (issue #33) sets a per-client field
|
||||||
/// that is OR'd with this so existing scripts and Docker setups that pin the
|
/// that is OR'd with this so existing scripts and Docker setups that pin the
|
||||||
@@ -262,13 +363,6 @@ pub struct ObscuraHttpClient {
|
|||||||
pub user_agent: RwLock<String>,
|
pub user_agent: RwLock<String>,
|
||||||
pub extra_headers: RwLock<HashMap<String, String>>,
|
pub extra_headers: RwLock<HashMap<String, String>>,
|
||||||
pub interceptor: RwLock<Option<Box<dyn RequestInterceptor + Send + Sync>>>,
|
pub interceptor: RwLock<Option<Box<dyn RequestInterceptor + Send + Sync>>>,
|
||||||
/// Passive request/response observers, each tagged with a stable id so a
|
|
||||||
/// specific one can be detached via `remove_on_request`/`remove_on_response`
|
|
||||||
/// (issue #408). Without ids the vecs were append-only: no detach, and
|
|
||||||
/// re-registering across navigations piled up duplicates.
|
|
||||||
pub on_request: RwLock<Vec<(u64, RequestCallback)>>,
|
|
||||||
pub on_response: RwLock<Vec<(u64, ResponseCallback)>>,
|
|
||||||
callback_id_counter: std::sync::atomic::AtomicU64,
|
|
||||||
pub timeout: Duration,
|
pub timeout: Duration,
|
||||||
pub in_flight: Arc<std::sync::atomic::AtomicU32>,
|
pub in_flight: Arc<std::sync::atomic::AtomicU32>,
|
||||||
pub block_trackers: bool,
|
pub block_trackers: bool,
|
||||||
@@ -345,9 +439,6 @@ impl ObscuraHttpClient {
|
|||||||
),
|
),
|
||||||
extra_headers: RwLock::new(HashMap::new()),
|
extra_headers: RwLock::new(HashMap::new()),
|
||||||
interceptor: RwLock::new(None),
|
interceptor: RwLock::new(None),
|
||||||
on_request: RwLock::new(Vec::new()),
|
|
||||||
on_response: RwLock::new(Vec::new()),
|
|
||||||
callback_id_counter: std::sync::atomic::AtomicU64::new(1),
|
|
||||||
in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),
|
in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),
|
||||||
timeout: Duration::from_secs(30),
|
timeout: Duration::from_secs(30),
|
||||||
block_trackers: false,
|
block_trackers: false,
|
||||||
@@ -384,55 +475,33 @@ impl ObscuraHttpClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
pub async fn fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
||||||
self.fetch_with_method(Method::GET, url, None).await
|
self.fetch_with_method(Method::GET, url, None, None).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Register a passive request observer, returning a stable id. Pass the id
|
/// `fetch` that also fires the page's passive on_request/on_response
|
||||||
/// to `remove_on_request` to detach it (issue #408).
|
/// callbacks (issue #408: callbacks are page-scoped, so the page-driven
|
||||||
pub fn register_on_request(&self, cb: RequestCallback) -> u64 {
|
/// fetch paths pass their registry in).
|
||||||
let id = self
|
pub async fn fetch_with_callbacks(
|
||||||
.callback_id_counter
|
&self,
|
||||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
url: &Url,
|
||||||
if let Ok(mut v) = self.on_request.try_write() {
|
callbacks: Option<&CallbackRegistry>,
|
||||||
v.push((id, cb));
|
) -> Result<Response, ObscuraNetError> {
|
||||||
}
|
self.fetch_with_method(Method::GET, url, None, callbacks).await
|
||||||
id
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Register a passive response observer, returning a stable id. Pass the id
|
|
||||||
/// to `remove_on_response` to detach it (issue #408).
|
|
||||||
pub fn register_on_response(&self, cb: ResponseCallback) -> u64 {
|
|
||||||
let id = self
|
|
||||||
.callback_id_counter
|
|
||||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
|
||||||
if let Ok(mut v) = self.on_response.try_write() {
|
|
||||||
v.push((id, cb));
|
|
||||||
}
|
|
||||||
id
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Detach a request observer by its id. Returns true if one was removed.
|
|
||||||
pub fn remove_on_request(&self, id: u64) -> bool {
|
|
||||||
if let Ok(mut v) = self.on_request.try_write() {
|
|
||||||
let before = v.len();
|
|
||||||
v.retain(|(cid, _)| *cid != id);
|
|
||||||
return v.len() != before;
|
|
||||||
}
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Detach a response observer by its id. Returns true if one was removed.
|
|
||||||
pub fn remove_on_response(&self, id: u64) -> bool {
|
|
||||||
if let Ok(mut v) = self.on_response.try_write() {
|
|
||||||
let before = v.len();
|
|
||||||
v.retain(|(cid, _)| *cid != id);
|
|
||||||
return v.len() != before;
|
|
||||||
}
|
|
||||||
false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn post_form(&self, url: &Url, body: &str) -> Result<Response, ObscuraNetError> {
|
pub async fn post_form(&self, url: &Url, body: &str) -> Result<Response, ObscuraNetError> {
|
||||||
self.fetch_with_method(Method::POST, url, Some(body.as_bytes().to_vec())).await
|
self.fetch_with_method(Method::POST, url, Some(body.as_bytes().to_vec()), None).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `post_form` variant of `fetch_with_callbacks`.
|
||||||
|
pub async fn post_form_with_callbacks(
|
||||||
|
&self,
|
||||||
|
url: &Url,
|
||||||
|
body: &str,
|
||||||
|
callbacks: Option<&CallbackRegistry>,
|
||||||
|
) -> Result<Response, ObscuraNetError> {
|
||||||
|
self.fetch_with_method(Method::POST, url, Some(body.as_bytes().to_vec()), callbacks)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn fetch_with_method(
|
pub async fn fetch_with_method(
|
||||||
@@ -440,6 +509,7 @@ impl ObscuraHttpClient {
|
|||||||
initial_method: Method,
|
initial_method: Method,
|
||||||
url: &Url,
|
url: &Url,
|
||||||
initial_body: Option<Vec<u8>>,
|
initial_body: Option<Vec<u8>>,
|
||||||
|
callbacks: Option<&CallbackRegistry>,
|
||||||
) -> Result<Response, ObscuraNetError> {
|
) -> Result<Response, ObscuraNetError> {
|
||||||
validate_url(url, self.allow_private_network)?;
|
validate_url(url, self.allow_private_network)?;
|
||||||
|
|
||||||
@@ -492,8 +562,8 @@ impl ObscuraHttpClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (_, cb) in self.on_request.read().await.iter() {
|
if let Some(cbs) = callbacks {
|
||||||
cb(&request_info);
|
cbs.fire_request(&request_info).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let ua = self.user_agent.read().await.clone();
|
let ua = self.user_agent.read().await.clone();
|
||||||
@@ -638,8 +708,8 @@ impl ObscuraHttpClient {
|
|||||||
redirected_from: redirects,
|
redirected_from: redirects,
|
||||||
};
|
};
|
||||||
|
|
||||||
for (_, cb) in self.on_response.read().await.iter() {
|
if let Some(cbs) = callbacks {
|
||||||
cb(&request_info, &response);
|
cbs.fire_response(&request_info, &response).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
return Ok(response);
|
return Ok(response);
|
||||||
|
|||||||
@@ -8,8 +8,9 @@ pub mod blocklist;
|
|||||||
pub mod wreq_client;
|
pub mod wreq_client;
|
||||||
|
|
||||||
pub use client::{
|
pub use client::{
|
||||||
env_allows_private_network, is_forbidden_ip, ObscuraHttpClient, ObscuraNetError, RequestCallback,
|
env_allows_private_network, is_forbidden_ip, CallbackRegistry, ObscuraHttpClient,
|
||||||
RequestInfo, ResourceType, Response, ResponseCallback, SsrfGuardResolver,
|
ObscuraNetError, RequestCallback, RequestInfo, ResourceType, Response, ResponseCallback,
|
||||||
|
SsrfGuardResolver,
|
||||||
};
|
};
|
||||||
pub use cookies::{default_cookie_path, CookieInfo, CookieJar};
|
pub use cookies::{default_cookie_path, CookieInfo, CookieJar};
|
||||||
pub use encoding::{
|
pub use encoding::{
|
||||||
|
|||||||
@@ -128,9 +128,9 @@ impl Page {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Detach a request callback previously registered with `on_request`.
|
/// Detach a request callback previously registered with `on_request`.
|
||||||
/// Returns true if a callback with that id was removed. Callbacks live on
|
/// Returns true if a callback with that id was removed. Callbacks are
|
||||||
/// the context's shared HTTP client, so detach them when a page or capture
|
/// scoped to this page — they never fire for sibling pages and are
|
||||||
/// phase ends to avoid firing for sibling pages (issue #408).
|
/// dropped with the page (issue #408).
|
||||||
pub fn off_request(&mut self, id: u64) -> bool {
|
pub fn off_request(&mut self, id: u64) -> bool {
|
||||||
self.inner.off_request(id)
|
self.inner.off_request(id)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -108,6 +108,53 @@ async fn page_intercepts_and_observes_js_fetch() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Issue #408 follow-up: callbacks are page-scoped. A callback registered on
|
||||||
|
/// page A must not fire for requests made by page B in the same browser
|
||||||
|
/// context, and must not survive page A being dropped.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn callbacks_do_not_bleed_across_pages() {
|
||||||
|
std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1");
|
||||||
|
let base = spawn_echo_server();
|
||||||
|
|
||||||
|
let browser = Browser::new().unwrap();
|
||||||
|
let mut page_a = browser.new_page().await.unwrap();
|
||||||
|
let mut page_b = browser.new_page().await.unwrap();
|
||||||
|
|
||||||
|
let a_hits = Arc::new(AtomicU32::new(0));
|
||||||
|
let a = a_hits.clone();
|
||||||
|
page_a.on_request(Arc::new(move |_info| {
|
||||||
|
a.fetch_add(1, Ordering::SeqCst);
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Page B navigates (navigation + inline fetch('/api')); page A's callback
|
||||||
|
// must stay silent.
|
||||||
|
page_b.goto(&base).await.unwrap();
|
||||||
|
page_b.settle(500).await;
|
||||||
|
assert_eq!(
|
||||||
|
a_hits.load(Ordering::SeqCst),
|
||||||
|
0,
|
||||||
|
"page A's on_request fired for page B's requests"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Page A navigates; its own callback fires.
|
||||||
|
page_a.goto(&base).await.unwrap();
|
||||||
|
assert!(
|
||||||
|
a_hits.load(Ordering::SeqCst) >= 1,
|
||||||
|
"page A's on_request did not fire for its own navigation"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Dropping page A must not leave its callback firing for page B.
|
||||||
|
let before_drop = a_hits.load(Ordering::SeqCst);
|
||||||
|
drop(page_a);
|
||||||
|
page_b.goto(&base).await.unwrap();
|
||||||
|
page_b.settle(500).await;
|
||||||
|
assert_eq!(
|
||||||
|
a_hits.load(Ordering::SeqCst),
|
||||||
|
before_drop,
|
||||||
|
"dropped page A's on_request still fired for page B's requests"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn page_rewrites_request_url_via_interception() {
|
async fn page_rewrites_request_url_via_interception() {
|
||||||
std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1");
|
std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1");
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ The interception API observes, blocks, mocks, and rewrites the requests a page m
|
|||||||
|
|
||||||
### Passive callbacks
|
### Passive callbacks
|
||||||
|
|
||||||
`on_request` and `on_response` fire for every request and response (navigation and JS `fetch()`/XHR) and are non-blocking. `on_response` is the main path for capturing the JSON an SPA loads asynchronously.
|
`on_request` and `on_response` fire for every request and response (navigation and JS `fetch()`/XHR) and are non-blocking. `on_response` is the main path for capturing the JSON an SPA loads asynchronously. Both return a stable id; pass it to `off_request` / `off_response` to detach the callback when a crawl phase is done. Callbacks are scoped to the page that registered them: they never fire for another page's requests and are dropped with the page.
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
use obscura::{Browser, ResourceType};
|
use obscura::{Browser, ResourceType};
|
||||||
|
|||||||
Reference in new issue
Block a user