V8 embedded defaults to ~1.4 GB max-old-space on 64-bit. Pages with heavy fingerprinting or analytics bundles (e.g. demo.fingerprint.com/ web-scraping) blow past it and the worker dies with 'Fatal JavaScript out of memory' and SIGTRAP, taking the CDP session with it. Always set --max-old-space-size=4096 (1024 on 32-bit targets) before V8 startup, matching headless Chrome's default. User-supplied --v8-flags are appended after the default so they win (V8 parses left-to-right and the last value wins), which keeps the existing escape hatch intact for memory-constrained environments.
This commit is contained in:
1 parent
682431f766
commit
1a1253118f
1 file changed
+49
-8
@@ -239,6 +239,24 @@ fn normalize_v8_flags(raw: Option<&str>) -> Option<String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Default V8 flags applied at startup unless the user disabled them via
|
||||||
|
/// `--v8-flags`. The default heap matches headless Chrome (~4 GB) so pages
|
||||||
|
/// that ship heavy fingerprinting or analytics bundles
|
||||||
|
/// (e.g. demo.fingerprint.com — issue #199) don't SIGTRAP out of the box.
|
||||||
|
/// V8 parses flags left-to-right and later wins, so anything the user
|
||||||
|
/// passes via `--v8-flags` overrides these.
|
||||||
|
#[cfg(target_pointer_width = "64")]
|
||||||
|
const DEFAULT_V8_FLAGS: &str = "--max-old-space-size=4096";
|
||||||
|
#[cfg(not(target_pointer_width = "64"))]
|
||||||
|
const DEFAULT_V8_FLAGS: &str = "--max-old-space-size=1024";
|
||||||
|
|
||||||
|
fn effective_v8_flags(user: Option<&str>) -> String {
|
||||||
|
match normalize_v8_flags(user) {
|
||||||
|
Some(u) => format!("{} {}", DEFAULT_V8_FLAGS, u),
|
||||||
|
None => DEFAULT_V8_FLAGS.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::main(flavor = "current_thread")]
|
#[tokio::main(flavor = "current_thread")]
|
||||||
async fn main() -> anyhow::Result<()> {
|
async fn main() -> anyhow::Result<()> {
|
||||||
let args = Args::parse();
|
let args = Args::parse();
|
||||||
@@ -253,10 +271,9 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
.with_writer(std::io::stderr)
|
.with_writer(std::io::stderr)
|
||||||
.init();
|
.init();
|
||||||
|
|
||||||
if let Some(flags) = normalize_v8_flags(args.v8_flags.as_deref()) {
|
let v8_flags = effective_v8_flags(args.v8_flags.as_deref());
|
||||||
tracing::info!("Applying V8 flags: {}", flags);
|
tracing::debug!("V8 flags: {}", v8_flags);
|
||||||
obscura_js::set_v8_flags(&flags);
|
obscura_js::set_v8_flags(&v8_flags);
|
||||||
}
|
|
||||||
|
|
||||||
let global_proxy = args.proxy.clone();
|
let global_proxy = args.proxy.clone();
|
||||||
|
|
||||||
@@ -1073,10 +1090,10 @@ fn dump_assets(page: &Page) -> String {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
extract_assets, extract_readable_text, fetch_original_bytes, is_quiet_command,
|
effective_v8_flags, extract_assets, extract_readable_text, fetch_original_bytes,
|
||||||
link_kind_from_rel, merge_proxy, normalize_v8_flags, reject_stealth_with_socks5,
|
is_quiet_command, link_kind_from_rel, merge_proxy, normalize_v8_flags,
|
||||||
resolve_asset_url, select_log_filter, write_or_print, write_or_print_bytes, Args,
|
reject_stealth_with_socks5, resolve_asset_url, select_log_filter, write_or_print,
|
||||||
Command, DumpFormat,
|
write_or_print_bytes, Args, Command, DumpFormat, DEFAULT_V8_FLAGS,
|
||||||
};
|
};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use obscura_dom::parse_html;
|
use obscura_dom::parse_html;
|
||||||
@@ -1325,6 +1342,30 @@ mod tests {
|
|||||||
assert_eq!(normalize_v8_flags(Some(input)).as_deref(), Some(input));
|
assert_eq!(normalize_v8_flags(Some(input)).as_deref(), Some(input));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn effective_v8_flags_returns_default_when_unset() {
|
||||||
|
assert_eq!(effective_v8_flags(None), DEFAULT_V8_FLAGS);
|
||||||
|
assert_eq!(effective_v8_flags(Some("")), DEFAULT_V8_FLAGS);
|
||||||
|
assert_eq!(effective_v8_flags(Some(" ")), DEFAULT_V8_FLAGS);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn effective_v8_flags_user_overrides_default() {
|
||||||
|
// V8 parses left-to-right and later wins, so the user value must
|
||||||
|
// come after the default in the merged string.
|
||||||
|
let user = "--max-old-space-size=8192";
|
||||||
|
let merged = effective_v8_flags(Some(user));
|
||||||
|
assert!(merged.starts_with(DEFAULT_V8_FLAGS));
|
||||||
|
assert!(merged.ends_with(user));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn effective_v8_flags_appends_user_extras() {
|
||||||
|
let merged = effective_v8_flags(Some("--expose-gc"));
|
||||||
|
assert!(merged.contains(DEFAULT_V8_FLAGS));
|
||||||
|
assert!(merged.contains("--expose-gc"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reject_stealth_with_socks5_passes_when_no_stealth() {
|
fn reject_stealth_with_socks5_passes_when_no_stealth() {
|
||||||
assert!(reject_stealth_with_socks5(Some("socks5://127.0.0.1:1080"), false).is_ok());
|
assert!(reject_stealth_with_socks5(Some("socks5://127.0.0.1:1080"), false).is_ok());
|
||||||
|
|||||||
Reference in new issue
Block a user