feat: cookie jar persistence + --storage-dir flag + fix HTTP cookie header filtering (#114)

* feat: cookie jar persistence + storage-dir + filtering fix (clean v4)

* fix: resolve build errors, add storage_dir param throughout server chain

* fix: storage_dir end-to-end, plus two test-literal field additions

  Five issues found while testing PR 114 end-to-end.

  Build fixes (two test literals out of sync with new struct fields):
  - crates/obscura-net/src/cookies.rs: CookieInfo literal in
    test_domain_matches_subdomain_without_leading_dot was missing the
    same_site and expires fields added in #170.
  - crates/obscura-cli/src/main.rs: Command::Fetch literal in
    matcher_still_uses_fetch_variant was missing the new storage_dir
    field. Without these cargo test --workspace would not compile.

  run_fetch was branching between BrowserContext::with_storage (which
  hardcodes proxy=None, stealth=false, user_agent=None) and
  with_options (which has no storage_dir slot), so --storage-dir +
  --proxy + --stealth silently dropped the proxy and stealth flags.
  Same bug SGavrl flagged on dispatch.rs in the previous round, just on
  the other code path. Switched to with_storage_full which carries all
  four. Verified the fetch now errors out against an invalid proxy.

  run_fetch --eval short-circuited the function before
  context.save_cookies() ran, so cookies set during an eval session
  were never persisted to {storage_dir}/cookies.json. Moved the save
  above the return.

  obscura serve loaded cookies on startup but never saved them
  anywhere, so a session that set cookies via Network.setCookie or
  navigation dropped them on shutdown. Subscribed to Ctrl-C once in
  put a single save_cookies() after the loop so both the Ctrl-C path
  and the channel-closed path persist cookies. Verified end-to-end.

---------

Co-authored-by: zhangyang-crazy-one <[email protected]>
Co-authored-by: SGavrl <[email protected]>

crates/obscura-cli/src/main.rs

@@ -32,6 +32,9 @@ struct Args {
     #[arg(long)]
     user_agent: Option<String>,
 
+    #[arg(long)]
+    storage_dir: Option<std::path::PathBuf>,
+
     /// Pass raw flags to V8, in the same form V8/Chromium/Node accept
     /// (e.g. `"--max-old-space-size=4096 --max-semi-space-size=64 --expose-gc"`).
     /// Applied once at startup before any isolate is created.
@@ -70,6 +73,9 @@ enum Command {
         /// and the port is on a trusted network.
         #[arg(long)]
         allow_file_access: bool,
+
+        #[arg(long)]
+        storage_dir: Option<std::path::PathBuf>,
     },
 
     Fetch {
@@ -104,6 +110,9 @@ enum Command {
 
         #[arg(long, short)]
         quiet: bool,
+
+        #[arg(long)]
+        storage_dir: Option<std::path::PathBuf>,
     },
 
     Scrape {
@@ -252,10 +261,13 @@ async fn main() -> anyhow::Result<()> {
     let global_proxy = args.proxy.clone();
 
     match args.command {
-        Some(Command::Serve { port, host, proxy, user_agent, stealth, workers, allow_file_access }) => {
+        Some(Command::Serve { port, host, proxy, user_agent, stealth, workers, allow_file_access, storage_dir }) => {
             let proxy = merge_proxy(global_proxy.clone(), proxy);
             reject_stealth_with_socks5(proxy.as_deref(), stealth)?;
             print_banner(port);
+            if let Some(ref dir) = storage_dir {
+                tracing::info!("Storage dir: {}", dir.display());
+            }
             if let Some(ref proxy) = proxy {
                 tracing::info!("Using proxy: {}", proxy);
             }
@@ -276,13 +288,13 @@ async fn main() -> anyhow::Result<()> {
                 run_multi_worker_serve(port, workers, proxy, stealth, user_agent).await?;
             } else {
                 obscura_cdp::start_with_host_and_security(
-                    port, &host, proxy, stealth, user_agent, allow_file_access,
+                    port, &host, proxy, stealth, user_agent, allow_file_access, storage_dir,
                 ).await?;
             }
         }
-        Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, output, quiet }) => {
+        Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, output, quiet, storage_dir }) => {
             reject_stealth_with_socks5(global_proxy.as_deref(), stealth)?;
-            run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, output, quiet, global_proxy).await?;
+            run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, output, quiet, global_proxy, storage_dir).await?;
         }
         Some(Command::Scrape { urls, eval, concurrency, format, timeout, quiet }) => {
             run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout, quiet, global_proxy).await?;
@@ -448,6 +460,7 @@ async fn run_fetch(
     output: Option<std::path::PathBuf>,
     quiet: bool,
     proxy: Option<String>,
+    storage_dir: Option<std::path::PathBuf>,
 ) -> anyhow::Result<()> {
     // --dump original short-circuits the browser stack entirely: fetch the raw
     // response body via HTTP and stream the bytes verbatim. Useful for binary
@@ -465,8 +478,14 @@ async fn run_fetch(
         return Ok(());
     }
 
-    let context = Arc::new(BrowserContext::with_options("fetch".to_string(), proxy, stealth));
-    let mut page = Page::new("fetch-page".to_string(), context);
+    let context = Arc::new(BrowserContext::with_storage_full(
+        "fetch".to_string(),
+        proxy,
+        stealth,
+        user_agent.clone(),
+        storage_dir.clone(),
+    ));
+    let mut page = Page::new("fetch-page".to_string(), context.clone());
 
     if let Some(ref ua) = user_agent {
         page.http_client.set_user_agent(ua).await;
@@ -506,6 +525,7 @@ async fn run_fetch(
             other => other.to_string(),
         };
         write_or_print(rendered, output.as_ref()).await?;
+        context.save_cookies();
         return Ok(());
     }
 
@@ -520,6 +540,9 @@ async fn run_fetch(
     };
     write_or_print(rendered, output.as_ref()).await?;
 
+    // Save cookies to disk if storage_dir is configured
+    context.save_cookies();
+
     Ok(())
 }
 
@@ -1364,6 +1387,7 @@ mod tests {
             eval: None,
             quiet: true,
             output: None,
+            storage_dir: None,
         });
         assert!(is_quiet_command(&cmd));
     }