diff --git a/.gitbook.yaml b/.gitbook.yaml new file mode 100644 index 0000000..23dad3a --- /dev/null +++ b/.gitbook.yaml @@ -0,0 +1,5 @@ +root: ./docs/ + +structure: + readme: README.md + summary: SUMMARY.md diff --git a/docs/Adding-a-CDP-method-or-Web-API.md b/docs/Adding-a-CDP-method-or-Web-API.md new file mode 100644 index 0000000..0eed392 --- /dev/null +++ b/docs/Adding-a-CDP-method-or-Web-API.md @@ -0,0 +1,154 @@ +Two recipes for the two most common extensions: a new CDP method, and a new JS Web API. + +## Adding a CDP method + +Worked example: `MyDomain.doThing` that takes `{ name }` and returns `{ ok }`. + +### 1. Add the handler + +Create or edit a file under `crates/obscura-cdp/src/domains/`: + +```rust +// crates/obscura-cdp/src/domains/my_domain.rs +use serde_json::{json, Value}; +use crate::dispatch::CdpContext; + +pub async fn do_thing( + params: &Value, + _ctx: &mut CdpContext, + _session_id: &Option, +) -> Result { + let name = params.get("name") + .and_then(|v| v.as_str()) + .ok_or("missing name")?; + + // do the work + + Ok(json!({ "ok": true, "name": name })) +} +``` + +### 2. Register in the dispatcher + +In `crates/obscura-cdp/src/dispatch.rs`, add a match arm: + +```rust +"MyDomain.doThing" => domains::my_domain::do_thing(&req.params, ctx, &req.session_id).await, +``` + +### 3. Test it + +`crates/obscura-cdp/tests/cdp_my_domain.rs`: + +```rust +use obscura_cdp::dispatch::{dispatch, CdpContext}; +use obscura_cdp::types::CdpRequest; +use serde_json::json; + +#[tokio::test(flavor = "current_thread")] +async fn my_domain_do_thing_returns_ok() { + let mut ctx = CdpContext::new(); + let resp = dispatch(&CdpRequest { + id: 1, + method: "MyDomain.doThing".into(), + params: json!({ "name": "test" }), + session_id: None, + }, &mut ctx).await; + + assert!(resp.error.is_none()); + assert_eq!(resp.result.unwrap()["ok"], true); +} +``` + +Run: + +```bash +cargo test -p obscura-cdp my_domain +``` + +## Adding a Web API + +Worked example: `crypto.subtle.digest`, real implementation backed by a Rust hash op. + +### 1. Add the Rust op + +In `crates/obscura-js/src/ops.rs`: + +```rust +#[op2] +#[buffer] +fn op_subtle_digest(#[string] algorithm: &str, #[buffer] data: &[u8]) -> Vec { + use sha1::Digest as _; + match algorithm.to_ascii_uppercase().as_str() { + "SHA-1" => sha1::Sha1::digest(data).to_vec(), + "SHA-256" => sha2::Sha256::digest(data).to_vec(), + "SHA-384" => sha2::Sha384::digest(data).to_vec(), + "SHA-512" => sha2::Sha512::digest(data).to_vec(), + _ => sha2::Sha256::digest(data).to_vec(), + } +} +``` + +### 2. Register the op + +In the same file, `build_extension()`: + +```rust +ops: std::borrow::Cow::Owned(vec![ + op_dom(), + op_console_msg(), + // ... + op_subtle_digest(), +]), +``` + +### 3. Add the JS shim + +In `crates/obscura-js/js/bootstrap.js`: + +```js +globalThis.crypto = globalThis.crypto || {}; +globalThis.crypto.subtle = globalThis.crypto.subtle || {}; +globalThis.crypto.subtle.digest = function digest(algorithm, data) { + const algName = typeof algorithm === 'string' ? algorithm : algorithm.name; + const bytes = data instanceof ArrayBuffer + ? new Uint8Array(data) + : new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + const out = Deno.core.ops.op_subtle_digest(algName, bytes); + return Promise.resolve(out.buffer); +}; +``` + +### 4. Add a dependency if needed + +`crates/obscura-js/Cargo.toml`: + +```toml +sha1 = "0.10" +sha2 = "0.10" +``` + +### 5. Smoke test + +```bash +cargo build --release +./target/release/obscura fetch https://example.com --eval " + crypto.subtle.digest('SHA-256', new TextEncoder().encode('hi')) + .then(buf => Array.from(new Uint8Array(buf)).map(b => b.toString(16).padStart(2, '0')).join('')) +" +``` + +## Tips + +- Keep the JS shim thin. All side effects go through ops. +- Use `Promise.resolve` to keep async-shaped APIs callable from sync ops. +- Match the spec: Web API names and shapes are checked by Puppeteer / Playwright wrappers. +- DOM mutations go through `op_dom`, not new ops. +- For events that need to fire across handlers, use the existing `_makeListenerBox` helper in `bootstrap.js`. + +## Worked examples in the tree + +- CDP method with intercept: `crates/obscura-cdp/src/domains/page.rs` `do_navigate`. +- Web API with op + JS shim: `crypto.subtle.digest` (above). +- Web API in pure JS (no op): `DOMParser` in `bootstrap.js`. +- Web API with async event firing: `WebSocket`, `IntersectionObserver` in `bootstrap.js`. diff --git a/docs/Architecture-overview.md b/docs/Architecture-overview.md new file mode 100644 index 0000000..05b1e64 --- /dev/null +++ b/docs/Architecture-overview.md @@ -0,0 +1,110 @@ +Obscura is a workspace of eight crates. + +``` +obscura-cli CLI entry point. fetch, serve, scrape, mcp. +obscura-cdp Chrome DevTools Protocol server. WebSocket, dispatch, domain handlers. +obscura-browser Page type, navigation, lifecycle events. +obscura-js V8 runtime via deno_core. bootstrap.js + Rust ops. +obscura-dom DOM tree implementation. +obscura-net HTTP client, stealth client, cookie jar, robots cache, tracker blocklist. +obscura-mcp Model Context Protocol server. +obscura Embeddable Rust library API (Browser, Page, Element, CookieStore). +``` + +## Request flow + +A `Page.navigate` from a CDP client: + +``` +CDP client (Puppeteer) + │ WebSocket frame + ▼ +obscura-cdp/server.rs accept, route by sessionId + │ + ▼ +obscura-cdp/dispatch.rs method router, acquires v8_lock + │ + ▼ +obscura-cdp/domains/page.rs Page.navigate handler + │ + ▼ +obscura-browser/page.rs navigate_with_wait + │ + ├──► obscura-net/client.rs HTTP fetch + │ + ├──► obscura-dom/tree.rs parse HTML into the tree + │ + └──► obscura-js/runtime.rs run inline scripts + │ + └──► bootstrap.js + ops.rs DOM bindings +``` + +The dispatcher emits CDP events (`Network.requestWillBeSent`, `Page.frameNavigated`, `Page.lifecycleEvent`) back to the client through the same WebSocket. + +## Single V8 isolate + +All pages in a process share one V8 isolate. The isolate is single-threaded by design. + +`obscura_js::v8_lock::global()` is a `tokio::sync::Mutex` that serializes V8 work. A handler that wants to run JS must acquire the lock first: + +```rust +let _guard = obscura_js::v8_lock::global().lock().await; +page.evaluate(expr).await +``` + +The dispatcher routes long-running operations (navigation, eval) through `process_with_interception` in `server.rs`, which spawns the work onto the tokio `LocalSet` and releases the dispatcher to keep handling other CDP messages. + +This is why `Target.createTarget` from many concurrent clients works: each `newPage` returns immediately while the actual navigation runs in a spawned task. + +## Robustness + +One page cannot hang or crash the process. `obscura-js/runtime.rs` provides a V8 termination watchdog (`arm_watchdog`, `run_event_loop_bounded`) that terminates the isolate from a separate thread when synchronous work overruns a budget, because `tokio::time::timeout` cannot preempt synchronous V8. It bounds the post-load settle, the navigation event-loop pumps, and `--eval`. `obscura-js/cdp_watchdog.rs` is a single shared watchdog the dispatcher arms around every CDP command, so a runaway page cannot hold the V8 lock and wedge other sessions (tunable via `OBSCURA_CDP_COMMAND_TIMEOUT_MS`). `op_dom` is wrapped in `catch_unwind` so a DOM-op panic degrades to a null result instead of aborting the process through V8's FFI frame, and `obscura-dom/tree.rs` rejects cyclic reparenting that would make tree walks loop forever. Scripted `fetch()`/XHR and module loads are timeout-bounded (`OBSCURA_FETCH_TIMEOUT_MS`), and the one-shot `fetch` CLI has a process-level hard deadline as a final backstop. + +## JS bridge + +`obscura-js/js/bootstrap.js` provides the browser globals: `document`, `window`, `navigator`, `location`, observers, fetch, indexedDB, etc. + +`obscura-js/src/ops.rs` registers Rust ops that the bootstrap calls into: + +```js +Deno.core.ops.op_dom('insert_before', parentNid, refNid, newNid); +``` + +Adding a Web API usually means: + +1. JS shim in `bootstrap.js` that exposes the API surface. +2. Rust op in `ops.rs` that performs the side effect (DOM mutation, fetch, crypto). +3. Register the op in `build_extension()`. + +Worked example: [Adding a CDP method or Web API](Adding-a-CDP-method-or-Web-API.md). + +## CDP session model + +Each CDP client connection gets attached to one or more targets. +Session IDs are `"{targetId}-session"`. The dispatcher routes by `sessionId` in the incoming frame to the right `Page`. + +Targets are created by `Target.createTarget`. Closing the WebSocket detaches all sessions but leaves the pages running. + +## Lifecycle + +Lifecycle events are emitted by `obscura-browser/lifecycle.rs` as the page transitions: + +``` +init → commit → domcontentloaded → load → networkidle2 → networkidle0 +``` + +`waitUntil` on `Page.navigate` blocks until the requested level is reached. The Puppeteer / Playwright `goto` resolves on the matching `Page.lifecycleEvent` client-side. + +## Storage + +`--storage-dir` persists cookies (`cookies.json`) and localStorage (`localStorage/.json`). Reads on process start, writes on every navigation and on graceful shutdown. + +## Stealth + +`--stealth` swaps the default `reqwest` client for `obscura-net/wreq_client.rs`, which randomizes TLS ClientHello and cipher order to match a real browser, and applies the bundled tracker blocklist before any request leaves the process. + +## Workspace conventions + +- One crate per layer. Cross-crate calls go through the layer above, not sideways. +- All async is `tokio` with a `LocalSet` because V8 is `!Send`. +- All DOM ops go through `op_dom` to keep the JS/Rust boundary narrow. diff --git a/docs/Build-from-source.md b/docs/Build-from-source.md new file mode 100644 index 0000000..84be00c --- /dev/null +++ b/docs/Build-from-source.md @@ -0,0 +1,60 @@ +## Requirements + +- Rust 1.75+ ([rustup.rs](https://rustup.rs)) +- C compiler (gcc or clang) +- ~5 GB free disk space (V8 compiles from source on first build) + +First build takes about 5 minutes. Incremental builds are seconds. + +## Build + +```bash +git clone https://github.com/h4ckf0r0day/obscura.git +cd obscura +cargo build --release +``` + +Binary is at `./target/release/obscura`. + +## With stealth + +```bash +cargo build --release --features stealth +``` + +Adds TLS fingerprint randomization and the tracker blocklist. See [Configure stealth and proxies](Configure-stealth-and-proxies.md). + +## OpenSSL on older systems + +If the build fails on the vendored OpenSSL with an AVX-512 assembler error (common on older VPS hosts): + +```bash +OPENSSL_NO_VENDOR=1 cargo build --release +``` + +Uses the system OpenSSL instead. + +## Run from the build + +```bash +./target/release/obscura --version +./target/release/obscura fetch https://example.com --eval "document.title" +``` + +Install system-wide: + +```bash +cargo install --path crates/obscura-cli +``` + +## Tests + +```bash +cargo test --release +``` + +Integration suite: + +```bash +python3 tests/test_all.py +``` diff --git a/docs/CLI-reference.md b/docs/CLI-reference.md new file mode 100644 index 0000000..592c4b9 --- /dev/null +++ b/docs/CLI-reference.md @@ -0,0 +1,106 @@ +## `obscura` + +Top-level flags apply to every subcommand. + +``` +-v, --verbose Enable info logging +-p, --port CDP port (default 9222) + --proxy HTTP or SOCKS5 proxy + --obey-robots Respect robots.txt + --user-agent Override the User-Agent + --storage-dir Persistent cookies and localStorage + --allow-private-network Permit loopback / RFC1918 / link-local + --v8-flags Raw V8 flags, applied at startup +-h, --help Help +-V, --version Version +``` + +## `obscura fetch ` + +Load a URL and print its content or an evaluated expression. + +``` + --dump html | text | links | markdown | original | assets + (default html) + --selector Narrow output to a CSS selector + --wait Extra wait after settle (default 5) + --timeout Navigation timeout (default 30) + --wait-until domcontentloaded | load | networkidle2 | networkidle0 + (default load) + --user-agent Override the User-Agent + --proxy HTTP or SOCKS5 proxy + --stealth TLS fingerprint randomization + tracker blocking +-e, --eval Evaluate JS, print the result as JSON +-o, --output Write to a file instead of stdout +-q, --quiet Suppress info logging +-v, --verbose Enable verbose logging +``` + +`--dump` values: + +| Value | Output | +| ---------- | --------------------------------------------------------- | +| `html` | Rendered HTML (default) | +| `text` | Plain text | +| `markdown` | Markdown conversion | +| `links` | Every ``, one URL per line | +| `assets` | Every external resource, one JSON object per line | +| `original` | Raw HTTP response body (binary-safe, bypasses the engine) | + +## `obscura serve` + +Run the CDP server. Puppeteer and Playwright connect over WebSocket. + +``` +-p, --port CDP port (default 9222) + --host Bind host (default 127.0.0.1) + --proxy HTTP or SOCKS5 proxy + --user-agent Override the User-Agent + --stealth TLS fingerprint randomization + tracker blocking + --workers Worker processes (default 1) + --allow-file-access Permit CDP clients to navigate to file:// URLs + --storage-dir Persistent cookies and localStorage + --allow-private-network Permit loopback / RFC1918 / link-local +-v, --verbose Enable info logging +``` + +Default endpoint is `ws://127.0.0.1:9222`. + +## `obscura scrape [URLS]...` + +Run a JS expression across many URLs in parallel. + +``` +-e, --eval JS to run on each page + --concurrency Parallel pages (default 10) + --format Output format (default json) + --timeout Per-URL timeout (default 60) + --proxy HTTP or SOCKS5 proxy + --allow-private-network Permit loopback / RFC1918 / link-local +-q, --quiet Suppress info logging +-v, --verbose Enable verbose logging +``` + +Read URLs from stdin with `-`: + +```bash +cat urls.txt | obscura scrape - --eval "document.title" --concurrency 20 +``` + +Requires `obscura-worker` next to `obscura` in `PATH`. + +## `obscura mcp` + +Run obscura as an MCP server. + +``` + --http HTTP transport instead of stdio + --port HTTP port (default 3000) + --proxy HTTP or SOCKS5 proxy + --user-agent Override the User-Agent + --stealth TLS fingerprint randomization + tracker blocking + --allow-private-network Permit loopback / RFC1918 / link-local +-v, --verbose Enable info logging +``` + +Default transport is stdio. See [Use the MCP server](Use-the-MCP-server.md). diff --git a/docs/Configure-stealth-and-proxies.md b/docs/Configure-stealth-and-proxies.md new file mode 100644 index 0000000..50b13dd --- /dev/null +++ b/docs/Configure-stealth-and-proxies.md @@ -0,0 +1,70 @@ +## Stealth mode + +```bash +obscura fetch https://example.com --stealth +obscura serve --stealth +obscura mcp --stealth +``` + +What `--stealth` changes: + +- Uses the wreq HTTP client with browser-matching TLS fingerprints (ClientHello, ALPN, cipher order). +- Loads a tracker blocklist that drops requests to known analytics and fingerprinting endpoints. +- Bundles webpki roots instead of relying on the system store. + +Requires a build that includes the stealth feature. Release binaries on the Releases page include it. To build it yourself: + +```bash +cargo build --release --features stealth +``` + +## What stealth handles + +- Basic bot detection that checks TLS fingerprint or User-Agent. +- Sites that rely on third-party analytics being reachable. + +## What stealth does not handle + +- Cloudflare interactive challenges. +- Datadome and Akamai bot manager active challenges. +- CAPTCHAs. +- IP-based rate limiting (use proxies). + +## Proxies + +HTTP proxy: + +```bash +obscura fetch https://example.com --proxy http://proxy.example.com:8080 +obscura serve --proxy http://proxy.example.com:8080 +``` + +With auth: + +```bash +obscura fetch https://example.com --proxy http://user:pass@proxy.example.com:8080 +``` + +SOCKS5: + +```bash +obscura fetch https://example.com --proxy socks5://proxy.example.com:1080 +``` + +## Custom User-Agent + +```bash +obscura fetch https://example.com --user-agent "Mozilla/5.0 (...) ..." +obscura serve --user-agent "Mozilla/5.0 (...) ..." +``` + +Default UA matches a recent Chrome on the build platform. + +## Combine + +```bash +obscura serve \ + --stealth \ + --proxy http://user:pass@proxy.example.com:8080 \ + --user-agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ..." +``` diff --git a/docs/Connect-Puppeteer-or-Playwright.md b/docs/Connect-Puppeteer-or-Playwright.md new file mode 100644 index 0000000..1653c02 --- /dev/null +++ b/docs/Connect-Puppeteer-or-Playwright.md @@ -0,0 +1,86 @@ +Obscura speaks the Chrome DevTools Protocol over WebSocket. Puppeteer and Playwright connect to it like remote Chrome. + +## Start the server + +```bash +obscura serve --port 9222 +``` + +``` +obscura listening on ws://127.0.0.1:9222 +``` + +## Puppeteer + +```bash +npm install puppeteer-core +``` + +```js +const puppeteer = require('puppeteer-core'); + +const browser = await puppeteer.connect({ + browserWSEndpoint: 'ws://127.0.0.1:9222', +}); + +const page = await browser.newPage(); +await page.goto('https://example.com'); +console.log(await page.title()); // "Example Domain" + +await browser.disconnect(); +``` + +Use `puppeteer-core`, not `puppeteer`. The `puppeteer` package bundles a Chrome download. + +## Playwright + +```bash +npm install playwright +``` + +```js +const { chromium } = require('playwright'); + +const browser = await chromium.connectOverCDP('ws://127.0.0.1:9222'); +const context = browser.contexts()[0] || await browser.newContext(); +const page = await context.newPage(); + +await page.goto('https://example.com'); +console.log(await page.title()); + +await browser.close(); +``` + +Use `connectOverCDP`, not `connect`. Playwright's `connect` speaks Playwright's own protocol, which obscura does not implement. + +## `waitUntil` + +Default is `domcontentloaded`. For full subresource load: + +```js +await page.goto('https://example.com', { waitUntil: 'load' }); +``` + +| Value | Returns when | +| ------------------ | --------------------------------------- | +| `domcontentloaded` | HTML parsed, scripts ran (default) | +| `load` | All subresources finished | +| `networkidle2` | ≤2 network connections active for 500ms | +| `networkidle0` | 0 network connections active for 500ms | + +## Supported + +- `page.goto`, `page.reload`, `page.goBack`, `page.goForward` +- `page.evaluate`, `page.evaluateHandle` +- `page.click`, `page.type`, `page.fill` +- `page.waitForSelector`, `page.waitForFunction`, `page.waitForNavigation` +- `page.cookies`, `page.setCookie`, `context.cookies` +- `page.setRequestInterception`, block / modify +- `page.exposeFunction` +- `page.content`, `page.title`, `page.url` + +## Not supported + +- `page.screenshot`: obscura doesn't render pixels. +- Per-page V8 isolation: pages share one V8 isolate. A heavy script on one page can stall others. +- Cloudflare / Datadome / Akamai bypass: see [Configure stealth and proxies](Configure-stealth-and-proxies.md). diff --git a/docs/Environment-variables.md b/docs/Environment-variables.md new file mode 100644 index 0000000..05ae02e --- /dev/null +++ b/docs/Environment-variables.md @@ -0,0 +1,83 @@ +## Runtime + +### `OBSCURA_ALLOW_PRIVATE_NETWORK` + +Allow fetches to loopback (`127.0.0.0/8`), RFC1918 (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`), and link-local (`169.254.0.0/16`) addresses. Off by default to block SSRF. + +Truthy values: `1`, `true`, `yes`, `on`. + +```bash +OBSCURA_ALLOW_PRIVATE_NETWORK=1 obscura fetch http://localhost:8080 +``` + +Per-process equivalent: `--allow-private-network` on any subcommand. + +### `OBSCURA_NAV_TIMEOUT_MS` + +Hard ceiling on a single navigation. Default 30000 (30 seconds). Applies to `Page.navigate` and the CLI `fetch` command. + +```bash +OBSCURA_NAV_TIMEOUT_MS=60000 obscura serve +``` + +### `OBSCURA_CDP_COMMAND_TIMEOUT_MS` + +Per-command deadline for the CDP server. A hung page (a runaway `Runtime.evaluate`, a synchronous DOM op) is terminated after this budget so one bad session cannot hold the shared V8 lock and stall the others. Default 60000 (60 seconds); `0` disables it. Navigation self-bounds via `OBSCURA_NAV_TIMEOUT_MS` well under this. + +```bash +OBSCURA_CDP_COMMAND_TIMEOUT_MS=30000 obscura serve +``` + +### `OBSCURA_FETCH_TIMEOUT_MS` + +Request timeout for scripted `fetch()`, `XMLHttpRequest`, and ES-module loads. Without it a request to a server that accepts the connection but never responds (including a CORS preflight) hangs forever and the XHR is stuck with no completion event. Default 30000 (30 seconds). + +```bash +OBSCURA_FETCH_TIMEOUT_MS=15000 obscura serve +``` + +### `OBSCURA_PROXY` + +Default proxy URL used by `obscura-worker` for the parallel `scrape` command when no `--proxy` flag is set. + +```bash +OBSCURA_PROXY=http://proxy.example.com:8080 obscura scrape - < urls.txt +``` + +## Logging + +### `RUST_LOG` + +Standard `tracing` filter. Common settings: + +```bash +RUST_LOG=obscura=info obscura serve +RUST_LOG=obscura=debug obscura serve +RUST_LOG=obscura_cdp=trace,obscura_browser=debug obscura serve +``` + +`--verbose` on the CLI is equivalent to `RUST_LOG=obscura=info`. + +## Build + +### `OPENSSL_NO_VENDOR` + +Forces `cargo build` to use the system OpenSSL instead of compiling the vendored copy. Set to `1` on hosts where the vendored OpenSSL fails (older VPS with AVX-512 issues). + +```bash +OPENSSL_NO_VENDOR=1 cargo build --release +``` + +## V8 + +V8 flags are passed via `--v8-flags`, not environment variables: + +```bash +obscura serve --v8-flags "--max-old-space-size=2048 --expose-gc" +``` + +Default heap is 4 GB on 64-bit systems. + +## HTTP proxy environment + +Obscura does not honor `HTTP_PROXY` / `HTTPS_PROXY` / `NO_PROXY`. Use `--proxy` or `OBSCURA_PROXY`. diff --git a/docs/Extract-data.md b/docs/Extract-data.md new file mode 100644 index 0000000..c37632b --- /dev/null +++ b/docs/Extract-data.md @@ -0,0 +1,75 @@ +`--dump` formats the page output without writing JavaScript. + +```bash +obscura fetch https://example.com --dump html +obscura fetch https://example.com --dump text +obscura fetch https://example.com --dump markdown +obscura fetch https://example.com --dump links +obscura fetch https://example.com --dump assets +obscura fetch https://example.com --dump original +``` + +## `html` + +Rendered HTML after JavaScript runs. Default. + +```bash +obscura fetch https://news.ycombinator.com --dump html > hn.html +``` + +## `text` + +Plain text. No markup. + +```bash +obscura fetch https://en.wikipedia.org/wiki/Rust_(programming_language) --dump text +``` + +## `markdown` + +Markdown conversion: headings, lists, links, code blocks, images. + +```bash +obscura fetch https://docs.example.com/page --dump markdown > page.md +``` + +## `links` + +Every `` on the page, one per line. + +```bash +obscura fetch https://example.com --dump links +``` + +## `assets` + +Every external resource (stylesheets, scripts, images, fonts, iframes), one per line. + +```bash +obscura fetch https://example.com --dump assets +``` + +## `original` + +The raw HTML the server sent, before JavaScript ran. + +```bash +obscura fetch https://my-spa.example --dump original > before.html +obscura fetch https://my-spa.example --dump html > after.html +diff before.html after.html +``` + +## With `--wait-until` + +`--dump` runs after the wait condition: + +```bash +obscura fetch https://my-spa.example --wait-until load --dump markdown +``` + +## Pipe and redirect + +```bash +obscura fetch https://example.com --dump markdown > example.md +obscura fetch https://example.com --dump text --quiet | wc -w +``` diff --git a/docs/Installation.md b/docs/Installation.md new file mode 100644 index 0000000..87e3988 --- /dev/null +++ b/docs/Installation.md @@ -0,0 +1,76 @@ +## Linux x86_64 + +```bash +curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-x86_64-linux.tar.gz +tar xzf obscura-x86_64-linux.tar.gz +./obscura --version +``` + +## Linux ARM64 + +```bash +curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-aarch64-linux.tar.gz +tar xzf obscura-aarch64-linux.tar.gz +./obscura --version +``` + +Linux builds target Ubuntu 22.04 and require glibc 2.35+. + +## macOS Apple Silicon + +```bash +curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-aarch64-macos.tar.gz +tar xzf obscura-aarch64-macos.tar.gz +./obscura --version +``` + +## macOS Intel + +```bash +curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-x86_64-macos.tar.gz +tar xzf obscura-x86_64-macos.tar.gz +./obscura --version +``` + +## Windows + +Download the `.zip` from [Releases](https://github.com/h4ckf0r0day/obscura/releases), extract, run `obscura.exe --version`. + +## Arch Linux (AUR) + +```bash +yay -S obscura-browser +``` + +## Docker + +```bash +docker run -d --name obscura -p 127.0.0.1:9222:9222 h4ckf0r0day/obscura +``` + +Image: [h4ckf0r0day/obscura](https://hub.docker.com/r/h4ckf0r0day/obscura). Built on `distroless/cc`, ~57 MB compressed. + +## From source + +See [Build from source](Build-from-source.md). + +## What's in the archive + +- `obscura`: CLI and CDP server. +- `obscura-worker`: helper for the parallel `scrape` command. Keep both in the same directory. + +## Smoke test + +```bash +./obscura fetch https://example.com --eval "document.title" +``` + +Expected output: `"Example Domain"`. + +## Troubleshooting + +`cannot execute binary file`: wrong arch. Check `uname -m`. + +`GLIBC_2.35 not found`: distro is older than Ubuntu 22.04. Use Docker or build from source. + +macOS Gatekeeper warning: `xattr -d com.apple.quarantine ./obscura`. diff --git a/docs/Intercept-and-modify-requests.md b/docs/Intercept-and-modify-requests.md new file mode 100644 index 0000000..d3300c3 --- /dev/null +++ b/docs/Intercept-and-modify-requests.md @@ -0,0 +1,120 @@ +CDP `Fetch.enable` lets you inspect, block, or modify every request the page makes. + +## Block by resource type + +Puppeteer: + +```js +await page.setRequestInterception(true); + +page.on('request', req => { + if (['image', 'media', 'font'].includes(req.resourceType())) { + req.abort(); + } else { + req.continue(); + } +}); +``` + +Playwright: + +```js +await page.route('**/*', route => { + if (['image', 'media', 'font'].includes(route.request().resourceType())) { + route.abort(); + } else { + route.continue(); + } +}); +``` + +## Block by URL pattern + +```js +// Puppeteer +page.on('request', req => { + const url = req.url(); + if (url.includes('google-analytics.com') || url.includes('doubleclick.net')) { + req.abort(); + } else { + req.continue(); + } +}); +``` + +```js +// Playwright +await page.route(/google-analytics\.com|doubleclick\.net/, route => route.abort()); +``` + +## Modify headers + +```js +// Puppeteer +page.on('request', req => { + req.continue({ + headers: { ...req.headers(), 'X-Custom': 'value' }, + }); +}); +``` + +```js +// Playwright +await page.route('**/*', route => { + route.continue({ + headers: { ...route.request().headers(), 'X-Custom': 'value' }, + }); +}); +``` + +## Return a fake response + +```js +// Puppeteer +page.on('request', req => { + if (req.url().endsWith('/api/feature-flags')) { + req.respond({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ newDashboard: true }), + }); + } else { + req.continue(); + } +}); +``` + +```js +// Playwright +await page.route('**/api/feature-flags', route => { + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ newDashboard: true }), + }); +}); +``` + +## Strip analytics in production scrapes + +```js +const BLOCK = [ + 'google-analytics.com', + 'googletagmanager.com', + 'doubleclick.net', + 'facebook.net', + 'segment.io', + 'mixpanel.com', + 'hotjar.com', +]; + +page.on('request', req => { + if (BLOCK.some(host => req.url().includes(host))) { + req.abort(); + } else { + req.continue(); + } +}); +``` + +Built-in: `--stealth` ships with a tracker blocklist that handles most of these without per-script setup. See [Configure stealth and proxies](Configure-stealth-and-proxies.md). diff --git a/docs/Markdown-extraction.md b/docs/Markdown-extraction.md new file mode 100644 index 0000000..13dfe44 --- /dev/null +++ b/docs/Markdown-extraction.md @@ -0,0 +1,57 @@ +`--dump markdown` converts the rendered page to markdown. + +```bash +obscura fetch https://example.com --dump markdown +``` + +## What gets converted + +- Headings (`

` through `

`) +- Paragraphs, line breaks +- Bold, italic, code spans +- Links (with `href`) +- Images (with `src` and `alt`) +- Ordered and unordered lists +- Block quotes +- Code blocks (`
`, ``)
+- Tables
+
+## What gets stripped
+
+- `