Initial Release
This commit is contained in:
h4ckf0r0day
2
.cargo/config.toml
Normal file
2
.cargo/config.toml
Normal file
@@ -0,0 +1,2 @@
|
||||
[env]
|
||||
RUST_TEST_THREADS = "1"
|
||||
73
.github/workflows/release.yml
vendored
Normal file
73
.github/workflows/release.yml
vendored
Normal file
@@ -0,0 +1,73 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
os: ubuntu-latest
|
||||
name: obscura-x86_64-linux
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
os: ubuntu-latest
|
||||
name: obscura-aarch64-linux
|
||||
- target: x86_64-apple-darwin
|
||||
os: macos-latest
|
||||
name: obscura-x86_64-macos
|
||||
- target: aarch64-apple-darwin
|
||||
os: macos-latest
|
||||
name: obscura-aarch64-macos
|
||||
- target: x86_64-pc-windows-msvc
|
||||
os: windows-latest
|
||||
name: obscura-x86_64-windows
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install cross-compilation tools
|
||||
if: matrix.target == 'aarch64-unknown-linux-gnu'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
|
||||
- name: Build
|
||||
run: cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Build stealth
|
||||
run: cargo build --release --target ${{ matrix.target }} --features stealth
|
||||
continue-on-error: true
|
||||
|
||||
- name: Package (Unix)
|
||||
if: runner.os != 'Windows'
|
||||
run: |
|
||||
cd target/${{ matrix.target }}/release
|
||||
tar czf ../../../${{ matrix.name }}.tar.gz obscura
|
||||
cd ../../..
|
||||
|
||||
- name: Package (Windows)
|
||||
if: runner.os == 'Windows'
|
||||
run: |
|
||||
cd target/${{ matrix.target }}/release
|
||||
7z a ../../../${{ matrix.name }}.zip obscura.exe
|
||||
cd ../../..
|
||||
|
||||
- name: Upload
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
${{ matrix.name }}.tar.gz
|
||||
${{ matrix.name }}.zip
|
||||
3531
Cargo.lock
generated
Normal file
3531
Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
41
Cargo.toml
Normal file
41
Cargo.toml
Normal file
@@ -0,0 +1,41 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"crates/obscura-dom",
|
||||
"crates/obscura-net",
|
||||
"crates/obscura-browser",
|
||||
"crates/obscura-cdp",
|
||||
"crates/obscura-js",
|
||||
"crates/obscura-cli",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/h4ckf0r0day/obscura"
|
||||
|
||||
[workspace.dependencies]
|
||||
obscura-dom = { path = "crates/obscura-dom" }
|
||||
obscura-net = { path = "crates/obscura-net" }
|
||||
obscura-browser = { path = "crates/obscura-browser" }
|
||||
obscura-cdp = { path = "crates/obscura-cdp" }
|
||||
obscura-js = { path = "crates/obscura-js" }
|
||||
|
||||
html5ever = "0.29"
|
||||
markup5ever = "0.14"
|
||||
selectors = "0.26"
|
||||
servo_arc = "0.4"
|
||||
cssparser = "0.34"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-tungstenite = "0.26"
|
||||
reqwest = { version = "0.12", features = ["cookies", "gzip", "brotli", "deflate", "native-tls-vendored"], default-features = false }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
url = "2"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
thiserror = "2"
|
||||
anyhow = "1"
|
||||
248
README.md
Normal file
248
README.md
Normal file
@@ -0,0 +1,248 @@
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/h4ckf0r0day/obscura/main/assets/icon.png" alt="Obscura" width="80" />
|
||||
</p>
|
||||
|
||||
<h2 align="center">Obscura</h1>
|
||||
|
||||
<p align="center">
|
||||
<strong>The open-source headless browser for AI agents and web scraping.</strong><br>
|
||||
Lightweight, stealthy, and built in Rust.
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
Obscura is a headless browser engine written in Rust, built for web scraping and AI agent automation. It runs real JavaScript via V8, supports the Chrome DevTools Protocol, and acts as a drop-in replacement for headless Chrome with Puppeteer and Playwright.
|
||||
|
||||
### Why Obscura over headless Chrome?
|
||||
|
||||
Designed for automation at scale, not desktop browsing.
|
||||
|
||||
| Metric | Obscura | Headless Chrome |
|
||||
|--------------|--------------|------------------|
|
||||
| Memory | **30 MB** | 200+ MB |
|
||||
| Binary size | **70 MB** | 300+ MB |
|
||||
| Anti-detect | **Built-in** | None |
|
||||
| Page load | **85 ms** | ~500 ms |
|
||||
| Startup | **Instant** | ~2s |
|
||||
| Puppeteer | **Yes** | Yes |
|
||||
| Playwright | **Yes** | Yes |
|
||||
|
||||
## Install
|
||||
|
||||
### Download
|
||||
|
||||
Grab the latest binary from [Releases](https://github.com/h4ckf0r0day/obscura/releases):
|
||||
|
||||
```bash
|
||||
# Linux x86_64
|
||||
curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-x86_64-linux.tar.gz
|
||||
tar xzf obscura-x86_64-linux.tar.gz
|
||||
./obscura fetch https://example.com --eval "document.title"
|
||||
|
||||
# macOS Apple Silicon
|
||||
curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-aarch64-macos.tar.gz
|
||||
tar xzf obscura-aarch64-macos.tar.gz
|
||||
|
||||
# macOS Intel
|
||||
curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-x86_64-macos.tar.gz
|
||||
tar xzf obscura-x86_64-macos.tar.gz
|
||||
|
||||
# Windows
|
||||
Download the `.zip` from the releases page and extract it manually.
|
||||
```
|
||||
|
||||
Single binary. No Chrome, no Node.js, no dependencies.
|
||||
|
||||
### Build from source
|
||||
|
||||
```bash
|
||||
git clone https://github.com/h4ckf0r0day/obscura.git
|
||||
cd obscura
|
||||
cargo build --release
|
||||
|
||||
# With stealth mode (anti-detection + tracker blocking)
|
||||
cargo build --release --features stealth
|
||||
```
|
||||
|
||||
Requires Rust 1.75+ ([rustup.rs](https://rustup.rs)). First build takes ~5 min (V8 compiles from source, cached after).
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Fetch a page
|
||||
|
||||
```bash
|
||||
# Get the page title
|
||||
obscura fetch https://example.com --eval "document.title"
|
||||
|
||||
# Extract all links
|
||||
obscura fetch https://example.com --dump links
|
||||
|
||||
# Render JavaScript and dump HTML
|
||||
obscura fetch https://news.ycombinator.com --dump html
|
||||
|
||||
# Wait for dynamic content
|
||||
obscura fetch https://example.com --wait-until networkidle0
|
||||
```
|
||||
|
||||
### Start the CDP server
|
||||
|
||||
```bash
|
||||
obscura serve --port 9222
|
||||
|
||||
# With stealth mode (anti-detection + tracker blocking)
|
||||
obscura serve --port 9222 --stealth
|
||||
```
|
||||
|
||||
### Scrape in parallel
|
||||
|
||||
```bash
|
||||
obscura scrape url1 url2 url3 ... \
|
||||
--concurrency 25 \
|
||||
--eval "document.querySelector('h1').textContent" \
|
||||
--format json
|
||||
```
|
||||
|
||||
## Puppeteer / Playwright
|
||||
|
||||
### Puppeteer
|
||||
|
||||
```bash
|
||||
npm install puppeteer-core
|
||||
```
|
||||
|
||||
```javascript
|
||||
import puppeteer from 'puppeteer-core';
|
||||
|
||||
const browser = await puppeteer.connect({
|
||||
browserWSEndpoint: 'ws://127.0.0.1:9222/devtools/browser',
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.goto('https://news.ycombinator.com');
|
||||
|
||||
const stories = await page.evaluate(() =>
|
||||
Array.from(document.querySelectorAll('.titleline > a'))
|
||||
.map(a => ({ title: a.textContent, url: a.href }))
|
||||
);
|
||||
console.log(stories);
|
||||
|
||||
await browser.disconnect();
|
||||
```
|
||||
|
||||
### Playwright
|
||||
|
||||
```bash
|
||||
npm install playwright-core
|
||||
```
|
||||
|
||||
```javascript
|
||||
import { chromium } from 'playwright-core';
|
||||
|
||||
const browser = await chromium.connectOverCDP({
|
||||
endpointURL: 'ws://127.0.0.1:9222',
|
||||
});
|
||||
|
||||
const page = await browser.newContext().then(ctx => ctx.newPage());
|
||||
await page.goto('https://en.wikipedia.org/wiki/Web_scraping');
|
||||
console.log(await page.title());
|
||||
|
||||
await browser.close();
|
||||
```
|
||||
|
||||
### Form submission & login
|
||||
|
||||
```javascript
|
||||
await page.goto('https://quotes.toscrape.com/login');
|
||||
await page.evaluate(() => {
|
||||
document.querySelector('#username').value = 'admin';
|
||||
document.querySelector('#password').value = 'admin';
|
||||
document.querySelector('form').submit();
|
||||
});
|
||||
// Obscura handles the POST, follows the 302 redirect, maintains cookies
|
||||
```
|
||||
|
||||
## Benchmarks
|
||||
|
||||
Page load:
|
||||
|
||||
| Page | Obscura | Chrome |
|
||||
|------|---------|--------|
|
||||
| Static HTML | **51 ms** | ~500 ms |
|
||||
| JS + XHR + fetch | **84 ms** | ~800 ms |
|
||||
| Dynamic scripts | **78 ms** | ~700 ms |
|
||||
|
||||
## Stealth Mode
|
||||
|
||||
Enable with `--features stealth`.
|
||||
|
||||
### Anti-fingerprinting
|
||||
- Per-session fingerprint randomization (GPU, screen, canvas, audio, battery)
|
||||
- Realistic `navigator.userAgentData` (Chrome 145, high-entropy values)
|
||||
- `event.isTrusted = true` for dispatched events
|
||||
- Hidden internal properties (`Object.keys(window)` safe)
|
||||
- Native function masking (`Function.prototype.toString()` → `[native code]`)
|
||||
- `navigator.webdriver = undefined` (matches real Chrome)
|
||||
|
||||
### Tracker Blocking
|
||||
- 3,520 domains blocked
|
||||
- Blocks analytics, ads, telemetry, and fingerprinting scripts
|
||||
- Prevents trackers from loading entirely
|
||||
- Enabled automatically with `--stealth`
|
||||
|
||||
## CDP API
|
||||
|
||||
Obscura implements the Chrome DevTools Protocol for Puppeteer/Playwright compatibility.
|
||||
|
||||
| Domain | Methods |
|
||||
|--------|---------|
|
||||
| **Target** | createTarget, closeTarget, attachToTarget, createBrowserContext, disposeBrowserContext |
|
||||
| **Page** | navigate, getFrameTree, addScriptToEvaluateOnNewDocument, lifecycleEvents |
|
||||
| **Runtime** | evaluate, callFunctionOn, getProperties, addBinding |
|
||||
| **DOM** | getDocument, querySelector, querySelectorAll, getOuterHTML, resolveNode |
|
||||
| **Network** | enable, setCookies, getCookies, setExtraHTTPHeaders, setUserAgentOverride |
|
||||
| **Fetch** | enable, continueRequest, fulfillRequest, failRequest (live interception) |
|
||||
| **Storage** | getCookies, setCookies, deleteCookies |
|
||||
| **Input** | dispatchMouseEvent, dispatchKeyEvent |
|
||||
| **LP** | getMarkdown (DOM-to-Markdown conversion) |
|
||||
## CLI Reference
|
||||
|
||||
### `obscura serve`
|
||||
|
||||
Start a CDP WebSocket server.
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--port` | `9222` | WebSocket port |
|
||||
| `--proxy` | — | HTTP/SOCKS5 proxy URL |
|
||||
| `--stealth` | off | Enable anti-detection + tracker blocking |
|
||||
| `--workers` | `1` | Number of parallel worker processes |
|
||||
| `--obey-robots` | off | Respect robots.txt |
|
||||
|
||||
### `obscura fetch <URL>`
|
||||
|
||||
Fetch and render a single page.
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--dump` | `html` | Output: `html`, `text`, or `links` |
|
||||
| `--eval` | — | JavaScript expression to evaluate |
|
||||
| `--wait-until` | `load` | Wait: `load`, `domcontentloaded`, `networkidle0` |
|
||||
| `--selector` | — | Wait for CSS selector |
|
||||
| `--stealth` | off | Anti-detection mode |
|
||||
| `--quiet` | off | Suppress banner |
|
||||
|
||||
### `obscura scrape <URL...>`
|
||||
|
||||
Scrape multiple URLs in parallel with worker processes.
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--concurrency` | `10` | Parallel workers |
|
||||
| `--eval` | — | JS expression per page |
|
||||
| `--format` | `json` | Output: `json` or `text` |
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
|
||||
---
|
||||
BIN
assets/icon.png
Normal file
BIN
assets/icon.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
20
crates/obscura-browser/Cargo.toml
Normal file
20
crates/obscura-browser/Cargo.toml
Normal file
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "obscura-browser"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
stealth = ["obscura-net/stealth"]
|
||||
|
||||
[dependencies]
|
||||
obscura-dom = { workspace = true }
|
||||
obscura-net = { workspace = true }
|
||||
obscura-js = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
futures = "0.3"
|
||||
url = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
57
crates/obscura-browser/src/context.rs
Normal file
57
crates/obscura-browser/src/context.rs
Normal file
@@ -0,0 +1,57 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use obscura_net::{CookieJar, ObscuraHttpClient, RobotsCache};
|
||||
|
||||
pub struct BrowserContext {
|
||||
pub id: String,
|
||||
pub cookie_jar: Arc<CookieJar>,
|
||||
pub http_client: Arc<ObscuraHttpClient>,
|
||||
pub user_agent: String,
|
||||
pub proxy_url: Option<String>,
|
||||
pub robots_cache: Arc<RobotsCache>,
|
||||
pub obey_robots: bool,
|
||||
pub stealth: bool,
|
||||
}
|
||||
|
||||
impl BrowserContext {
|
||||
pub fn new(id: String) -> Self {
|
||||
let cookie_jar = Arc::new(CookieJar::new());
|
||||
let http_client = Arc::new(ObscuraHttpClient::with_cookie_jar(cookie_jar.clone()));
|
||||
BrowserContext {
|
||||
id,
|
||||
cookie_jar,
|
||||
http_client,
|
||||
user_agent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36".to_string(),
|
||||
proxy_url: None,
|
||||
robots_cache: Arc::new(RobotsCache::new()),
|
||||
obey_robots: false,
|
||||
stealth: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_options(id: String, proxy_url: Option<String>, stealth: bool) -> Self {
|
||||
let cookie_jar = Arc::new(CookieJar::new());
|
||||
let mut client = ObscuraHttpClient::with_options(
|
||||
cookie_jar.clone(),
|
||||
proxy_url.as_deref(),
|
||||
);
|
||||
if stealth {
|
||||
client.block_trackers = true;
|
||||
}
|
||||
let http_client = Arc::new(client);
|
||||
BrowserContext {
|
||||
id,
|
||||
cookie_jar,
|
||||
http_client,
|
||||
user_agent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36".to_string(),
|
||||
proxy_url,
|
||||
robots_cache: Arc::new(RobotsCache::new()),
|
||||
obey_robots: false,
|
||||
stealth,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_proxy(id: String, proxy_url: Option<String>) -> Self {
|
||||
Self::with_options(id, proxy_url, false)
|
||||
}
|
||||
}
|
||||
7
crates/obscura-browser/src/lib.rs
Normal file
7
crates/obscura-browser/src/lib.rs
Normal file
@@ -0,0 +1,7 @@
|
||||
pub mod page;
|
||||
pub mod context;
|
||||
pub mod lifecycle;
|
||||
|
||||
pub use page::{Page, PageError};
|
||||
pub use context::BrowserContext;
|
||||
pub use lifecycle::{LifecycleState, WaitUntil};
|
||||
42
crates/obscura-browser/src/lifecycle.rs
Normal file
42
crates/obscura-browser/src/lifecycle.rs
Normal file
@@ -0,0 +1,42 @@
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LifecycleState {
|
||||
Idle,
|
||||
Loading,
|
||||
DomContentLoaded,
|
||||
Loaded,
|
||||
NetworkIdle,
|
||||
Failed,
|
||||
}
|
||||
|
||||
impl LifecycleState {
|
||||
pub fn is_loading(&self) -> bool {
|
||||
matches!(self, LifecycleState::Loading)
|
||||
}
|
||||
|
||||
pub fn is_loaded(&self) -> bool {
|
||||
matches!(self, LifecycleState::Loaded | LifecycleState::NetworkIdle)
|
||||
}
|
||||
|
||||
pub fn is_network_idle(&self) -> bool {
|
||||
matches!(self, LifecycleState::NetworkIdle)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum WaitUntil {
|
||||
Load,
|
||||
DomContentLoaded,
|
||||
NetworkIdle0,
|
||||
NetworkIdle2,
|
||||
}
|
||||
|
||||
impl WaitUntil {
|
||||
pub fn from_str(s: &str) -> Self {
|
||||
match s {
|
||||
"domcontentloaded" => WaitUntil::DomContentLoaded,
|
||||
"networkidle0" | "networkIdle" | "networkidle" => WaitUntil::NetworkIdle0,
|
||||
"networkidle2" => WaitUntil::NetworkIdle2,
|
||||
_ => WaitUntil::Load,
|
||||
}
|
||||
}
|
||||
}
|
||||
859
crates/obscura-browser/src/page.rs
Normal file
859
crates/obscura-browser/src/page.rs
Normal file
@@ -0,0 +1,859 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use obscura_dom::{parse_html, DomTree};
|
||||
use obscura_js::runtime::ObscuraJsRuntime;
|
||||
use obscura_net::{ObscuraHttpClient, ObscuraNetError, Response};
|
||||
use url::Url;
|
||||
|
||||
use crate::context::BrowserContext;
|
||||
use crate::lifecycle::LifecycleState;
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
use obscura_net::StealthHttpClient;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NetworkEvent {
|
||||
pub request_id: String,
|
||||
pub url: String,
|
||||
pub method: String,
|
||||
pub resource_type: String,
|
||||
pub status: u16,
|
||||
pub headers: std::collections::HashMap<String, String>,
|
||||
pub response_headers: Arc<std::collections::HashMap<String, String>>,
|
||||
pub body_size: usize,
|
||||
pub timestamp: f64,
|
||||
}
|
||||
|
||||
pub struct Page {
|
||||
pub id: String,
|
||||
pub frame_id: String,
|
||||
pub url: Option<Url>,
|
||||
pub dom: Option<DomTree>,
|
||||
pub js: Option<ObscuraJsRuntime>,
|
||||
pub lifecycle: LifecycleState,
|
||||
pub http_client: Arc<ObscuraHttpClient>,
|
||||
pub context: Arc<BrowserContext>,
|
||||
pub title: String,
|
||||
pub network_events: Vec<NetworkEvent>,
|
||||
network_event_counter: u32,
|
||||
pub intercept_enabled: bool,
|
||||
pub intercept_block_patterns: Vec<String>,
|
||||
intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<obscura_js::ops::InterceptedRequest>>,
|
||||
#[cfg(feature = "stealth")]
|
||||
pub stealth_client: Option<Arc<StealthHttpClient>>,
|
||||
}
|
||||
|
||||
impl Page {
|
||||
pub fn new(id: String, context: Arc<BrowserContext>) -> Self {
|
||||
let http_client = context.http_client.clone();
|
||||
let frame_id = format!("{}.1", id);
|
||||
#[cfg(feature = "stealth")]
|
||||
let stealth_client = if context.stealth {
|
||||
Some(Arc::new(StealthHttpClient::new(context.cookie_jar.clone())))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Page {
|
||||
id,
|
||||
frame_id,
|
||||
url: None,
|
||||
dom: None,
|
||||
js: None,
|
||||
lifecycle: LifecycleState::Idle,
|
||||
http_client,
|
||||
context,
|
||||
title: String::new(),
|
||||
network_events: Vec::new(),
|
||||
network_event_counter: 0,
|
||||
intercept_enabled: false,
|
||||
intercept_block_patterns: Vec::new(),
|
||||
intercept_tx: None,
|
||||
#[cfg(feature = "stealth")]
|
||||
stealth_client,
|
||||
}
|
||||
}
|
||||
|
||||
fn should_block_url(&self, url: &str) -> bool {
|
||||
if !self.intercept_enabled || self.intercept_block_patterns.is_empty() {
|
||||
return false;
|
||||
}
|
||||
for pattern in &self.intercept_block_patterns {
|
||||
if pattern == "*" { return true; }
|
||||
if pattern.starts_with('*') && pattern.ends_with('*') {
|
||||
if url.contains(&pattern[1..pattern.len()-1]) { return true; }
|
||||
} else if pattern.starts_with('*') {
|
||||
if url.ends_with(&pattern[1..]) { return true; }
|
||||
} else if pattern.ends_with('*') {
|
||||
if url.starts_with(&pattern[..pattern.len()-1]) { return true; }
|
||||
} else if url.contains(pattern) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
async fn do_fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
||||
#[cfg(feature = "stealth")]
|
||||
if let Some(ref stealth) = self.stealth_client {
|
||||
return stealth.fetch(url).await;
|
||||
}
|
||||
self.http_client.fetch(url).await
|
||||
}
|
||||
fn init_js(&mut self) {
|
||||
if self.js.is_some() {
|
||||
let url_str = self.url_string();
|
||||
let title = self.title.clone();
|
||||
let dom = self.dom.take();
|
||||
|
||||
let js = self.js.as_mut().unwrap();
|
||||
js.set_url(&url_str);
|
||||
js.set_title(&title);
|
||||
|
||||
if let Some(d) = dom {
|
||||
js.set_dom(d);
|
||||
}
|
||||
|
||||
let _ = js.execute_script("<reset>",
|
||||
"_cache.clear(); globalThis.__obscura_objects = {}; globalThis.__obscura_oid = 0; \
|
||||
_iframeRegistry.length = 0; globalThis.length = 0; \
|
||||
globalThis._formValues = {}; globalThis._formChecked = {}; \
|
||||
globalThis._eventRegistry = {}; \
|
||||
globalThis.document = new Document(+_dom('document_node_id'));");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
let mut rt = ObscuraJsRuntime::with_base_url(&self.url_string());
|
||||
rt.set_url(&self.url_string());
|
||||
rt.set_title(&self.title);
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
if self.stealth_client.is_some() {
|
||||
rt.set_user_agent(obscura_net::STEALTH_USER_AGENT);
|
||||
} else if let Ok(ua) = self.http_client.user_agent.try_read() {
|
||||
rt.set_user_agent(&ua);
|
||||
}
|
||||
#[cfg(not(feature = "stealth"))]
|
||||
if let Ok(ua) = self.http_client.user_agent.try_read() {
|
||||
rt.set_user_agent(&ua);
|
||||
}
|
||||
|
||||
rt.set_cookie_jar(self.context.cookie_jar.clone());
|
||||
rt.set_http_client(self.http_client.clone());
|
||||
|
||||
if let Some(tx) = &self.intercept_tx {
|
||||
rt.set_intercept_tx(tx.clone());
|
||||
}
|
||||
|
||||
if let Some(dom) = self.dom.take() {
|
||||
rt.set_dom(dom);
|
||||
}
|
||||
|
||||
self.js = Some(rt);
|
||||
}
|
||||
|
||||
async fn execute_scripts(&mut self) {
|
||||
tracing::info!("execute_scripts called, js runtime exists: {}", self.js.is_some());
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ScriptInfo {
|
||||
src: Option<String>,
|
||||
inline: String,
|
||||
is_defer: bool,
|
||||
is_async: bool,
|
||||
is_module: bool,
|
||||
}
|
||||
|
||||
let all_scripts = match &self.js {
|
||||
Some(js) => {
|
||||
js.with_dom(|dom| {
|
||||
let script_ids = dom.query_selector_all("script").unwrap_or_default();
|
||||
let mut scripts = Vec::new();
|
||||
|
||||
for sid in script_ids {
|
||||
if let Some(node) = dom.get_node(sid) {
|
||||
let src = node.get_attribute("src").map(|s| s.to_string());
|
||||
let script_type = node.get_attribute("type").unwrap_or("").to_string();
|
||||
let is_defer = node.get_attribute("defer").is_some();
|
||||
let is_async = node.get_attribute("async").is_some();
|
||||
let is_module = script_type == "module";
|
||||
|
||||
if !script_type.is_empty()
|
||||
&& script_type != "text/javascript"
|
||||
&& script_type != "application/javascript"
|
||||
&& script_type != "module"
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let inline_code = if src.is_none() {
|
||||
dom.text_content(sid)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
if src.is_some() || !inline_code.trim().is_empty() {
|
||||
scripts.push(ScriptInfo {
|
||||
src,
|
||||
inline: inline_code,
|
||||
is_defer,
|
||||
is_async,
|
||||
is_module,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
scripts
|
||||
}).unwrap_or_default()
|
||||
}
|
||||
None => return,
|
||||
};
|
||||
|
||||
let mut regular = Vec::new();
|
||||
let mut deferred = Vec::new();
|
||||
let mut async_scripts = Vec::new();
|
||||
|
||||
let mut module_scripts = Vec::new();
|
||||
|
||||
for script in all_scripts {
|
||||
if script.is_module {
|
||||
module_scripts.push(script);
|
||||
continue;
|
||||
}
|
||||
if script.is_defer {
|
||||
deferred.push(script);
|
||||
} else if script.is_async {
|
||||
async_scripts.push(script);
|
||||
} else {
|
||||
regular.push(script);
|
||||
}
|
||||
}
|
||||
|
||||
let scripts = regular;
|
||||
|
||||
tracing::info!("Found {} regular + {} deferred + {} async scripts", scripts.len(), deferred.len(), async_scripts.len());
|
||||
let all_to_execute: Vec<ScriptInfo> = scripts.into_iter()
|
||||
.chain(deferred.into_iter())
|
||||
.chain(async_scripts.into_iter())
|
||||
.collect();
|
||||
|
||||
let mut resolved: Vec<(usize, String)> = Vec::new();
|
||||
let mut fetch_tasks: Vec<(usize, String)> = Vec::new();
|
||||
|
||||
for (i, script) in all_to_execute.iter().enumerate() {
|
||||
if let Some(src_url) = &script.src {
|
||||
let full_url = if src_url.starts_with("http://") || src_url.starts_with("https://") {
|
||||
src_url.clone()
|
||||
} else if let Some(base) = &self.url {
|
||||
base.join(src_url).map(|u| u.to_string()).unwrap_or_else(|_| src_url.clone())
|
||||
} else {
|
||||
src_url.clone()
|
||||
};
|
||||
|
||||
if self.should_block_url(&full_url) {
|
||||
tracing::info!("Blocked script by interception: {}", full_url);
|
||||
continue;
|
||||
}
|
||||
resolved.push((i, full_url.clone()));
|
||||
fetch_tasks.push((i, full_url));
|
||||
}
|
||||
}
|
||||
|
||||
let client = self.http_client.clone();
|
||||
let fetch_futures: Vec<_> = fetch_tasks.iter().map(|(idx, url)| {
|
||||
let client = client.clone();
|
||||
let url = url.clone();
|
||||
let idx = *idx;
|
||||
async move {
|
||||
let parsed = Url::parse(&url).unwrap_or_else(|_| Url::parse("about:blank").unwrap());
|
||||
match client.fetch(&parsed).await {
|
||||
Ok(resp) => Some((idx, url, resp)),
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to fetch script {}: {}", url, e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}).collect();
|
||||
|
||||
let fetch_results = futures::future::join_all(fetch_futures).await;
|
||||
|
||||
let mut fetched: std::collections::HashMap<usize, (String, String, obscura_net::Response)> = std::collections::HashMap::new();
|
||||
for result in fetch_results {
|
||||
if let Some((idx, url, resp)) = result {
|
||||
let code = String::from_utf8_lossy(&resp.body).to_string();
|
||||
fetched.insert(idx, (url, code, resp));
|
||||
}
|
||||
}
|
||||
|
||||
for (i, script) in all_to_execute.iter().enumerate() {
|
||||
if script.src.is_some() {
|
||||
if let Some((url, code, resp)) = fetched.remove(&i) {
|
||||
tracing::info!("Executing script ({} bytes): {}", code.len(), url);
|
||||
self.record_network_event(&url, "GET", "Script", resp.status, &resp.headers, resp.body.len());
|
||||
if let Some(js) = &mut self.js {
|
||||
if let Err(e) = js.execute_script_guarded(&url, &code) {
|
||||
tracing::warn!("Script error ({}): {}", url, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if !script.inline.is_empty() {
|
||||
if let Some(js) = &mut self.js {
|
||||
if let Err(e) = js.execute_script_guarded("<inline>", &script.inline) {
|
||||
tracing::warn!("Inline script error: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for module_script in &module_scripts {
|
||||
if let Some(ref src) = module_script.src {
|
||||
let full_url = if src.starts_with("http://") || src.starts_with("https://") {
|
||||
src.clone()
|
||||
} else if let Some(base) = &self.url {
|
||||
base.join(src).map(|u| u.to_string()).unwrap_or_else(|_| src.clone())
|
||||
} else {
|
||||
src.clone()
|
||||
};
|
||||
|
||||
tracing::info!("Loading ES module: {}", full_url);
|
||||
if let Some(js) = &mut self.js {
|
||||
match js.load_module(&full_url).await {
|
||||
Ok(()) => {
|
||||
tracing::info!("ES module loaded: {}", full_url);
|
||||
self.record_network_event(&full_url, "GET", "Script", 200, &std::collections::HashMap::new(), 0);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("ES module error ({}): {}", full_url, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if !module_script.inline.is_empty() {
|
||||
let base = self.url_string();
|
||||
if let Some(js) = &mut self.js {
|
||||
if let Err(e) = js.load_inline_module(&module_script.inline, &base).await {
|
||||
tracing::warn!("Inline ES module error: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(js) = &mut self.js {
|
||||
let _ = js.execute_script("<load-events>",
|
||||
"if (typeof window.onload === 'function') { try { window.onload(); } catch(e) {} }\n\
|
||||
try { document.dispatchEvent(new Event('DOMContentLoaded')); } catch(e) {}\n\
|
||||
try { window.dispatchEvent(new Event('load')); } catch(e) {}");
|
||||
}
|
||||
|
||||
if let Some(js) = &mut self.js {
|
||||
let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_millis(500);
|
||||
let mut idle_count = 0u32;
|
||||
loop {
|
||||
let result = tokio::time::timeout(
|
||||
tokio::time::Duration::from_millis(10),
|
||||
js.run_event_loop(),
|
||||
).await;
|
||||
|
||||
match result {
|
||||
Ok(Ok(())) => {
|
||||
if self.http_client.active_requests() == 0 {
|
||||
idle_count += 1;
|
||||
if idle_count >= 2 {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
} else {
|
||||
idle_count = 0;
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(1)).await;
|
||||
}
|
||||
}
|
||||
Ok(Err(_)) => break,
|
||||
Err(_) => {
|
||||
idle_count = 0;
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn navigate(&mut self, url_str: &str) -> Result<(), PageError> {
|
||||
self.navigate_with_wait(url_str, crate::lifecycle::WaitUntil::Load).await
|
||||
}
|
||||
|
||||
pub async fn navigate_with_wait(
|
||||
&mut self,
|
||||
url_str: &str,
|
||||
wait_until: crate::lifecycle::WaitUntil,
|
||||
) -> Result<(), PageError> {
|
||||
self.navigate_with_wait_post(url_str, wait_until, "GET", "").await
|
||||
}
|
||||
|
||||
pub async fn navigate_with_wait_post(
|
||||
&mut self,
|
||||
url_str: &str,
|
||||
wait_until: crate::lifecycle::WaitUntil,
|
||||
method: &str,
|
||||
body: &str,
|
||||
) -> Result<(), PageError> {
|
||||
let mut current_url = url_str.to_string();
|
||||
let mut current_method = method.to_string();
|
||||
let mut current_body = body.to_string();
|
||||
for _chain in 0..10 {
|
||||
self.navigate_single(¤t_url, wait_until, ¤t_method, ¤t_body).await?;
|
||||
if let Some((next_url, next_method, next_body)) = self.take_pending_navigation() {
|
||||
tracing::info!("JS-triggered navigation chain: {} {} -> {}", current_method, current_url, next_url);
|
||||
current_url = next_url;
|
||||
current_method = next_method;
|
||||
current_body = next_body;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn navigate_single(
|
||||
&mut self,
|
||||
url_str: &str,
|
||||
wait_until: crate::lifecycle::WaitUntil,
|
||||
method: &str,
|
||||
body: &str,
|
||||
) -> Result<(), PageError> {
|
||||
let url = Url::parse(url_str).map_err(|e| PageError::InvalidUrl(e.to_string()))?;
|
||||
|
||||
self.lifecycle = LifecycleState::Loading;
|
||||
self.url = Some(url.clone());
|
||||
self.network_events.clear();
|
||||
|
||||
if self.context.obey_robots {
|
||||
if let Some(domain) = url.host_str() {
|
||||
if self.context.robots_cache.is_allowed(domain, "/robots.txt") {
|
||||
let robots_url = format!("{}://{}/robots.txt", url.scheme(), domain);
|
||||
if let Ok(robots_url) = Url::parse(&robots_url) {
|
||||
if let Ok(resp) = self.http_client.fetch(&robots_url).await {
|
||||
if resp.status == 200 {
|
||||
let body = String::from_utf8_lossy(&resp.body);
|
||||
self.context.robots_cache.parse_and_store(
|
||||
domain,
|
||||
&body,
|
||||
&self.context.user_agent,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !self.context.robots_cache.is_allowed(domain, url.path()) {
|
||||
self.lifecycle = LifecycleState::Failed;
|
||||
return Err(PageError::NetworkError(format!(
|
||||
"Blocked by robots.txt: {}",
|
||||
url
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let response = if method == "POST" {
|
||||
self.http_client.post_form(&url, body).await
|
||||
} else {
|
||||
self.do_fetch(&url).await
|
||||
}.map_err(|e| {
|
||||
self.lifecycle = LifecycleState::Failed;
|
||||
PageError::NetworkError(e.to_string())
|
||||
})?;
|
||||
|
||||
self.record_network_event(
|
||||
url.as_str(),
|
||||
"GET",
|
||||
"Document",
|
||||
response.status,
|
||||
&response.headers,
|
||||
response.body.len(),
|
||||
);
|
||||
|
||||
if !response.redirected_from.is_empty() {
|
||||
self.url = Some(response.url.clone());
|
||||
}
|
||||
|
||||
let body_text = String::from_utf8_lossy(&response.body).to_string();
|
||||
let dom = parse_html(&body_text);
|
||||
|
||||
self.title = dom
|
||||
.query_selector("title")
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|title_id| dom.text_content(title_id))
|
||||
.unwrap_or_default();
|
||||
|
||||
let stylesheet_urls: Vec<String> = dom
|
||||
.query_selector_all("link")
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.filter_map(|&nid| {
|
||||
let node = dom.get_node(nid)?;
|
||||
let rel = node.get_attribute("rel")?;
|
||||
if rel.to_lowercase() != "stylesheet" {
|
||||
return None;
|
||||
}
|
||||
node.get_attribute("href").map(|s| s.to_string())
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut css_fetch_urls: Vec<String> = Vec::new();
|
||||
for href in &stylesheet_urls {
|
||||
let full_url = if href.starts_with("http://") || href.starts_with("https://") {
|
||||
href.clone()
|
||||
} else if let Some(base) = &self.url {
|
||||
base.join(href).map(|u| u.to_string()).unwrap_or_else(|_| href.clone())
|
||||
} else {
|
||||
href.clone()
|
||||
};
|
||||
if self.should_block_url(&full_url) {
|
||||
tracing::info!("Blocked stylesheet by interception: {}", full_url);
|
||||
continue;
|
||||
}
|
||||
css_fetch_urls.push(full_url);
|
||||
}
|
||||
|
||||
let client = self.http_client.clone();
|
||||
let css_futures: Vec<_> = css_fetch_urls.iter().map(|full_url| {
|
||||
let client = client.clone();
|
||||
let url_str = full_url.clone();
|
||||
async move {
|
||||
let parsed = Url::parse(&url_str).unwrap_or_else(|_| Url::parse("about:blank").unwrap());
|
||||
match client.fetch(&parsed).await {
|
||||
Ok(resp) => Some((url_str, resp)),
|
||||
Err(e) => {
|
||||
tracing::debug!("Failed to fetch stylesheet {}: {}", url_str, e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}).collect();
|
||||
|
||||
let css_results = futures::future::join_all(css_futures).await;
|
||||
let mut css_sources = Vec::new();
|
||||
for result in css_results {
|
||||
if let Some((url_str, resp)) = result {
|
||||
let css = String::from_utf8_lossy(&resp.body).to_string();
|
||||
self.record_network_event(&url_str, "GET", "Stylesheet", resp.status, &resp.headers, resp.body.len());
|
||||
css_sources.push(css);
|
||||
}
|
||||
}
|
||||
|
||||
self.dom = Some(dom);
|
||||
self.lifecycle = LifecycleState::DomContentLoaded;
|
||||
|
||||
if wait_until == crate::lifecycle::WaitUntil::DomContentLoaded {
|
||||
self.init_js();
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.init_js();
|
||||
|
||||
if !css_sources.is_empty() {
|
||||
if let Some(js) = &mut self.js {
|
||||
let combined_css = css_sources.join("\n");
|
||||
let escaped = combined_css
|
||||
.replace('\\', "\\\\")
|
||||
.replace('`', "\\`")
|
||||
.replace("${", "\\${");
|
||||
let code = format!("globalThis.__obscura_css = `{}`;", escaped);
|
||||
let _ = js.execute_script("<css>", &code);
|
||||
}
|
||||
}
|
||||
if let Some(js) = &mut self.js {
|
||||
let _ = js.execute_script("<iframe-load>",
|
||||
"(function() { var iframes = document.querySelectorAll('iframe[src]'); for (var i = 0; i < iframes.length; i++) { var src = iframes[i].getAttribute('src'); if (src && src !== 'about:blank') iframes[i]._loadIframeSrc(src); } })()");
|
||||
}
|
||||
|
||||
self.execute_scripts().await;
|
||||
|
||||
if let Some(js) = &mut self.js {
|
||||
if let Ok(new_title) = js.evaluate("document.title") {
|
||||
if let Some(t) = new_title.as_str() {
|
||||
self.title = t.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.lifecycle = LifecycleState::Loaded;
|
||||
|
||||
if matches!(
|
||||
wait_until,
|
||||
crate::lifecycle::WaitUntil::NetworkIdle0 | crate::lifecycle::WaitUntil::NetworkIdle2
|
||||
) {
|
||||
let threshold = match wait_until {
|
||||
crate::lifecycle::WaitUntil::NetworkIdle0 => 0,
|
||||
crate::lifecycle::WaitUntil::NetworkIdle2 => 2,
|
||||
_ => 0,
|
||||
};
|
||||
|
||||
let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_secs(5);
|
||||
let mut idle_since: Option<tokio::time::Instant> = None;
|
||||
|
||||
loop {
|
||||
let active = self.http_client.active_requests();
|
||||
let now = tokio::time::Instant::now();
|
||||
|
||||
if active <= threshold {
|
||||
if idle_since.is_none() {
|
||||
idle_since = Some(now);
|
||||
}
|
||||
if now.duration_since(idle_since.unwrap()) >= tokio::time::Duration::from_millis(500) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
idle_since = None;
|
||||
}
|
||||
|
||||
if now >= deadline {
|
||||
tracing::debug!("Network idle timeout reached with {} active requests", active);
|
||||
break;
|
||||
}
|
||||
|
||||
if let Some(js) = &mut self.js {
|
||||
let _ = tokio::time::timeout(
|
||||
tokio::time::Duration::from_millis(50),
|
||||
js.run_event_loop(),
|
||||
).await;
|
||||
} else {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
self.lifecycle = LifecycleState::NetworkIdle;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn navigate_blank(&mut self) {
|
||||
self.js = None;
|
||||
self.url = Some(Url::parse("about:blank").unwrap());
|
||||
self.dom = Some(parse_html("<!DOCTYPE html><html><head></head><body></body></html>"));
|
||||
self.title = String::new();
|
||||
self.lifecycle = LifecycleState::Loaded;
|
||||
}
|
||||
|
||||
pub fn url_string(&self) -> String {
|
||||
self.url
|
||||
.as_ref()
|
||||
.map(|u| u.to_string())
|
||||
.unwrap_or_else(|| "about:blank".to_string())
|
||||
}
|
||||
|
||||
pub fn with_dom<R>(&self, f: impl FnOnce(&DomTree) -> R) -> Option<R> {
|
||||
if let Some(js) = &self.js {
|
||||
return js.with_dom(f);
|
||||
}
|
||||
self.dom.as_ref().map(f)
|
||||
}
|
||||
|
||||
pub fn dom(&self) -> Option<&DomTree> {
|
||||
self.dom.as_ref()
|
||||
}
|
||||
|
||||
pub fn evaluate(&mut self, expression: &str) -> serde_json::Value {
|
||||
if let Some(js) = &mut self.js {
|
||||
match js.evaluate(expression) {
|
||||
Ok(val) => val,
|
||||
Err(e) => {
|
||||
tracing::debug!("JS eval error for '{}': {}", &expression[..expression.len().min(80)], e);
|
||||
serde_json::Value::Null
|
||||
}
|
||||
}
|
||||
} else {
|
||||
match expression.trim() {
|
||||
"document.title" => serde_json::Value::String(self.title.clone()),
|
||||
"document.URL" | "document.location.href" | "window.location.href" => {
|
||||
serde_json::Value::String(self.url_string())
|
||||
}
|
||||
_ => serde_json::Value::Null,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn evaluate_for_cdp(
|
||||
&mut self,
|
||||
expression: &str,
|
||||
return_by_value: bool,
|
||||
) -> obscura_js::runtime::RemoteObjectInfo {
|
||||
if let Some(js) = &mut self.js {
|
||||
match js.evaluate_for_cdp(expression, return_by_value) {
|
||||
Ok(info) => info,
|
||||
Err(e) => {
|
||||
tracing::debug!("evaluate_for_cdp error: {}", e);
|
||||
obscura_js::runtime::RemoteObjectInfo {
|
||||
js_type: "undefined".into(),
|
||||
subtype: None,
|
||||
class_name: String::new(),
|
||||
description: String::new(),
|
||||
object_id: None,
|
||||
value: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let val = self.evaluate(expression);
|
||||
obscura_js::runtime::RemoteObjectInfo {
|
||||
js_type: match &val {
|
||||
serde_json::Value::String(_) => "string".into(),
|
||||
serde_json::Value::Number(_) => "number".into(),
|
||||
serde_json::Value::Bool(_) => "boolean".into(),
|
||||
_ => "undefined".into(),
|
||||
},
|
||||
subtype: None,
|
||||
class_name: String::new(),
|
||||
description: String::new(),
|
||||
object_id: None,
|
||||
value: Some(val),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn call_function_on_for_cdp(
|
||||
&mut self,
|
||||
function_declaration: &str,
|
||||
object_id: Option<&str>,
|
||||
args: &[serde_json::Value],
|
||||
return_by_value: bool,
|
||||
await_promise: bool,
|
||||
) -> obscura_js::runtime::RemoteObjectInfo {
|
||||
if let Some(js) = &mut self.js {
|
||||
match js.call_function_on_for_cdp(function_declaration, object_id, args, return_by_value, await_promise).await {
|
||||
Ok(info) => info,
|
||||
Err(e) => {
|
||||
tracing::debug!("callFunctionOn error: {}", e);
|
||||
obscura_js::runtime::RemoteObjectInfo {
|
||||
js_type: "undefined".into(),
|
||||
subtype: None,
|
||||
class_name: String::new(),
|
||||
description: String::new(),
|
||||
object_id: None,
|
||||
value: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
obscura_js::runtime::RemoteObjectInfo {
|
||||
js_type: "undefined".into(),
|
||||
subtype: None,
|
||||
class_name: String::new(),
|
||||
description: String::new(),
|
||||
object_id: None,
|
||||
value: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_blocked_urls(&mut self, patterns: Vec<String>) {
|
||||
if let Some(js) = &self.js {
|
||||
js.set_blocked_urls(patterns);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn release_object(&mut self, object_id: &str) {
|
||||
if let Some(js) = &mut self.js {
|
||||
js.release_object(object_id);
|
||||
}
|
||||
}
|
||||
|
||||
fn record_network_event(
|
||||
&mut self,
|
||||
url: &str,
|
||||
method: &str,
|
||||
resource_type: &str,
|
||||
status: u16,
|
||||
response_headers: &std::collections::HashMap<String, String>,
|
||||
body_size: usize,
|
||||
) {
|
||||
self.network_event_counter += 1;
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs_f64();
|
||||
self.network_events.push(NetworkEvent {
|
||||
request_id: format!("{}.{}", self.id, self.network_event_counter),
|
||||
url: url.to_string(),
|
||||
method: method.to_string(),
|
||||
resource_type: resource_type.to_string(),
|
||||
status,
|
||||
headers: std::collections::HashMap::new(),
|
||||
response_headers: Arc::new(response_headers.clone()),
|
||||
body_size,
|
||||
timestamp,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn execute_preload_script(&mut self, source: &str) -> Result<(), String> {
|
||||
if let Some(js) = &mut self.js {
|
||||
js.execute_script("<preload>", source)
|
||||
} else {
|
||||
Err("No JS runtime".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn suspend_js(&mut self) {
|
||||
if let Some(js) = &self.js {
|
||||
if let Some(dom) = js.take_dom() {
|
||||
self.dom = Some(dom);
|
||||
}
|
||||
}
|
||||
self.js = None;
|
||||
}
|
||||
|
||||
pub fn resume_js(&mut self) {
|
||||
if self.js.is_some() {
|
||||
return;
|
||||
}
|
||||
self.init_js();
|
||||
}
|
||||
|
||||
pub fn has_js(&self) -> bool {
|
||||
self.js.is_some()
|
||||
}
|
||||
|
||||
pub fn release_object_group(&mut self) {
|
||||
if let Some(js) = &mut self.js {
|
||||
js.release_object_group();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn take_pending_navigation(&self) -> Option<(String, String, String)> {
|
||||
if let Some(js) = &self.js {
|
||||
js.take_pending_navigation()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_intercept_tx(&mut self, tx: tokio::sync::mpsc::UnboundedSender<obscura_js::ops::InterceptedRequest>) {
|
||||
self.intercept_tx = Some(tx.clone());
|
||||
if let Some(js) = &self.js {
|
||||
js.set_intercept_tx(tx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum PageError {
|
||||
#[error("Invalid URL: {0}")]
|
||||
InvalidUrl(String),
|
||||
|
||||
#[error("Network error: {0}")]
|
||||
NetworkError(String),
|
||||
|
||||
#[error("Parse error: {0}")]
|
||||
ParseError(String),
|
||||
}
|
||||
|
||||
impl From<ObscuraNetError> for PageError {
|
||||
fn from(e: ObscuraNetError) -> Self {
|
||||
PageError::NetworkError(e.to_string())
|
||||
}
|
||||
}
|
||||
20
crates/obscura-cdp/Cargo.toml
Normal file
20
crates/obscura-cdp/Cargo.toml
Normal file
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "obscura-cdp"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
obscura-dom = { workspace = true }
|
||||
obscura-browser = { workspace = true }
|
||||
obscura-js = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tokio-tungstenite = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
url = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
futures-util = "0.3"
|
||||
obscura-net = { workspace = true }
|
||||
136
crates/obscura-cdp/src/dispatch.rs
Normal file
136
crates/obscura-cdp/src/dispatch.rs
Normal file
@@ -0,0 +1,136 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use obscura_browser::{BrowserContext, Page};
|
||||
use obscura_js::ops::{InterceptResolution, InterceptedRequest};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::domains;
|
||||
use crate::domains::fetch::FetchInterceptState;
|
||||
use crate::types::{CdpEvent, CdpRequest, CdpResponse};
|
||||
|
||||
pub struct CdpContext {
|
||||
pub pages: Vec<Page>,
|
||||
pub sessions: HashMap<String, String>, // session_id -> page_id
|
||||
pub pending_events: Vec<CdpEvent>,
|
||||
pub default_context: Arc<BrowserContext>,
|
||||
page_counter: u32,
|
||||
pub preload_scripts: Vec<(String, String)>, // (identifier, source)
|
||||
pub preload_counter: u32,
|
||||
pub fetch_intercept: FetchInterceptState,
|
||||
pub intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<InterceptedRequest>>,
|
||||
}
|
||||
|
||||
impl CdpContext {
|
||||
pub fn new() -> Self {
|
||||
Self::new_with_proxy(None)
|
||||
}
|
||||
|
||||
pub fn new_with_proxy(proxy: Option<String>) -> Self {
|
||||
let default_context = Arc::new(BrowserContext::with_proxy("default".to_string(), proxy));
|
||||
CdpContext {
|
||||
pages: Vec::new(),
|
||||
sessions: HashMap::new(),
|
||||
pending_events: Vec::new(),
|
||||
default_context,
|
||||
page_counter: 0,
|
||||
preload_scripts: Vec::new(),
|
||||
preload_counter: 0,
|
||||
fetch_intercept: FetchInterceptState::new(),
|
||||
intercept_tx: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create_page(&mut self) -> String {
|
||||
self.page_counter += 1;
|
||||
let page_id = format!("page-{}", self.page_counter);
|
||||
let mut page = Page::new(page_id.clone(), self.default_context.clone());
|
||||
page.navigate_blank();
|
||||
self.pages.push(page);
|
||||
page_id
|
||||
}
|
||||
|
||||
pub fn get_page(&self, id: &str) -> Option<&Page> {
|
||||
self.pages.iter().find(|p| p.id == id)
|
||||
}
|
||||
|
||||
pub fn get_page_mut(&mut self, id: &str) -> Option<&mut Page> {
|
||||
self.pages.iter_mut().find(|p| p.id == id)
|
||||
}
|
||||
|
||||
pub fn remove_page(&mut self, id: &str) {
|
||||
self.pages.retain(|p| p.id != id);
|
||||
self.sessions.retain(|_, v| v != id);
|
||||
}
|
||||
|
||||
pub fn get_session_page(&self, session_id: &Option<String>) -> Option<&Page> {
|
||||
let page_id = session_id
|
||||
.as_ref()
|
||||
.and_then(|sid| self.sessions.get(sid))?;
|
||||
self.get_page(page_id)
|
||||
}
|
||||
|
||||
pub fn get_session_page_mut(&mut self, session_id: &Option<String>) -> Option<&mut Page> {
|
||||
let page_id = session_id
|
||||
.as_ref()
|
||||
.and_then(|sid| self.sessions.get(sid))
|
||||
.cloned()?;
|
||||
|
||||
let target_has_js = self.pages.iter().any(|p| p.id == page_id && p.has_js());
|
||||
|
||||
if !target_has_js {
|
||||
for page in &mut self.pages {
|
||||
if page.id != page_id && page.has_js() {
|
||||
page.suspend_js();
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(target) = self.pages.iter_mut().find(|p| p.id == page_id) {
|
||||
target.resume_js();
|
||||
}
|
||||
}
|
||||
|
||||
self.get_page_mut(&page_id)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn dispatch(req: &CdpRequest, ctx: &mut CdpContext) -> CdpResponse {
|
||||
let (domain, method) = match req.method.split_once('.') {
|
||||
Some((d, m)) => (d, m),
|
||||
None => {
|
||||
return CdpResponse::error(
|
||||
req.id,
|
||||
-32601,
|
||||
format!("Invalid method format: {}", req.method),
|
||||
req.session_id.clone(),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
let result = match domain {
|
||||
"Target" => domains::target::handle(method, &req.params, ctx).await,
|
||||
"Browser" => domains::browser::handle(method, &req.params).await,
|
||||
"Page" => domains::page::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"DOM" => domains::dom::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Runtime" => domains::runtime::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Network" => domains::network::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Fetch" => domains::fetch::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Input" => domains::input::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Storage" => domains::storage::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"LP" => domains::lp::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Emulation" | "Log" | "Performance" | "Security" | "CSS"
|
||||
| "Accessibility" | "ServiceWorker" | "Inspector"
|
||||
| "Debugger" | "Profiler" | "HeapProfiler" | "Overlay" => {
|
||||
Ok(json!({}))
|
||||
}
|
||||
_ => Err(format!("Unknown domain: {}", domain)),
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(value) => CdpResponse::success(req.id, value, req.session_id.clone()),
|
||||
Err(msg) => {
|
||||
tracing::warn!("CDP error for {}: {}", req.method, msg);
|
||||
CdpResponse::error(req.id, -32601, msg, req.session_id.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
31
crates/obscura-cdp/src/domains/browser.rs
Normal file
31
crates/obscura-cdp/src/domains/browser.rs
Normal file
@@ -0,0 +1,31 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub async fn handle(method: &str, _params: &Value) -> Result<Value, String> {
|
||||
match method {
|
||||
"getVersion" => Ok(json!({
|
||||
"protocolVersion": "1.3",
|
||||
"product": "Obscura/0.1.0",
|
||||
"revision": "0",
|
||||
"userAgent": "Obscura/0.1.0 (Headless Browser)",
|
||||
"jsVersion": "N/A",
|
||||
})),
|
||||
"close" => {
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getWindowForTarget" => Ok(json!({
|
||||
"windowId": 1,
|
||||
"bounds": {
|
||||
"left": 0,
|
||||
"top": 0,
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"windowState": "normal",
|
||||
}
|
||||
})),
|
||||
"setDownloadBehavior" => Ok(json!({})),
|
||||
"getWindowBounds" => Ok(json!({
|
||||
"bounds": { "left": 0, "top": 0, "width": 1280, "height": 720, "windowState": "normal" }
|
||||
})),
|
||||
_ => Err(format!("Unknown Browser method: {}", method)),
|
||||
}
|
||||
}
|
||||
204
crates/obscura-cdp/src/domains/dom.rs
Normal file
204
crates/obscura-cdp/src/domains/dom.rs
Normal file
@@ -0,0 +1,204 @@
|
||||
use obscura_dom::{DomTree, NodeData, NodeId};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => Ok(json!({})),
|
||||
"getDocument" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let depth = params.get("depth").and_then(|v| v.as_i64()).unwrap_or(2);
|
||||
page.with_dom(|dom| {
|
||||
let node = serialize_node(dom, dom.document(), depth as u32, 0);
|
||||
json!({ "root": node })
|
||||
}).ok_or_else(|| "No DOM loaded".to_string())
|
||||
}
|
||||
"querySelector" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let selector = params.get("selector").and_then(|v| v.as_str()).ok_or("selector required")?;
|
||||
let result = page.with_dom(|dom| {
|
||||
dom.query_selector(selector).ok().flatten().map(|id| id.index()).unwrap_or(0)
|
||||
}).unwrap_or(0);
|
||||
Ok(json!({ "nodeId": result }))
|
||||
}
|
||||
"querySelectorAll" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let selector = params.get("selector").and_then(|v| v.as_str()).ok_or("selector required")?;
|
||||
let ids = page.with_dom(|dom| {
|
||||
dom.query_selector_all(selector).ok()
|
||||
.map(|ids| ids.iter().map(|id| id.index() as u64).collect::<Vec<_>>())
|
||||
.unwrap_or_default()
|
||||
}).unwrap_or_default();
|
||||
Ok(json!({ "nodeIds": ids }))
|
||||
}
|
||||
"getOuterHTML" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let node_id = params.get("nodeId").and_then(|v| v.as_u64())
|
||||
.or_else(|| params.get("backendNodeId").and_then(|v| v.as_u64()))
|
||||
.ok_or("nodeId required")?;
|
||||
let html = page.with_dom(|dom| {
|
||||
dom.outer_html(NodeId::new(node_id as u32))
|
||||
}).unwrap_or_default();
|
||||
Ok(json!({ "outerHTML": html }))
|
||||
}
|
||||
"describeNode" => {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
let depth = params.get("depth").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
|
||||
let node_id = if let Some(nid) = params.get("nodeId").and_then(|v| v.as_u64())
|
||||
.or_else(|| params.get("backendNodeId").and_then(|v| v.as_u64()))
|
||||
{
|
||||
nid
|
||||
} else if let Some(oid) = params.get("objectId").and_then(|v| v.as_str()) {
|
||||
let escaped_oid = oid.replace('\\', "\\\\").replace('\'', "\\'");
|
||||
let code = format!(
|
||||
"(function() {{ var o = globalThis.__obscura_objects['{}']; if (!o) return -1; return (typeof o._nid === 'number') ? o._nid : -1; }})()",
|
||||
escaped_oid
|
||||
);
|
||||
let result = page.evaluate(&code);
|
||||
result.as_f64().map(|n| n as u64).unwrap_or(0)
|
||||
} else {
|
||||
return Err("nodeId or objectId required".to_string());
|
||||
};
|
||||
|
||||
let node = page.with_dom(|dom| {
|
||||
serialize_node(dom, NodeId::new(node_id as u32), depth as u32, 0)
|
||||
}).unwrap_or(json!(null));
|
||||
Ok(json!({ "node": node }))
|
||||
}
|
||||
"resolveNode" => {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
let node_id = if let Some(nid) = params.get("nodeId").and_then(|v| v.as_u64())
|
||||
.or_else(|| params.get("backendNodeId").and_then(|v| v.as_u64()))
|
||||
{
|
||||
nid
|
||||
} else if let Some(oid) = params.get("objectId").and_then(|v| v.as_str()) {
|
||||
let code = format!(
|
||||
"(function() {{ var o = globalThis.__obscura_objects['{}']; return (o && typeof o._nid === 'number') ? o._nid : -1; }})()",
|
||||
oid
|
||||
);
|
||||
let result = page.evaluate(&code);
|
||||
result.as_f64().map(|n| n as u64).unwrap_or(0)
|
||||
} else {
|
||||
return Err("nodeId or objectId required".to_string());
|
||||
};
|
||||
|
||||
let js_code = format!(
|
||||
"(function() {{\
|
||||
var nid = {};\
|
||||
var node = null;\
|
||||
if (globalThis._cache && globalThis._cache.has(nid)) {{\
|
||||
node = globalThis._cache.get(nid);\
|
||||
}} else {{\
|
||||
var t = +Deno.core.ops.op_dom('node_type', String(nid), '');\
|
||||
if (t === 1) node = new Element(nid);\
|
||||
else if (t === 9) node = globalThis.document;\
|
||||
else node = new Node(nid);\
|
||||
if (globalThis._cache) globalThis._cache.set(nid, node);\
|
||||
}}\
|
||||
return node;\
|
||||
}})()",
|
||||
node_id,
|
||||
);
|
||||
|
||||
let info = if let Some(js) = &mut page.js {
|
||||
match js.store_object_with_meta(&js_code) {
|
||||
Ok(info) => info,
|
||||
Err(_) => {
|
||||
return Ok(json!({
|
||||
"object": {
|
||||
"type": "object",
|
||||
"subtype": "node",
|
||||
"className": "HTMLElement",
|
||||
"objectId": format!("node-{}", node_id),
|
||||
}
|
||||
}));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return Err("No JS runtime".to_string());
|
||||
};
|
||||
|
||||
Ok(json!({
|
||||
"object": {
|
||||
"type": "object",
|
||||
"subtype": "node",
|
||||
"className": if info.class_name.is_empty() { "HTMLElement".to_string() } else { info.class_name },
|
||||
"description": info.description,
|
||||
"objectId": info.object_id.unwrap_or_else(|| format!("node-{}", node_id)),
|
||||
}
|
||||
}))
|
||||
}
|
||||
"setAttributeValue" => Ok(json!({})),
|
||||
"removeNode" => Ok(json!({})),
|
||||
"getBoxModel" => {
|
||||
Ok(json!({
|
||||
"model": {
|
||||
"content": [8,8, 108,8, 108,28, 8,28],
|
||||
"padding": [8,8, 108,8, 108,28, 8,28],
|
||||
"border": [8,8, 108,8, 108,28, 8,28],
|
||||
"margin": [0,0, 116,0, 116,36, 0,36],
|
||||
"width": 100, "height": 20,
|
||||
}
|
||||
}))
|
||||
}
|
||||
_ => Err(format!("Unknown DOM method: {}", method)),
|
||||
}
|
||||
}
|
||||
|
||||
fn serialize_node(dom: &DomTree, node_id: NodeId, max_depth: u32, current_depth: u32) -> Value {
|
||||
let node = match dom.get_node(node_id) {
|
||||
Some(n) => n,
|
||||
None => return json!(null),
|
||||
};
|
||||
let children_ids = dom.children(node_id);
|
||||
let child_count = children_ids.len();
|
||||
let mut result = json!({ "nodeId": node_id.index(), "backendNodeId": node_id.index(), "childNodeCount": child_count });
|
||||
|
||||
match &node.data {
|
||||
NodeData::Document => {
|
||||
result["nodeType"] = json!(9); result["nodeName"] = json!("#document");
|
||||
result["localName"] = json!(""); result["nodeValue"] = json!("");
|
||||
result["documentURL"] = json!(""); result["baseURL"] = json!(""); result["xmlVersion"] = json!("");
|
||||
}
|
||||
NodeData::Doctype { name, public_id, system_id } => {
|
||||
result["nodeType"] = json!(10); result["nodeName"] = json!(name);
|
||||
result["localName"] = json!(""); result["nodeValue"] = json!("");
|
||||
result["publicId"] = json!(public_id); result["systemId"] = json!(system_id);
|
||||
}
|
||||
NodeData::Element { name, attrs, .. } => {
|
||||
result["nodeType"] = json!(1);
|
||||
result["nodeName"] = json!(name.local.as_ref().to_ascii_uppercase());
|
||||
result["localName"] = json!(name.local.as_ref());
|
||||
result["nodeValue"] = json!("");
|
||||
let cdp_attrs: Vec<String> = attrs.iter()
|
||||
.flat_map(|a| vec![a.name.local.to_string(), a.value.clone()]).collect();
|
||||
result["attributes"] = json!(cdp_attrs);
|
||||
}
|
||||
NodeData::Text { contents } => {
|
||||
result["nodeType"] = json!(3); result["nodeName"] = json!("#text");
|
||||
result["localName"] = json!(""); result["nodeValue"] = json!(contents);
|
||||
}
|
||||
NodeData::Comment { contents } => {
|
||||
result["nodeType"] = json!(8); result["nodeName"] = json!("#comment");
|
||||
result["localName"] = json!(""); result["nodeValue"] = json!(contents);
|
||||
}
|
||||
NodeData::ProcessingInstruction { target, data } => {
|
||||
result["nodeType"] = json!(7); result["nodeName"] = json!(target);
|
||||
result["localName"] = json!(""); result["nodeValue"] = json!(data);
|
||||
}
|
||||
}
|
||||
|
||||
if current_depth < max_depth && !children_ids.is_empty() {
|
||||
let children: Vec<Value> = children_ids.iter()
|
||||
.map(|&cid| serialize_node(dom, cid, max_depth, current_depth + 1)).collect();
|
||||
result["children"] = json!(children);
|
||||
}
|
||||
result
|
||||
}
|
||||
189
crates/obscura-cdp/src/domains/fetch.rs
Normal file
189
crates/obscura-cdp/src/domains/fetch.rs
Normal file
@@ -0,0 +1,189 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::{json, Value};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
use crate::types::CdpEvent;
|
||||
|
||||
pub struct PausedRequest {
|
||||
pub request_id: String,
|
||||
pub url: String,
|
||||
pub method: String,
|
||||
pub headers: HashMap<String, String>,
|
||||
pub resource_type: String,
|
||||
pub resolver: tokio::sync::oneshot::Sender<FetchResolution>,
|
||||
}
|
||||
|
||||
pub enum FetchResolution {
|
||||
Continue {
|
||||
url: Option<String>,
|
||||
method: Option<String>,
|
||||
headers: Option<HashMap<String, String>>,
|
||||
post_data: Option<String>,
|
||||
},
|
||||
Fulfill {
|
||||
status: u16,
|
||||
headers: Vec<(String, String)>,
|
||||
body: String,
|
||||
},
|
||||
Fail {
|
||||
reason: String,
|
||||
},
|
||||
}
|
||||
|
||||
pub struct FetchInterceptState {
|
||||
pub enabled: bool,
|
||||
pub patterns: Vec<String>,
|
||||
pub paused: HashMap<String, PausedRequest>,
|
||||
request_counter: u64,
|
||||
}
|
||||
|
||||
impl FetchInterceptState {
|
||||
pub fn new() -> Self {
|
||||
FetchInterceptState {
|
||||
enabled: false,
|
||||
patterns: Vec::new(),
|
||||
paused: HashMap::new(),
|
||||
request_counter: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn next_request_id(&mut self) -> String {
|
||||
self.request_counter += 1;
|
||||
format!("interception-{}", self.request_counter)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => {
|
||||
let patterns = params
|
||||
.get("patterns")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|arr| {
|
||||
arr.iter()
|
||||
.filter_map(|p| {
|
||||
p.get("urlPattern")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_else(|| vec!["*".to_string()]);
|
||||
|
||||
ctx.fetch_intercept.enabled = true;
|
||||
ctx.fetch_intercept.patterns = patterns.clone();
|
||||
let tx_clone = ctx.intercept_tx.clone();
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.intercept_enabled = false;
|
||||
page.intercept_block_patterns.clear();
|
||||
if let Some(tx) = tx_clone {
|
||||
page.set_intercept_tx(tx);
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!("Fetch interception enabled");
|
||||
Ok(json!({}))
|
||||
}
|
||||
"disable" => {
|
||||
ctx.fetch_intercept.enabled = false;
|
||||
ctx.fetch_intercept.patterns.clear();
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.intercept_enabled = false;
|
||||
page.intercept_block_patterns.clear();
|
||||
}
|
||||
let paused: Vec<_> = ctx.fetch_intercept.paused.drain().collect();
|
||||
for (_, req) in paused {
|
||||
let _ = req.resolver.send(FetchResolution::Continue {
|
||||
url: None,
|
||||
method: None,
|
||||
headers: None,
|
||||
post_data: None,
|
||||
});
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"continueRequest" => {
|
||||
let request_id = params
|
||||
.get("requestId")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("requestId required")?;
|
||||
|
||||
if let Some(paused) = ctx.fetch_intercept.paused.remove(request_id) {
|
||||
let _ = paused.resolver.send(FetchResolution::Continue {
|
||||
url: params.get("url").and_then(|v| v.as_str()).map(|s| s.to_string()),
|
||||
method: params.get("method").and_then(|v| v.as_str()).map(|s| s.to_string()),
|
||||
headers: None,
|
||||
post_data: params.get("postData").and_then(|v| v.as_str()).map(|s| s.to_string()),
|
||||
});
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"fulfillRequest" => {
|
||||
let request_id = params
|
||||
.get("requestId")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("requestId required")?;
|
||||
|
||||
let status = params
|
||||
.get("responseCode")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(200) as u16;
|
||||
let headers: HashMap<String, String> = params
|
||||
.get("responseHeaders")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|arr| {
|
||||
arr.iter()
|
||||
.filter_map(|h| {
|
||||
let name = h.get("name")?.as_str()?.to_string();
|
||||
let value = h.get("value")?.as_str()?.to_string();
|
||||
Some((name, value))
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let body = params
|
||||
.get("body")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if let Some(paused) = ctx.fetch_intercept.paused.remove(request_id) {
|
||||
let _ = paused.resolver.send(FetchResolution::Fulfill {
|
||||
status,
|
||||
headers: headers.into_iter().collect(),
|
||||
body,
|
||||
});
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"failRequest" => {
|
||||
let request_id = params
|
||||
.get("requestId")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("requestId required")?;
|
||||
|
||||
let reason = params
|
||||
.get("errorReason")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("Failed")
|
||||
.to_string();
|
||||
|
||||
if let Some(paused) = ctx.fetch_intercept.paused.remove(request_id) {
|
||||
let _ = paused.resolver.send(FetchResolution::Fail { reason });
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getResponseBody" => {
|
||||
Ok(json!({ "body": "", "base64Encoded": false }))
|
||||
}
|
||||
_ => Err(format!("Unknown Fetch method: {}", method)),
|
||||
}
|
||||
}
|
||||
155
crates/obscura-cdp/src/domains/input.rs
Normal file
155
crates/obscura-cdp/src/domains/input.rs
Normal file
@@ -0,0 +1,155 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"dispatchMouseEvent" => {
|
||||
let event_type = params.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let x = params.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let y = params.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let button = params.get("button").and_then(|v| v.as_str()).unwrap_or("left");
|
||||
let click_count = params.get("clickCount").and_then(|v| v.as_u64()).unwrap_or(1);
|
||||
|
||||
if event_type == "mousePressed" {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
let code = format!(
|
||||
"(function() {{\
|
||||
var target = globalThis.__obscura_click_target || document.activeElement || document.body;\
|
||||
if (!target) return;\
|
||||
var evt = new MouseEvent('mousedown', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}});\
|
||||
target.dispatchEvent(evt);\
|
||||
var click = new MouseEvent('click', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}});\
|
||||
var cancelled = !target.dispatchEvent(click);\
|
||||
if (!cancelled) {{\
|
||||
var link = target.closest ? target.closest('a[href]') : null;\
|
||||
if (!link && target.tagName === 'A' && target.getAttribute('href')) link = target;\
|
||||
if (link) {{\
|
||||
var href = link.getAttribute('href');\
|
||||
if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {{\
|
||||
location.assign(href);\
|
||||
}}\
|
||||
}}\
|
||||
}}\
|
||||
}})()",
|
||||
x = x, y = y,
|
||||
);
|
||||
page.evaluate(&code);
|
||||
}
|
||||
} else if event_type == "mouseReleased" {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
let code = format!(
|
||||
"(function() {{\
|
||||
var target = globalThis.__obscura_click_target || document.activeElement || document.body;\
|
||||
if (!target) return;\
|
||||
var evt = new MouseEvent('mouseup', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}});\
|
||||
target.dispatchEvent(evt);\
|
||||
}})()",
|
||||
x = x, y = y,
|
||||
);
|
||||
page.evaluate(&code);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(json!({}))
|
||||
}
|
||||
"dispatchKeyEvent" => {
|
||||
let event_type = params.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let key = params.get("key").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let code = params.get("code").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let text = params.get("text").and_then(|v| v.as_str()).unwrap_or("");
|
||||
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
match event_type {
|
||||
"keyDown" | "rawKeyDown" => {
|
||||
let js = format!(
|
||||
"(function() {{\
|
||||
var target = document.activeElement || document.body;\
|
||||
var evt = new KeyboardEvent('keydown', {{bubbles:true,cancelable:true,key:'{key}',code:'{code}'}});\
|
||||
target.dispatchEvent(evt);\
|
||||
}})()",
|
||||
key = key.replace('\'', "\\'"),
|
||||
code = code.replace('\'', "\\'"),
|
||||
);
|
||||
page.evaluate(&js);
|
||||
|
||||
if !text.is_empty() && text != "\r" && text != "\n" {
|
||||
let js = format!(
|
||||
"(function() {{\
|
||||
var target = document.activeElement;\
|
||||
if (target && (target.localName === 'input' || target.localName === 'textarea')) {{\
|
||||
target.value = (target.value || '') + '{text}';\
|
||||
target.dispatchEvent(new Event('input', {{bubbles:true}}));\
|
||||
}}\
|
||||
}})()",
|
||||
text = text.replace('\'', "\\'").replace('\\', "\\\\"),
|
||||
);
|
||||
page.evaluate(&js);
|
||||
}
|
||||
|
||||
if key == "Enter" {
|
||||
let js = "(function() {\
|
||||
var target = document.activeElement;\
|
||||
if (target) {\
|
||||
target.dispatchEvent(new KeyboardEvent('keypress', {bubbles:true,key:'Enter',code:'Enter'}));\
|
||||
var form = target.form || target.closest && target.closest('form');\
|
||||
if (form && typeof form.submit === 'function') form.submit();\
|
||||
}\
|
||||
})()";
|
||||
page.evaluate(js);
|
||||
}
|
||||
|
||||
if key == "Backspace" {
|
||||
let js = "(function() {\
|
||||
var target = document.activeElement;\
|
||||
if (target && (target.localName === 'input' || target.localName === 'textarea')) {\
|
||||
target.value = target.value.slice(0, -1);\
|
||||
target.dispatchEvent(new Event('input', {bubbles:true}));\
|
||||
}\
|
||||
})()";
|
||||
page.evaluate(js);
|
||||
}
|
||||
}
|
||||
"keyUp" => {
|
||||
let js = format!(
|
||||
"(function() {{\
|
||||
var target = document.activeElement || document.body;\
|
||||
var evt = new KeyboardEvent('keyup', {{bubbles:true,key:'{key}',code:'{code}'}});\
|
||||
target.dispatchEvent(evt);\
|
||||
}})()",
|
||||
key = key.replace('\'', "\\'"),
|
||||
code = code.replace('\'', "\\'"),
|
||||
);
|
||||
page.evaluate(&js);
|
||||
}
|
||||
"char" => {
|
||||
if !text.is_empty() {
|
||||
let js = format!(
|
||||
"(function() {{\
|
||||
var target = document.activeElement;\
|
||||
if (target && (target.localName === 'input' || target.localName === 'textarea')) {{\
|
||||
target.value = (target.value || '') + '{text}';\
|
||||
target.dispatchEvent(new Event('input', {{bubbles:true}}));\
|
||||
}}\
|
||||
}})()",
|
||||
text = text.replace('\'', "\\'").replace('\\', "\\\\"),
|
||||
);
|
||||
page.evaluate(&js);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(json!({}))
|
||||
}
|
||||
"dispatchTouchEvent" => Ok(json!({})),
|
||||
"setIgnoreInputEvents" => Ok(json!({})),
|
||||
_ => Err(format!("Unknown Input method: {}", method)),
|
||||
}
|
||||
}
|
||||
85
crates/obscura-cdp/src/domains/lp.rs
Normal file
85
crates/obscura-cdp/src/domains/lp.rs
Normal file
@@ -0,0 +1,85 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
_params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"getMarkdown" => {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
let code = r#"
|
||||
(function() {
|
||||
function toMd(el, depth) {
|
||||
if (!el) return '';
|
||||
var out = '';
|
||||
if (el.nodeType === 3) return el.textContent || '';
|
||||
if (el.nodeType !== 1) return '';
|
||||
var tag = (el.tagName || '').toLowerCase();
|
||||
var children = '';
|
||||
var cn = el.childNodes || [];
|
||||
for (var i = 0; i < cn.length; i++) children += toMd(cn[i], depth);
|
||||
children = children.replace(/\n{3,}/g, '\n\n');
|
||||
switch(tag) {
|
||||
case 'h1': return '\n# ' + children.trim() + '\n\n';
|
||||
case 'h2': return '\n## ' + children.trim() + '\n\n';
|
||||
case 'h3': return '\n### ' + children.trim() + '\n\n';
|
||||
case 'h4': return '\n#### ' + children.trim() + '\n\n';
|
||||
case 'h5': return '\n##### ' + children.trim() + '\n\n';
|
||||
case 'h6': return '\n###### ' + children.trim() + '\n\n';
|
||||
case 'p': return '\n' + children.trim() + '\n\n';
|
||||
case 'br': return '\n';
|
||||
case 'hr': return '\n---\n\n';
|
||||
case 'strong': case 'b': return '**' + children + '**';
|
||||
case 'em': case 'i': return '*' + children + '*';
|
||||
case 'code': return '`' + children + '`';
|
||||
case 'pre': return '\n```\n' + children + '\n```\n\n';
|
||||
case 'blockquote': return '\n> ' + children.trim().replace(/\n/g, '\n> ') + '\n\n';
|
||||
case 'a':
|
||||
var href = el.getAttribute('href') || '';
|
||||
if (href && children.trim()) return '[' + children.trim() + '](' + href + ')';
|
||||
return children;
|
||||
case 'img':
|
||||
var src = el.getAttribute('src') || '';
|
||||
var alt = el.getAttribute('alt') || '';
|
||||
return '';
|
||||
case 'ul': case 'ol':
|
||||
return '\n' + children + '\n';
|
||||
case 'li':
|
||||
var parent = el.parentNode;
|
||||
var isOrdered = parent && parent.tagName && parent.tagName.toLowerCase() === 'ol';
|
||||
var bullet = isOrdered ? '1. ' : '- ';
|
||||
return bullet + children.trim() + '\n';
|
||||
case 'table': return '\n' + children + '\n';
|
||||
case 'thead': case 'tbody': case 'tfoot': return children;
|
||||
case 'tr':
|
||||
var cells = [];
|
||||
var tds = el.childNodes || [];
|
||||
for (var j = 0; j < tds.length; j++) {
|
||||
if (tds[j].nodeType === 1) cells.push(toMd(tds[j], depth).trim());
|
||||
}
|
||||
return '| ' + cells.join(' | ') + ' |\n';
|
||||
case 'th': case 'td': return children;
|
||||
case 'script': case 'style': case 'noscript': case 'link': case 'meta': return '';
|
||||
case 'div': case 'section': case 'article': case 'main': case 'aside': case 'nav': case 'header': case 'footer':
|
||||
return '\n' + children;
|
||||
case 'span': return children;
|
||||
default: return children;
|
||||
}
|
||||
}
|
||||
var body = document.body || document.documentElement;
|
||||
var md = toMd(body, 0);
|
||||
md = md.replace(/\n{3,}/g, '\n\n').trim();
|
||||
return md;
|
||||
})()
|
||||
"#;
|
||||
let result = page.evaluate(code);
|
||||
let markdown = result.as_str().unwrap_or("").to_string();
|
||||
Ok(json!({ "markdown": markdown }))
|
||||
}
|
||||
_ => Err(format!("Unknown LP method: {}", method)),
|
||||
}
|
||||
}
|
||||
10
crates/obscura-cdp/src/domains/mod.rs
Normal file
10
crates/obscura-cdp/src/domains/mod.rs
Normal file
@@ -0,0 +1,10 @@
|
||||
pub mod target;
|
||||
pub mod browser;
|
||||
pub mod page;
|
||||
pub mod dom;
|
||||
pub mod runtime;
|
||||
pub mod network;
|
||||
pub mod fetch;
|
||||
pub mod input;
|
||||
pub mod storage;
|
||||
pub mod lp;
|
||||
81
crates/obscura-cdp/src/domains/network.rs
Normal file
81
crates/obscura-cdp/src/domains/network.rs
Normal file
@@ -0,0 +1,81 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => Ok(json!({})),
|
||||
"setExtraHTTPHeaders" => {
|
||||
let headers = params.get("headers").and_then(|v| v.as_object());
|
||||
if let Some(page) = ctx.get_session_page(session_id) {
|
||||
if let Some(headers) = headers {
|
||||
let header_map: std::collections::HashMap<String, String> = headers
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), v.as_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
page.http_client.set_extra_headers(header_map).await;
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"setUserAgentOverride" => {
|
||||
let ua = params.get("userAgent").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if let Some(page) = ctx.get_session_page(session_id) {
|
||||
page.http_client.set_user_agent(ua).await;
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getCookies" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let cookies = page.context.cookie_jar.get_all_cookies();
|
||||
let cdp_cookies: Vec<Value> = cookies.iter().map(|c| {
|
||||
json!({
|
||||
"name": c.name,
|
||||
"value": c.value,
|
||||
"domain": c.domain,
|
||||
"path": c.path,
|
||||
"expires": -1,
|
||||
"size": c.name.len() + c.value.len(),
|
||||
"httpOnly": c.http_only,
|
||||
"secure": c.secure,
|
||||
"session": true,
|
||||
"sameParty": false,
|
||||
"sourceScheme": "Secure",
|
||||
"sourcePort": 443,
|
||||
})
|
||||
}).collect();
|
||||
Ok(json!({ "cookies": cdp_cookies }))
|
||||
}
|
||||
"setCookies" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
if let Some(cookies) = params.get("cookies").and_then(|v| v.as_array()) {
|
||||
let cookie_infos: Vec<obscura_net::CookieInfo> = cookies.iter().filter_map(|c| {
|
||||
Some(obscura_net::CookieInfo {
|
||||
name: c.get("name")?.as_str()?.to_string(),
|
||||
value: c.get("value")?.as_str()?.to_string(),
|
||||
domain: c.get("domain").and_then(|v| v.as_str()).unwrap_or("").to_string(),
|
||||
path: c.get("path").and_then(|v| v.as_str()).unwrap_or("/").to_string(),
|
||||
secure: c.get("secure").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
http_only: c.get("httpOnly").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
})
|
||||
}).collect();
|
||||
page.context.cookie_jar.set_cookies_from_cdp(cookie_infos);
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"clearBrowserCookies" => {
|
||||
if let Some(page) = ctx.get_session_page(session_id) {
|
||||
page.context.cookie_jar.clear();
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"setCacheDisabled" => Ok(json!({})),
|
||||
"setRequestInterception" => Ok(json!({})),
|
||||
_ => Err(format!("Unknown Network method: {}", method)),
|
||||
}
|
||||
}
|
||||
221
crates/obscura-cdp/src/domains/page.rs
Normal file
221
crates/obscura-cdp/src/domains/page.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
use obscura_browser::lifecycle::WaitUntil;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
use crate::types::CdpEvent;
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => Ok(json!({})),
|
||||
"navigate" => {
|
||||
let url = params.get("url").and_then(|v| v.as_str())
|
||||
.ok_or("url required")?;
|
||||
|
||||
let wait_until = params.get("waitUntil")
|
||||
.and_then(|v| {
|
||||
if let Some(s) = v.as_str() {
|
||||
Some(WaitUntil::from_str(s))
|
||||
} else if let Some(arr) = v.as_array() {
|
||||
arr.iter()
|
||||
.filter_map(|item| item.as_str())
|
||||
.map(WaitUntil::from_str)
|
||||
.max_by_key(|w| match w {
|
||||
WaitUntil::DomContentLoaded => 0,
|
||||
WaitUntil::Load => 1,
|
||||
WaitUntil::NetworkIdle2 => 2,
|
||||
WaitUntil::NetworkIdle0 => 3,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or(WaitUntil::Load);
|
||||
|
||||
let preload_scripts: Vec<String> = ctx.preload_scripts.iter().map(|(_, s)| s.clone()).collect();
|
||||
|
||||
let (frame_id, loader_id, network_events, page_url, page_id, reached_network_idle) = {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page for session")?;
|
||||
let frame_id = page.frame_id.clone();
|
||||
let loader_id = format!("loader-{}", uuid::Uuid::new_v4());
|
||||
|
||||
let nav_method = params.get("__method").and_then(|v| v.as_str()).unwrap_or("GET");
|
||||
let nav_body = params.get("__body").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if nav_method == "POST" && !nav_body.is_empty() {
|
||||
page.navigate_with_wait_post(url, wait_until, nav_method, nav_body).await.map_err(|e| e.to_string())?;
|
||||
} else {
|
||||
page.navigate_with_wait(url, wait_until).await.map_err(|e| e.to_string())?;
|
||||
}
|
||||
|
||||
for source in &preload_scripts {
|
||||
if let Err(e) = page.execute_preload_script(source) {
|
||||
tracing::debug!("Preload script error: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
let reached_network_idle = page.lifecycle.is_network_idle();
|
||||
let network_events: Vec<_> = page.network_events.drain(..).collect();
|
||||
let page_url = page.url_string();
|
||||
let page_id = page.id.clone();
|
||||
(frame_id, loader_id, network_events, page_url, page_id, reached_network_idle)
|
||||
};
|
||||
|
||||
let es = session_id.clone();
|
||||
let ts = timestamp();
|
||||
|
||||
let phase1 = vec![
|
||||
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "init", "timestamp": ts}), session_id: es.clone() },
|
||||
CdpEvent { method: "Runtime.executionContextsCleared".into(), params: json!({}), session_id: es.clone() },
|
||||
CdpEvent { method: "Page.frameNavigated".into(), params: json!({"frame": {"id": frame_id, "loaderId": loader_id, "url": page_url, "domainAndRegistry": "", "securityOrigin": page_url, "mimeType": "text/html", "adFrameStatus": {"adFrameType": "none"}}, "type": "Navigation"}), session_id: es.clone() },
|
||||
CdpEvent { method: "Runtime.executionContextCreated".into(), params: json!({"context": {"id": 2, "origin": page_url, "name": "", "uniqueId": format!("ctx-nav-{}", page_id), "auxData": {"isDefault": true, "type": "default", "frameId": frame_id}}}), session_id: es.clone() },
|
||||
CdpEvent { method: "Runtime.executionContextCreated".into(), params: json!({"context": {"id": 100, "origin": page_url, "name": "__puppeteer_utility_world__24.40.0", "uniqueId": format!("ctx-isolated-nav-{}", page_id), "auxData": {"isDefault": false, "type": "isolated", "frameId": frame_id}}}), session_id: es.clone() },
|
||||
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "commit", "timestamp": ts}), session_id: es.clone() },
|
||||
];
|
||||
ctx.pending_events.extend(phase1);
|
||||
|
||||
if ctx.fetch_intercept.enabled {
|
||||
for net_event in &network_events {
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Fetch.requestPaused".into(),
|
||||
params: json!({
|
||||
"requestId": net_event.request_id,
|
||||
"request": {
|
||||
"url": net_event.url,
|
||||
"method": net_event.method,
|
||||
"headers": net_event.headers,
|
||||
},
|
||||
"frameId": frame_id,
|
||||
"resourceType": net_event.resource_type,
|
||||
"networkId": net_event.request_id,
|
||||
}),
|
||||
session_id: es.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for net_event in &network_events {
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Network.requestWillBeSent".into(),
|
||||
params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}),
|
||||
session_id: es.clone(),
|
||||
});
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Network.responseReceived".into(),
|
||||
params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "timestamp": net_event.timestamp, "type": net_event.resource_type, "response": {"url": net_event.url, "status": net_event.status, "statusText": "", "headers": &*net_event.response_headers, "mimeType": net_event.response_headers.get("content-type").cloned().unwrap_or_default()}, "frameId": frame_id}),
|
||||
session_id: es.clone(),
|
||||
});
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Network.loadingFinished".into(),
|
||||
params: json!({"requestId": net_event.request_id, "timestamp": net_event.timestamp, "encodedDataLength": net_event.body_size}),
|
||||
session_id: es.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut phase3 = vec![
|
||||
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "DOMContentLoaded", "timestamp": ts}), session_id: es.clone() },
|
||||
CdpEvent { method: "Page.domContentEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() },
|
||||
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "load", "timestamp": ts}), session_id: es.clone() },
|
||||
CdpEvent { method: "Page.loadEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() },
|
||||
];
|
||||
if reached_network_idle || matches!(wait_until, WaitUntil::Load | WaitUntil::DomContentLoaded) {
|
||||
let idle_ts = timestamp();
|
||||
phase3.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "networkIdle", "timestamp": idle_ts}), session_id: es.clone() });
|
||||
}
|
||||
phase3.push(CdpEvent { method: "Page.frameStoppedLoading".into(), params: json!({"frameId": frame_id}), session_id: es });
|
||||
ctx.pending_events.extend(phase3);
|
||||
|
||||
Ok(json!({
|
||||
"frameId": frame_id,
|
||||
"loaderId": loader_id,
|
||||
}))
|
||||
}
|
||||
"getFrameTree" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page for session")?;
|
||||
Ok(json!({
|
||||
"frameTree": {
|
||||
"frame": {
|
||||
"id": page.frame_id,
|
||||
"loaderId": "initial-loader",
|
||||
"url": page.url_string(),
|
||||
"domainAndRegistry": "",
|
||||
"securityOrigin": page.url_string(),
|
||||
"mimeType": "text/html",
|
||||
"adFrameStatus": { "adFrameType": "none" },
|
||||
},
|
||||
"childFrames": [],
|
||||
}
|
||||
}))
|
||||
}
|
||||
"createIsolatedWorld" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page for session")?;
|
||||
let frame_id_param = params.get("frameId").and_then(|v| v.as_str())
|
||||
.unwrap_or(&page.frame_id).to_string();
|
||||
let world_name = params.get("worldName").and_then(|v| v.as_str())
|
||||
.unwrap_or("").to_string();
|
||||
let page_url = page.url_string();
|
||||
let page_id = page.id.clone();
|
||||
let context_id = 100;
|
||||
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Runtime.executionContextCreated".to_string(),
|
||||
params: json!({
|
||||
"context": {
|
||||
"id": context_id,
|
||||
"origin": page_url,
|
||||
"name": world_name,
|
||||
"uniqueId": format!("ctx-isolated-{}", page_id),
|
||||
"auxData": {
|
||||
"isDefault": false,
|
||||
"type": "isolated",
|
||||
"frameId": frame_id_param,
|
||||
}
|
||||
}
|
||||
}),
|
||||
session_id: session_id.clone(),
|
||||
});
|
||||
|
||||
Ok(json!({ "executionContextId": context_id }))
|
||||
}
|
||||
"setLifecycleEventsEnabled" => Ok(json!({})),
|
||||
"addScriptToEvaluateOnNewDocument" => {
|
||||
let source = params.get("source").and_then(|v| v.as_str()).unwrap_or("");
|
||||
ctx.preload_counter += 1;
|
||||
let identifier = format!("{}", ctx.preload_counter);
|
||||
if !source.is_empty() {
|
||||
ctx.preload_scripts.push((identifier.clone(), source.to_string()));
|
||||
}
|
||||
Ok(json!({ "identifier": identifier }))
|
||||
}
|
||||
"removeScriptToEvaluateOnNewDocument" => {
|
||||
let identifier = params.get("identifier").and_then(|v| v.as_str()).unwrap_or("");
|
||||
ctx.preload_scripts.retain(|(id, _)| id != identifier);
|
||||
Ok(json!({}))
|
||||
}
|
||||
"setInterceptFileChooserDialog" => Ok(json!({})),
|
||||
"getNavigationHistory" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page for session")?;
|
||||
Ok(json!({
|
||||
"currentIndex": 0,
|
||||
"entries": [{
|
||||
"id": 0,
|
||||
"url": page.url_string(),
|
||||
"userTypedURL": page.url_string(),
|
||||
"title": page.title,
|
||||
"transitionType": "typed",
|
||||
}]
|
||||
}))
|
||||
}
|
||||
_ => Err(format!("Unknown Page method: {}", method)),
|
||||
}
|
||||
}
|
||||
|
||||
fn timestamp() -> f64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs_f64()
|
||||
}
|
||||
213
crates/obscura-cdp/src/domains/runtime.rs
Normal file
213
crates/obscura-cdp/src/domains/runtime.rs
Normal file
@@ -0,0 +1,213 @@
|
||||
use obscura_js::runtime::RemoteObjectInfo;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let event = crate::types::CdpEvent {
|
||||
method: "Runtime.executionContextCreated".to_string(),
|
||||
params: json!({
|
||||
"context": {
|
||||
"id": 1,
|
||||
"origin": page.url_string(),
|
||||
"name": "",
|
||||
"uniqueId": format!("ctx-{}", page.id),
|
||||
"auxData": {
|
||||
"isDefault": true,
|
||||
"type": "default",
|
||||
"frameId": page.frame_id,
|
||||
}
|
||||
}
|
||||
}),
|
||||
session_id: session_id.clone(),
|
||||
};
|
||||
ctx.pending_events.push(event);
|
||||
Ok(json!({}))
|
||||
}
|
||||
"evaluate" => {
|
||||
let expression = params
|
||||
.get("expression")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("expression required")?;
|
||||
let return_by_value = params
|
||||
.get("returnByValue")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
let page = ctx
|
||||
.get_session_page_mut(session_id)
|
||||
.ok_or("No page")?;
|
||||
let info = page.evaluate_for_cdp(expression, return_by_value);
|
||||
|
||||
Ok(json!({ "result": remote_object_from_info(&info) }))
|
||||
}
|
||||
"callFunctionOn" => {
|
||||
let function_declaration = params
|
||||
.get("functionDeclaration")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("() => undefined");
|
||||
let return_by_value = params
|
||||
.get("returnByValue")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
let await_promise = params
|
||||
.get("awaitPromise")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
let object_id = params.get("objectId").and_then(|v| v.as_str());
|
||||
let arguments = params
|
||||
.get("arguments")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|a| a.to_vec())
|
||||
.unwrap_or_default();
|
||||
|
||||
let page = ctx
|
||||
.get_session_page_mut(session_id)
|
||||
.ok_or("No page")?;
|
||||
let info =
|
||||
page.call_function_on_for_cdp(function_declaration, object_id, &arguments, return_by_value, await_promise).await;
|
||||
|
||||
Ok(json!({ "result": remote_object_from_info(&info) }))
|
||||
}
|
||||
"getProperties" => {
|
||||
let object_id = params.get("objectId").and_then(|v| v.as_str());
|
||||
if let Some(oid) = object_id {
|
||||
let page = ctx
|
||||
.get_session_page_mut(session_id)
|
||||
.ok_or("No page")?;
|
||||
let escaped_oid = oid.replace('\\', "\\\\").replace('\'', "\\'");
|
||||
let code = format!(
|
||||
"(function() {{\
|
||||
var obj = globalThis.__obscura_objects['{oid}'];\
|
||||
if (!obj || typeof obj !== 'object') return [];\
|
||||
return Object.keys(obj).map(function(k) {{\
|
||||
var v = obj[k];\
|
||||
return {{ name: k, value: v, type: typeof v }};\
|
||||
}});\
|
||||
}})()",
|
||||
oid = escaped_oid,
|
||||
);
|
||||
let result = page.evaluate(&code);
|
||||
if let serde_json::Value::Array(props) = result {
|
||||
let descriptors: Vec<Value> = props
|
||||
.iter()
|
||||
.map(|p| {
|
||||
let name = p.get("name").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let value = p.get("value").unwrap_or(&Value::Null);
|
||||
let prop_type =
|
||||
p.get("type").and_then(|v| v.as_str()).unwrap_or("undefined");
|
||||
let mut remote = json!({
|
||||
"type": prop_type,
|
||||
});
|
||||
match value {
|
||||
Value::Null => {
|
||||
remote["type"] = json!("object");
|
||||
remote["subtype"] = json!("null");
|
||||
remote["value"] = json!(null);
|
||||
}
|
||||
Value::String(s) => {
|
||||
remote["type"] = json!("string");
|
||||
remote["value"] = json!(s);
|
||||
}
|
||||
Value::Number(n) => {
|
||||
remote["type"] = json!("number");
|
||||
remote["value"] = json!(n);
|
||||
}
|
||||
Value::Bool(b) => {
|
||||
remote["type"] = json!("boolean");
|
||||
remote["value"] = json!(b);
|
||||
}
|
||||
_ => {
|
||||
remote["value"] = value.clone();
|
||||
}
|
||||
}
|
||||
json!({
|
||||
"name": name,
|
||||
"value": remote,
|
||||
"configurable": true,
|
||||
"enumerable": true,
|
||||
"writable": true,
|
||||
"isOwn": true,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Ok(json!({ "result": descriptors, "internalProperties": [] }))
|
||||
} else {
|
||||
Ok(json!({ "result": [], "internalProperties": [] }))
|
||||
}
|
||||
} else {
|
||||
Ok(json!({ "result": [], "internalProperties": [] }))
|
||||
}
|
||||
}
|
||||
"releaseObject" => {
|
||||
if let Some(oid) = params.get("objectId").and_then(|v| v.as_str()) {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.release_object(oid);
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"releaseObjectGroup" => {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.release_object_group();
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"addBinding" => {
|
||||
let name = params.get("name").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if !name.is_empty() {
|
||||
if name.chars().all(|c| c.is_alphanumeric() || c == '_' || c == '$')
|
||||
&& !name.chars().next().unwrap_or('0').is_ascii_digit() {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
let code = format!(
|
||||
"if (typeof globalThis.{name} === 'undefined') {{\
|
||||
globalThis.{name} = function() {{ return null; }};\
|
||||
}}",
|
||||
name = name,
|
||||
);
|
||||
page.evaluate(&code);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"runIfWaitingForDebugger" => Ok(json!({})),
|
||||
"getExceptionDetails" => Ok(json!({ "exceptionDetails": null })),
|
||||
"discardConsoleEntries" => Ok(json!({})),
|
||||
_ => Err(format!("Unknown Runtime method: {}", method)),
|
||||
}
|
||||
}
|
||||
|
||||
fn remote_object_from_info(info: &RemoteObjectInfo) -> Value {
|
||||
let mut obj = json!({ "type": info.js_type });
|
||||
|
||||
if let Some(ref subtype) = info.subtype {
|
||||
obj["subtype"] = json!(subtype);
|
||||
}
|
||||
|
||||
if !info.class_name.is_empty() {
|
||||
obj["className"] = json!(info.class_name);
|
||||
}
|
||||
|
||||
if !info.description.is_empty() {
|
||||
obj["description"] = json!(info.description);
|
||||
}
|
||||
|
||||
if let Some(ref oid) = info.object_id {
|
||||
obj["objectId"] = json!(oid);
|
||||
}
|
||||
|
||||
if let Some(ref value) = info.value {
|
||||
obj["value"] = value.clone();
|
||||
}
|
||||
|
||||
obj
|
||||
}
|
||||
117
crates/obscura-cdp/src/domains/storage.rs
Normal file
117
crates/obscura-cdp/src/domains/storage.rs
Normal file
@@ -0,0 +1,117 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
_session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"getCookies" => {
|
||||
let cookies = ctx.default_context.cookie_jar.get_all_cookies();
|
||||
let cdp_cookies: Vec<Value> = cookies
|
||||
.iter()
|
||||
.map(|c| {
|
||||
json!({
|
||||
"name": c.name,
|
||||
"value": c.value,
|
||||
"domain": c.domain,
|
||||
"path": c.path,
|
||||
"expires": -1,
|
||||
"size": c.name.len() + c.value.len(),
|
||||
"httpOnly": c.http_only,
|
||||
"secure": c.secure,
|
||||
"session": true,
|
||||
"priority": "Medium",
|
||||
"sameParty": false,
|
||||
"sourceScheme": if c.secure { "Secure" } else { "NonSecure" },
|
||||
"sourcePort": 80,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Ok(json!({ "cookies": cdp_cookies }))
|
||||
}
|
||||
"setCookies" => {
|
||||
if let Some(cookies) = params.get("cookies").and_then(|v| v.as_array()) {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs_f64();
|
||||
|
||||
for c in cookies {
|
||||
let name = match c.get("name").and_then(|v| v.as_str()) {
|
||||
Some(n) => n.to_string(),
|
||||
None => continue,
|
||||
};
|
||||
let value = c.get("value").and_then(|v| v.as_str()).unwrap_or("").to_string();
|
||||
|
||||
let domain = c
|
||||
.get("domain")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.or_else(|| {
|
||||
c.get("url")
|
||||
.and_then(|v| v.as_str())
|
||||
.and_then(|u| url::Url::parse(u).ok())
|
||||
.and_then(|u| u.host_str().map(|h| h.to_string()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let expires = c.get("expires").and_then(|v| v.as_f64());
|
||||
if let Some(exp) = expires {
|
||||
if exp > 0.0 && exp < now {
|
||||
ctx.default_context.cookie_jar.delete_cookie(&name, &domain);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let path = c
|
||||
.get("path")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("/")
|
||||
.to_string();
|
||||
let secure =
|
||||
c.get("secure").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||
let http_only =
|
||||
c.get("httpOnly").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||
|
||||
ctx.default_context.cookie_jar.set_cookies_from_cdp(vec![
|
||||
obscura_net::CookieInfo {
|
||||
name,
|
||||
value,
|
||||
domain,
|
||||
path,
|
||||
secure,
|
||||
http_only,
|
||||
},
|
||||
]);
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"deleteCookies" => {
|
||||
let name = params.get("name").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let domain_from_url = params
|
||||
.get("url")
|
||||
.and_then(|v| v.as_str())
|
||||
.and_then(|u| url::Url::parse(u).ok())
|
||||
.and_then(|u| u.host_str().map(|h| h.to_string()));
|
||||
let domain = params
|
||||
.get("domain")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.or(domain_from_url)
|
||||
.unwrap_or_default();
|
||||
|
||||
if !name.is_empty() {
|
||||
ctx.default_context
|
||||
.cookie_jar
|
||||
.delete_cookie(name, &domain);
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
_ => Ok(json!({})),
|
||||
}
|
||||
}
|
||||
196
crates/obscura-cdp/src/domains/target.rs
Normal file
196
crates/obscura-cdp/src/domains/target.rs
Normal file
@@ -0,0 +1,196 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
use crate::types::CdpEvent;
|
||||
|
||||
pub async fn handle(method: &str, params: &Value, ctx: &mut CdpContext) -> Result<Value, String> {
|
||||
match method {
|
||||
"setDiscoverTargets" => {
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.targetCreated",
|
||||
json!({
|
||||
"targetInfo": {
|
||||
"targetId": "browser",
|
||||
"type": "browser",
|
||||
"title": "",
|
||||
"url": "",
|
||||
"attached": true,
|
||||
"browserContextId": "",
|
||||
}
|
||||
}),
|
||||
));
|
||||
for page in &ctx.pages {
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.targetCreated",
|
||||
json!({
|
||||
"targetInfo": {
|
||||
"targetId": page.id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": false,
|
||||
"browserContextId": page.context.id,
|
||||
}
|
||||
}),
|
||||
));
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getTargets" => {
|
||||
let targets: Vec<Value> = ctx
|
||||
.pages
|
||||
.iter()
|
||||
.map(|page| {
|
||||
json!({
|
||||
"targetId": page.id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": true,
|
||||
"browserContextId": page.context.id,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Ok(json!({ "targetInfos": targets }))
|
||||
}
|
||||
"createTarget" => {
|
||||
let url = params.get("url").and_then(|v| v.as_str()).unwrap_or("about:blank");
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = format!("{}-session", page_id);
|
||||
|
||||
if let Some(page) = ctx.get_page_mut(&page_id) {
|
||||
if url == "about:blank" || url.is_empty() {
|
||||
page.navigate_blank();
|
||||
} else {
|
||||
let _ = page.navigate(url).await;
|
||||
}
|
||||
}
|
||||
|
||||
ctx.sessions.insert(session_id.clone(), page_id.clone());
|
||||
|
||||
if let Some(page) = ctx.get_page(&page_id) {
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.targetCreated",
|
||||
json!({
|
||||
"targetInfo": {
|
||||
"targetId": page_id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": false,
|
||||
"browserContextId": page.context.id,
|
||||
}
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(page) = ctx.get_page(&page_id) {
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.attachedToTarget",
|
||||
json!({
|
||||
"sessionId": session_id,
|
||||
"targetInfo": {
|
||||
"targetId": page_id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": true,
|
||||
"browserContextId": page.context.id,
|
||||
},
|
||||
"waitingForDebugger": false,
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(json!({ "targetId": page_id }))
|
||||
}
|
||||
"attachToTarget" => {
|
||||
let target_id = params.get("targetId").and_then(|v| v.as_str())
|
||||
.ok_or("targetId required")?;
|
||||
let session_id = format!("{}-session", target_id);
|
||||
ctx.sessions.insert(session_id.clone(), target_id.to_string());
|
||||
|
||||
if let Some(page) = ctx.get_page(target_id) {
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.attachedToTarget",
|
||||
json!({
|
||||
"sessionId": session_id,
|
||||
"targetInfo": {
|
||||
"targetId": target_id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": true,
|
||||
"browserContextId": page.context.id,
|
||||
},
|
||||
"waitingForDebugger": false,
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(json!({ "sessionId": session_id }))
|
||||
}
|
||||
"closeTarget" => {
|
||||
let target_id = params.get("targetId").and_then(|v| v.as_str())
|
||||
.ok_or("targetId required")?;
|
||||
let session_id = format!("{}-session", target_id);
|
||||
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.detachedFromTarget",
|
||||
json!({
|
||||
"sessionId": session_id,
|
||||
"targetId": target_id,
|
||||
}),
|
||||
));
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.targetDestroyed",
|
||||
json!({ "targetId": target_id }),
|
||||
));
|
||||
|
||||
ctx.remove_page(target_id);
|
||||
Ok(json!({ "success": true }))
|
||||
}
|
||||
"setAutoAttach" => Ok(json!({})),
|
||||
"getBrowserContexts" => {
|
||||
Ok(json!({ "browserContextIds": [ctx.default_context.id] }))
|
||||
}
|
||||
"createBrowserContext" => {
|
||||
ctx.default_context.cookie_jar.clear();
|
||||
Ok(json!({ "browserContextId": ctx.default_context.id }))
|
||||
}
|
||||
"disposeBrowserContext" => {
|
||||
ctx.default_context.cookie_jar.clear();
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getTargetInfo" => {
|
||||
let target_id = params.get("targetId").and_then(|v| v.as_str());
|
||||
match target_id {
|
||||
Some(id) => {
|
||||
let page = ctx.get_page(id).ok_or("Target not found")?;
|
||||
Ok(json!({
|
||||
"targetInfo": {
|
||||
"targetId": id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": true,
|
||||
"browserContextId": page.context.id,
|
||||
}
|
||||
}))
|
||||
}
|
||||
None => {
|
||||
Ok(json!({
|
||||
"targetInfo": {
|
||||
"targetId": "browser",
|
||||
"type": "browser",
|
||||
"title": "",
|
||||
"url": "",
|
||||
"attached": true,
|
||||
}
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => Err(format!("Unknown Target method: {}", method)),
|
||||
}
|
||||
}
|
||||
6
crates/obscura-cdp/src/lib.rs
Normal file
6
crates/obscura-cdp/src/lib.rs
Normal file
@@ -0,0 +1,6 @@
|
||||
pub mod server;
|
||||
pub mod dispatch;
|
||||
pub mod types;
|
||||
pub mod domains;
|
||||
|
||||
pub use server::{start, start_with_options};
|
||||
629
crates/obscura-cdp/src/server.rs
Normal file
629
crates/obscura-cdp/src/server.rs
Normal file
@@ -0,0 +1,629 @@
|
||||
use std::collections::HashMap;
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use serde_json::json;
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use crate::dispatch::{self, CdpContext};
|
||||
use crate::types::CdpRequest;
|
||||
|
||||
struct CdpMessage {
|
||||
text: String,
|
||||
reply_tx: mpsc::UnboundedSender<String>,
|
||||
}
|
||||
|
||||
enum ServerMessage {
|
||||
Cdp(CdpMessage),
|
||||
NewConnection {
|
||||
reply_tx: mpsc::UnboundedSender<String>,
|
||||
},
|
||||
}
|
||||
|
||||
pub async fn start(port: u16) -> anyhow::Result<()> {
|
||||
start_with_options(port, None).await
|
||||
}
|
||||
|
||||
pub async fn start_with_options(port: u16, proxy: Option<String>) -> anyhow::Result<()> {
|
||||
let addr = SocketAddr::from(([127, 0, 0, 1], port));
|
||||
let listener = TcpListener::bind(&addr).await?;
|
||||
|
||||
info!("Obscura CDP server listening on ws://127.0.0.1:{}", port);
|
||||
info!(
|
||||
"DevTools endpoint: ws://127.0.0.1:{}/devtools/browser",
|
||||
port
|
||||
);
|
||||
|
||||
let local = tokio::task::LocalSet::new();
|
||||
local
|
||||
.run_until(async {
|
||||
let (msg_tx, msg_rx) = mpsc::unbounded_channel::<ServerMessage>();
|
||||
|
||||
let processor_handle = tokio::task::spawn_local(cdp_processor(msg_rx, proxy));
|
||||
|
||||
loop {
|
||||
match listener.accept().await {
|
||||
Ok((stream, peer_addr)) => {
|
||||
info!("New connection from {}", peer_addr);
|
||||
let tx = msg_tx.clone();
|
||||
tokio::task::spawn_local(async move {
|
||||
if let Err(e) = handle_connection(stream, port, tx).await {
|
||||
if !format!("{}", e).contains("close") {
|
||||
error!("Connection error from {}: {}", peer_addr, e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
Err(e) => error!("Accept error: {}", e),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn cdp_processor(
|
||||
mut rx: mpsc::UnboundedReceiver<ServerMessage>,
|
||||
proxy: Option<String>,
|
||||
) {
|
||||
let mut ctx = CdpContext::new_with_proxy(proxy);
|
||||
let (itx, irx) = mpsc::unbounded_channel::<obscura_js::ops::InterceptedRequest>();
|
||||
ctx.intercept_tx = Some(itx);
|
||||
let mut intercept_rx: Option<mpsc::UnboundedReceiver<obscura_js::ops::InterceptedRequest>> = Some(irx);
|
||||
let mut intercepted_paused: HashMap<String, tokio::sync::oneshot::Sender<obscura_js::ops::InterceptResolution>> = HashMap::new();
|
||||
|
||||
while let Some(msg) = rx.recv().await {
|
||||
match msg {
|
||||
ServerMessage::NewConnection { reply_tx } => {
|
||||
let _ = reply_tx.send(
|
||||
json!({"__init": true})
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
ServerMessage::Cdp(cdp_msg) => {
|
||||
let is_navigation = cdp_msg.text.contains("Page.navigate");
|
||||
let has_interception = ctx.fetch_intercept.enabled;
|
||||
|
||||
if is_navigation && has_interception {
|
||||
process_with_interception(
|
||||
&cdp_msg.text, &mut ctx, &cdp_msg.reply_tx, &mut rx,
|
||||
&mut intercept_rx, &mut intercepted_paused,
|
||||
).await;
|
||||
} else {
|
||||
if cdp_msg.text.contains("Fetch.") {
|
||||
handle_fetch_resolution(&cdp_msg.text, &mut ctx, &cdp_msg.reply_tx, &mut intercepted_paused);
|
||||
}
|
||||
process_cdp_message(&cdp_msg.text, &mut ctx, &cdp_msg.reply_tx).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_fetch_resolution(
|
||||
text: &str,
|
||||
ctx: &mut CdpContext,
|
||||
reply_tx: &mpsc::UnboundedSender<String>,
|
||||
intercepted_paused: &mut HashMap<String, tokio::sync::oneshot::Sender<obscura_js::ops::InterceptResolution>>,
|
||||
) {
|
||||
if let Ok(req) = serde_json::from_str::<CdpRequest>(text) {
|
||||
let method = req.method.as_str();
|
||||
let request_id = req.params.get("requestId").and_then(|v| v.as_str()).unwrap_or("");
|
||||
tracing::info!("INTERCEPTION resolution: {} for {}, paused_count={}", method, request_id, intercepted_paused.len());
|
||||
|
||||
if let Some(resolver) = intercepted_paused.remove(request_id) {
|
||||
tracing::info!("INTERCEPTION resolved: {}", request_id);
|
||||
let resolution = match method {
|
||||
"Fetch.continueRequest" => obscura_js::ops::InterceptResolution::Continue {
|
||||
url: None, method: None, headers: None, body: None,
|
||||
},
|
||||
"Fetch.fulfillRequest" => {
|
||||
let status = req.params.get("responseCode").and_then(|v| v.as_u64()).unwrap_or(200) as u16;
|
||||
let raw_body = req.params.get("body").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let body = decode_base64(raw_body);
|
||||
let headers = req.params.get("responseHeaders")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|arr| arr.iter().filter_map(|h| {
|
||||
Some((h.get("name")?.as_str()?.to_string(), h.get("value")?.as_str()?.to_string()))
|
||||
}).collect())
|
||||
.unwrap_or_default();
|
||||
obscura_js::ops::InterceptResolution::Fulfill { status, headers, body }
|
||||
}
|
||||
"Fetch.failRequest" => {
|
||||
let reason = req.params.get("errorReason").and_then(|v| v.as_str()).unwrap_or("Failed").to_string();
|
||||
obscura_js::ops::InterceptResolution::Fail { reason }
|
||||
}
|
||||
_ => return,
|
||||
};
|
||||
let _ = resolver.send(resolution);
|
||||
let resp = crate::types::CdpResponse::success(req.id, json!({}), req.session_id);
|
||||
if let Ok(json) = serde_json::to_string(&resp) {
|
||||
let _ = reply_tx.send(json);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_with_interception(
|
||||
text: &str,
|
||||
ctx: &mut CdpContext,
|
||||
reply_tx: &mpsc::UnboundedSender<String>,
|
||||
rx: &mut mpsc::UnboundedReceiver<ServerMessage>,
|
||||
intercept_rx: &mut Option<mpsc::UnboundedReceiver<obscura_js::ops::InterceptedRequest>>,
|
||||
intercepted_paused: &mut HashMap<String, tokio::sync::oneshot::Sender<obscura_js::ops::InterceptResolution>>,
|
||||
) {
|
||||
let req: CdpRequest = match serde_json::from_str(text) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
warn!("Invalid CDP: {}", e);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
tracing::info!("INTERCEPTION navigate: {} (id={})", req.method, req.id);
|
||||
|
||||
let session_id = &req.session_id;
|
||||
let page_id = session_id
|
||||
.as_ref()
|
||||
.and_then(|sid| ctx.sessions.get(sid))
|
||||
.cloned();
|
||||
|
||||
let page_id = match page_id {
|
||||
Some(id) => id,
|
||||
None => {
|
||||
process_cdp_message(text, ctx, reply_tx).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let page_index = ctx.pages.iter().position(|p| p.id == page_id);
|
||||
let mut page = match page_index {
|
||||
Some(idx) => ctx.pages.remove(idx),
|
||||
None => {
|
||||
process_cdp_message(text, ctx, reply_tx).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let url = req.params.get("url").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let wait_until = req.params.get("waitUntil")
|
||||
.and_then(|v| {
|
||||
if let Some(s) = v.as_str() {
|
||||
Some(obscura_browser::WaitUntil::from_str(s))
|
||||
} else if let Some(arr) = v.as_array() {
|
||||
arr.iter()
|
||||
.filter_map(|item| item.as_str())
|
||||
.map(obscura_browser::WaitUntil::from_str)
|
||||
.max_by_key(|w| match w {
|
||||
obscura_browser::WaitUntil::DomContentLoaded => 0,
|
||||
obscura_browser::WaitUntil::Load => 1,
|
||||
obscura_browser::WaitUntil::NetworkIdle2 => 2,
|
||||
obscura_browser::WaitUntil::NetworkIdle0 => 3,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or(obscura_browser::WaitUntil::Load);
|
||||
|
||||
let preload_scripts: Vec<String> = ctx.preload_scripts.iter().map(|(_, s)| s.clone()).collect();
|
||||
|
||||
if let Some(tx) = &ctx.intercept_tx {
|
||||
page.set_intercept_tx(tx.clone());
|
||||
}
|
||||
|
||||
let session_for_events = req.session_id.clone();
|
||||
let frame_id = page.frame_id.clone();
|
||||
let loader_id = format!("loader-{}", uuid::Uuid::new_v4());
|
||||
|
||||
let (nav_done_tx, mut nav_done_rx) = mpsc::channel::<(obscura_browser::Page, Result<(), String>)>(1);
|
||||
let url_owned = url.to_string();
|
||||
|
||||
tokio::task::spawn_local(async move {
|
||||
let result = page.navigate_with_wait(&url_owned, wait_until).await.map_err(|e| e.to_string());
|
||||
for source in &preload_scripts {
|
||||
if let Err(e) = page.execute_preload_script(source) {
|
||||
tracing::debug!("Preload script error: {}", e);
|
||||
}
|
||||
}
|
||||
let _ = nav_done_tx.send((page, result)).await;
|
||||
});
|
||||
|
||||
let mut navigate_result: Result<(), String> = Ok(());
|
||||
let mut page_back: Option<obscura_browser::Page> = None;
|
||||
|
||||
loop {
|
||||
let has_irx = intercept_rx.is_some();
|
||||
|
||||
tokio::select! {
|
||||
Some((returned_page, result)) = nav_done_rx.recv() => {
|
||||
page_back = Some(returned_page);
|
||||
navigate_result = result;
|
||||
break;
|
||||
}
|
||||
Some(intercepted) = async {
|
||||
if let Some(ref mut irx) = intercept_rx {
|
||||
irx.recv().await
|
||||
} else {
|
||||
std::future::pending().await
|
||||
}
|
||||
}, if has_irx => {
|
||||
tracing::info!("INTERCEPTION: requestPaused for {} {} (sending to client)", intercepted.method, intercepted.url);
|
||||
let rws_event = json!({
|
||||
"method": "Network.requestWillBeSent",
|
||||
"params": {
|
||||
"requestId": intercepted.request_id,
|
||||
"loaderId": "",
|
||||
"documentURL": "",
|
||||
"request": {
|
||||
"url": intercepted.url,
|
||||
"method": intercepted.method,
|
||||
"headers": intercepted.headers,
|
||||
"initialPriority": "High",
|
||||
"referrerPolicy": "strict-origin-when-cross-origin",
|
||||
},
|
||||
"timestamp": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_secs_f64(),
|
||||
"wallTime": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_secs_f64(),
|
||||
"initiator": {"type": "script"},
|
||||
"type": intercepted.resource_type,
|
||||
"frameId": frame_id,
|
||||
},
|
||||
"sessionId": session_for_events,
|
||||
});
|
||||
let _ = reply_tx.send(rws_event.to_string());
|
||||
|
||||
let event_json = json!({
|
||||
"method": "Fetch.requestPaused",
|
||||
"params": {
|
||||
"requestId": intercepted.request_id,
|
||||
"request": {
|
||||
"url": intercepted.url,
|
||||
"method": intercepted.method,
|
||||
"headers": intercepted.headers,
|
||||
"initialPriority": "High",
|
||||
"referrerPolicy": "strict-origin-when-cross-origin",
|
||||
},
|
||||
"frameId": frame_id,
|
||||
"resourceType": intercepted.resource_type,
|
||||
"networkId": intercepted.request_id,
|
||||
"responseErrorReason": null,
|
||||
"responseStatusCode": null,
|
||||
"responseHeaders": null,
|
||||
},
|
||||
"sessionId": session_for_events,
|
||||
});
|
||||
let event_str = event_json.to_string();
|
||||
tracing::info!("INTERCEPTION event JSON: {}", &event_str[..event_str.len().min(300)]);
|
||||
let _ = reply_tx.send(event_str);
|
||||
intercepted_paused.insert(intercepted.request_id.clone(), intercepted.resolver);
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
Some(msg) = rx.recv() => {
|
||||
tracing::info!("INTERCEPTION select: received CDP message during navigation");
|
||||
match msg {
|
||||
ServerMessage::NewConnection { reply_tx: new_tx } => {
|
||||
let pid = ctx.create_page();
|
||||
let sid = format!("{}-session", pid);
|
||||
ctx.sessions.insert(sid.clone(), pid.clone());
|
||||
let _ = new_tx.send(json!({"__init": true, "pageId": pid, "sessionId": sid}).to_string());
|
||||
}
|
||||
ServerMessage::Cdp(msg) => {
|
||||
if msg.text.contains("Fetch.") {
|
||||
handle_fetch_resolution(&msg.text, ctx, &msg.reply_tx, intercepted_paused);
|
||||
} else {
|
||||
process_cdp_message(&msg.text, ctx, &msg.reply_tx).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut page = page_back.expect("navigation task should return the page");
|
||||
|
||||
let network_events: Vec<_> = page.network_events.drain(..).collect();
|
||||
let page_url = page.url_string();
|
||||
let page_id_for_events = page.id.clone();
|
||||
let reached_network_idle = page.lifecycle.is_network_idle();
|
||||
|
||||
ctx.pages.push(page);
|
||||
|
||||
let response = match navigate_result {
|
||||
Ok(()) => crate::types::CdpResponse::success(
|
||||
req.id,
|
||||
json!({"frameId": frame_id, "loaderId": loader_id}),
|
||||
req.session_id.clone(),
|
||||
),
|
||||
Err(e) => crate::types::CdpResponse::error(req.id, -32000, e, req.session_id.clone()),
|
||||
};
|
||||
|
||||
if let Ok(json) = serde_json::to_string(&response) {
|
||||
let _ = reply_tx.send(json);
|
||||
}
|
||||
|
||||
let ts = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_secs_f64();
|
||||
let es = session_for_events;
|
||||
|
||||
for event in [
|
||||
crate::types::CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "init", "timestamp": ts}), session_id: es.clone() },
|
||||
crate::types::CdpEvent { method: "Runtime.executionContextsCleared".into(), params: json!({}), session_id: es.clone() },
|
||||
crate::types::CdpEvent { method: "Page.frameNavigated".into(), params: json!({"frame": {"id": frame_id, "loaderId": loader_id, "url": page_url, "domainAndRegistry": "", "securityOrigin": page_url, "mimeType": "text/html", "adFrameStatus": {"adFrameType": "none"}}, "type": "Navigation"}), session_id: es.clone() },
|
||||
crate::types::CdpEvent { method: "Runtime.executionContextCreated".into(), params: json!({"context": {"id": 2, "origin": page_url, "name": "", "uniqueId": format!("ctx-nav-{}", page_id_for_events), "auxData": {"isDefault": true, "type": "default", "frameId": frame_id}}}), session_id: es.clone() },
|
||||
crate::types::CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "commit", "timestamp": ts}), session_id: es.clone() },
|
||||
] {
|
||||
if let Ok(json) = serde_json::to_string(&event) { let _ = reply_tx.send(json); }
|
||||
}
|
||||
|
||||
for net_event in &network_events {
|
||||
for event in [
|
||||
crate::types::CdpEvent { method: "Network.requestWillBeSent".into(), params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}), session_id: es.clone() },
|
||||
crate::types::CdpEvent { method: "Network.responseReceived".into(), params: json!({"requestId": net_event.request_id, "loaderId": loader_id, "timestamp": net_event.timestamp, "type": net_event.resource_type, "response": {"url": net_event.url, "status": net_event.status, "statusText": "", "headers": &*net_event.response_headers, "mimeType": ""}, "frameId": frame_id}), session_id: es.clone() },
|
||||
crate::types::CdpEvent { method: "Network.loadingFinished".into(), params: json!({"requestId": net_event.request_id, "timestamp": net_event.timestamp, "encodedDataLength": net_event.body_size}), session_id: es.clone() },
|
||||
] {
|
||||
if let Ok(json) = serde_json::to_string(&event) { let _ = reply_tx.send(json); }
|
||||
}
|
||||
}
|
||||
|
||||
for event in [
|
||||
crate::types::CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "DOMContentLoaded", "timestamp": ts}), session_id: es.clone() },
|
||||
crate::types::CdpEvent { method: "Page.domContentEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() },
|
||||
crate::types::CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "load", "timestamp": ts}), session_id: es.clone() },
|
||||
crate::types::CdpEvent { method: "Page.loadEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() },
|
||||
] {
|
||||
if let Ok(json) = serde_json::to_string(&event) { let _ = reply_tx.send(json); }
|
||||
}
|
||||
if reached_network_idle {
|
||||
let idle_event = crate::types::CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "networkIdle", "timestamp": ts}), session_id: es.clone() };
|
||||
if let Ok(json) = serde_json::to_string(&idle_event) { let _ = reply_tx.send(json); }
|
||||
}
|
||||
let stop_event = crate::types::CdpEvent { method: "Page.frameStoppedLoading".into(), params: json!({"frameId": frame_id}), session_id: es };
|
||||
if let Ok(json) = serde_json::to_string(&stop_event) { let _ = reply_tx.send(json); }
|
||||
}
|
||||
|
||||
async fn process_cdp_message(
|
||||
text: &str,
|
||||
ctx: &mut CdpContext,
|
||||
reply_tx: &mpsc::UnboundedSender<String>,
|
||||
) {
|
||||
let req: CdpRequest = match serde_json::from_str(text) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
warn!("Invalid CDP: {}: {}", e, &text[..text.len().min(200)]);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
tracing::debug!("CDP: {} (id={}, s={:?})", req.method, req.id, req.session_id);
|
||||
|
||||
let response = dispatch::dispatch(&req, ctx).await;
|
||||
|
||||
if let Ok(json) = serde_json::to_string(&response) {
|
||||
let _ = reply_tx.send(json);
|
||||
}
|
||||
|
||||
for event in ctx.pending_events.drain(..) {
|
||||
if let Ok(json) = serde_json::to_string(&event) {
|
||||
let _ = reply_tx.send(json);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some((nav_url, nav_method, nav_body)) = check_pending_navigation(ctx, &req.session_id) {
|
||||
tracing::info!("JS-triggered nav: {} {} (body: {} bytes)", nav_method, nav_url, nav_body.len());
|
||||
let nav_req = CdpRequest {
|
||||
id: 0,
|
||||
method: "Page.navigate".to_string(),
|
||||
params: json!({"url": nav_url, "__method": nav_method, "__body": nav_body}),
|
||||
session_id: req.session_id.clone(),
|
||||
};
|
||||
let _ = dispatch::dispatch(&nav_req, ctx).await;
|
||||
for event in ctx.pending_events.drain(..) {
|
||||
if let Ok(json) = serde_json::to_string(&event) {
|
||||
let _ = reply_tx.send(json);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_base64(input: &str) -> String {
|
||||
fn val(c: u8) -> Option<u8> {
|
||||
match c {
|
||||
b'A'..=b'Z' => Some(c - b'A'),
|
||||
b'a'..=b'z' => Some(c - b'a' + 26),
|
||||
b'0'..=b'9' => Some(c - b'0' + 52),
|
||||
b'+' => Some(62),
|
||||
b'/' => Some(63),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
let bytes: Vec<u8> = input.bytes().filter_map(val).collect();
|
||||
let mut out = Vec::with_capacity(bytes.len() * 3 / 4);
|
||||
for chunk in bytes.chunks(4) {
|
||||
let b = [
|
||||
chunk.first().copied().unwrap_or(0),
|
||||
chunk.get(1).copied().unwrap_or(0),
|
||||
chunk.get(2).copied().unwrap_or(0),
|
||||
chunk.get(3).copied().unwrap_or(0),
|
||||
];
|
||||
out.push((b[0] << 2) | (b[1] >> 4));
|
||||
if chunk.len() > 2 { out.push((b[1] << 4) | (b[2] >> 2)); }
|
||||
if chunk.len() > 3 { out.push((b[2] << 6) | b[3]); }
|
||||
}
|
||||
String::from_utf8_lossy(&out).to_string()
|
||||
}
|
||||
|
||||
fn fast_path_response(text: &str) -> Option<String> {
|
||||
let req: CdpRequest = serde_json::from_str(text).ok()?;
|
||||
|
||||
let result = match req.method.as_str() {
|
||||
"Network.enable" | "Network.setCacheDisabled" | "Network.setRequestInterception" |
|
||||
"Page.enable" | "Page.setLifecycleEventsEnabled" | "Page.setInterceptFileChooserDialog" |
|
||||
"Runtime.runIfWaitingForDebugger" | "Runtime.discardConsoleEntries" |
|
||||
"Performance.enable" | "Log.enable" | "Security.enable" |
|
||||
"Emulation.setDeviceMetricsOverride" | "Emulation.setTouchEmulationEnabled" |
|
||||
"CSS.enable" | "Accessibility.enable" | "ServiceWorker.enable" |
|
||||
"Inspector.enable" | "Debugger.enable" | "Profiler.enable" |
|
||||
"HeapProfiler.enable" | "Overlay.enable" | "Storage.enable" |
|
||||
"Target.setAutoAttach" => {
|
||||
Some(json!({}))
|
||||
}
|
||||
"Browser.getVersion" => {
|
||||
Some(json!({
|
||||
"protocolVersion": "1.3",
|
||||
"product": "Obscura/0.1.0",
|
||||
"revision": "0",
|
||||
"userAgent": "Obscura/0.1.0",
|
||||
"jsVersion": "V8",
|
||||
}))
|
||||
}
|
||||
"Browser.setDownloadBehavior" | "Browser.getWindowBounds" => {
|
||||
Some(json!({}))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
if let Some(value) = result {
|
||||
let resp = crate::types::CdpResponse::success(req.id, value, req.session_id);
|
||||
serde_json::to_string(&resp).ok()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn check_pending_navigation(ctx: &CdpContext, session_id: &Option<String>) -> Option<(String, String, String)> {
|
||||
let page_id = session_id
|
||||
.as_ref()
|
||||
.and_then(|sid| ctx.sessions.get(sid))?;
|
||||
let page = ctx.pages.iter().find(|p| &p.id == page_id)?;
|
||||
page.take_pending_navigation()
|
||||
}
|
||||
|
||||
async fn handle_connection(
|
||||
stream: TcpStream,
|
||||
port: u16,
|
||||
msg_tx: mpsc::UnboundedSender<ServerMessage>,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut buf = [0u8; 4];
|
||||
stream.peek(&mut buf).await?;
|
||||
|
||||
if &buf == b"GET " {
|
||||
let mut peek_buf = [0u8; 1024];
|
||||
let n = stream.peek(&mut peek_buf).await?;
|
||||
let line = String::from_utf8_lossy(&peek_buf[..n]);
|
||||
|
||||
if line.contains("/json/version") {
|
||||
return handle_http_json(stream, port, "version").await;
|
||||
} else if line.contains("/json/list") || line.contains("/json\r\n") || line.contains("/json HTTP") {
|
||||
return handle_http_json(stream, port, "list").await;
|
||||
} else if line.contains("/json/protocol") {
|
||||
return handle_http_json(stream, port, "protocol").await;
|
||||
}
|
||||
}
|
||||
|
||||
let ws_stream = tokio_tungstenite::accept_async(stream).await?;
|
||||
info!("WebSocket connected");
|
||||
let (mut ws_sender, mut ws_receiver) = ws_stream.split();
|
||||
|
||||
let (reply_tx, mut reply_rx) = mpsc::unbounded_channel::<String>();
|
||||
|
||||
let _ = msg_tx.send(ServerMessage::NewConnection {
|
||||
reply_tx: reply_tx.clone(),
|
||||
});
|
||||
if let Some(init_msg) = reply_rx.recv().await {
|
||||
tracing::debug!("Connection init: {}", &init_msg[..init_msg.len().min(100)]);
|
||||
}
|
||||
|
||||
let send_task = tokio::task::spawn_local(async move {
|
||||
while let Some(msg) = reply_rx.recv().await {
|
||||
if msg.contains("\"__init\"") {
|
||||
continue;
|
||||
}
|
||||
if ws_sender.send(Message::Text(msg.into())).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
while let Some(msg) = ws_receiver.next().await {
|
||||
let msg = match msg {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
warn!("WS read error: {}", e);
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
match msg {
|
||||
Message::Text(text) => {
|
||||
if text.contains("\"Browser.close\"") {
|
||||
if let Ok(req) = serde_json::from_str::<CdpRequest>(&text) {
|
||||
let resp = crate::types::CdpResponse::success(req.id, json!({}), None);
|
||||
if let Ok(json) = serde_json::to_string(&resp) {
|
||||
let _ = reply_tx.send(json);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if let Some(resp) = fast_path_response(&text) {
|
||||
let _ = reply_tx.send(resp);
|
||||
} else {
|
||||
let _ = msg_tx.send(ServerMessage::Cdp(CdpMessage {
|
||||
text: text.to_string(),
|
||||
reply_tx: reply_tx.clone(),
|
||||
}));
|
||||
}
|
||||
}
|
||||
Message::Close(_) => {
|
||||
info!("WS closed by client");
|
||||
break;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
send_task.abort();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_http_json(stream: TcpStream, port: u16, endpoint: &str) -> anyhow::Result<()> {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
let mut stream = stream;
|
||||
let mut buf = vec![0u8; 4096];
|
||||
let _ = stream.read(&mut buf).await?;
|
||||
|
||||
let body = match endpoint {
|
||||
"version" => serde_json::to_string_pretty(&json!({
|
||||
"Browser": "Obscura/0.1.0",
|
||||
"Protocol-Version": "1.3",
|
||||
"User-Agent": "Obscura/0.1.0 (Headless Browser)",
|
||||
"V8-Version": "N/A",
|
||||
"WebKit-Version": "N/A",
|
||||
"webSocketDebuggerUrl": format!("ws://127.0.0.1:{}/devtools/browser", port),
|
||||
}))?,
|
||||
"list" => serde_json::to_string_pretty(&json!([{
|
||||
"description": "",
|
||||
"devtoolsFrontendUrl": "",
|
||||
"id": "page-1",
|
||||
"title": "",
|
||||
"type": "page",
|
||||
"url": "about:blank",
|
||||
"webSocketDebuggerUrl": format!("ws://127.0.0.1:{}/devtools/page/page-1", port),
|
||||
}]))?,
|
||||
"protocol" => {
|
||||
serde_json::to_string_pretty(&json!({ "version": { "major": "1", "minor": "3" } }))?
|
||||
}
|
||||
_ => "{}".to_string(),
|
||||
};
|
||||
|
||||
let resp = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
|
||||
body.len(), body,
|
||||
);
|
||||
stream.write_all(resp.as_bytes()).await?;
|
||||
stream.flush().await?;
|
||||
Ok(())
|
||||
}
|
||||
74
crates/obscura-cdp/src/types.rs
Normal file
74
crates/obscura-cdp/src/types.rs
Normal file
@@ -0,0 +1,74 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct CdpRequest {
|
||||
pub id: u64,
|
||||
pub method: String,
|
||||
#[serde(default)]
|
||||
pub params: serde_json::Value,
|
||||
#[serde(rename = "sessionId")]
|
||||
pub session_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct CdpResponse {
|
||||
pub id: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub result: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<CdpError>,
|
||||
#[serde(rename = "sessionId", skip_serializing_if = "Option::is_none")]
|
||||
pub session_id: Option<String>,
|
||||
}
|
||||
|
||||
impl CdpResponse {
|
||||
pub fn success(id: u64, result: serde_json::Value, session_id: Option<String>) -> Self {
|
||||
CdpResponse {
|
||||
id,
|
||||
result: Some(result),
|
||||
error: None,
|
||||
session_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn error(id: u64, code: i64, message: String, session_id: Option<String>) -> Self {
|
||||
CdpResponse {
|
||||
id,
|
||||
result: None,
|
||||
error: Some(CdpError { code, message }),
|
||||
session_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct CdpError {
|
||||
pub code: i64,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct CdpEvent {
|
||||
pub method: String,
|
||||
pub params: serde_json::Value,
|
||||
#[serde(rename = "sessionId", skip_serializing_if = "Option::is_none")]
|
||||
pub session_id: Option<String>,
|
||||
}
|
||||
|
||||
impl CdpEvent {
|
||||
pub fn new(method: &str, params: serde_json::Value) -> Self {
|
||||
CdpEvent {
|
||||
method: method.to_string(),
|
||||
params,
|
||||
session_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_session(method: &str, params: serde_json::Value, session_id: String) -> Self {
|
||||
CdpEvent {
|
||||
method: method.to_string(),
|
||||
params,
|
||||
session_id: Some(session_id),
|
||||
}
|
||||
}
|
||||
}
|
||||
31
crates/obscura-cli/Cargo.toml
Normal file
31
crates/obscura-cli/Cargo.toml
Normal file
@@ -0,0 +1,31 @@
|
||||
[package]
|
||||
name = "obscura-cli"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "obscura"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "obscura-worker"
|
||||
path = "src/worker.rs"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
stealth = ["obscura-browser/stealth", "obscura-net/stealth"]
|
||||
|
||||
[dependencies]
|
||||
obscura-browser = { workspace = true }
|
||||
obscura-cdp = { workspace = true }
|
||||
obscura-dom = { workspace = true }
|
||||
obscura-net = { workspace = true }
|
||||
obscura-js = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
url = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
604
crates/obscura-cli/src/main.rs
Normal file
604
crates/obscura-cli/src/main.rs
Normal file
@@ -0,0 +1,604 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use obscura_browser::{BrowserContext, Page};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::process::Command as TokioCommand;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "obscura", about = "Obscura - A lightweight headless browser for web scraping and automation")]
|
||||
struct Args {
|
||||
#[arg(short, long, global = true)]
|
||||
verbose: bool,
|
||||
|
||||
#[command(subcommand)]
|
||||
command: Option<Command>,
|
||||
|
||||
#[arg(short, long, default_value_t = 9222)]
|
||||
port: u16,
|
||||
|
||||
#[arg(long)]
|
||||
proxy: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
obey_robots: bool,
|
||||
|
||||
#[arg(long)]
|
||||
user_agent: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Command {
|
||||
Serve {
|
||||
#[arg(short, long, default_value_t = 9222)]
|
||||
port: u16,
|
||||
|
||||
#[arg(long)]
|
||||
proxy: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
user_agent: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
stealth: bool,
|
||||
|
||||
#[arg(long, default_value_t = 1)]
|
||||
workers: u16,
|
||||
},
|
||||
|
||||
Fetch {
|
||||
url: String,
|
||||
|
||||
#[arg(long, default_value = "html")]
|
||||
dump: DumpFormat,
|
||||
|
||||
#[arg(long)]
|
||||
selector: Option<String>,
|
||||
|
||||
#[arg(long, default_value_t = 5)]
|
||||
wait: u64,
|
||||
|
||||
#[arg(long, default_value = "load")]
|
||||
wait_until: String,
|
||||
|
||||
#[arg(long)]
|
||||
user_agent: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
stealth: bool,
|
||||
|
||||
#[arg(long, short)]
|
||||
eval: Option<String>,
|
||||
|
||||
#[arg(long, short)]
|
||||
quiet: bool,
|
||||
},
|
||||
|
||||
Scrape {
|
||||
urls: Vec<String>,
|
||||
|
||||
#[arg(long, short)]
|
||||
eval: Option<String>,
|
||||
|
||||
#[arg(long, default_value_t = 10)]
|
||||
concurrency: usize,
|
||||
|
||||
#[arg(long, default_value = "json")]
|
||||
format: String,
|
||||
},
|
||||
|
||||
}
|
||||
|
||||
|
||||
#[derive(Clone, Debug, clap::ValueEnum)]
|
||||
enum DumpFormat {
|
||||
Html,
|
||||
Text,
|
||||
Links,
|
||||
}
|
||||
|
||||
fn print_banner(port: u16) {
|
||||
println!(r#"
|
||||
____ _
|
||||
/ __ \| |
|
||||
| | | | |__ ___ ___ _ _ _ __ __ _
|
||||
| | | | '_ \/ __|/ __| | | | '__/ _` |
|
||||
| |__| | |_) \__ \ (__| |_| | | | (_| |
|
||||
\____/|_.__/|___/\___|\__,_|_| \__,_|
|
||||
|
||||
Headless Browser v0.1.0
|
||||
CDP server: ws://127.0.0.1:{}/devtools/browser
|
||||
"#, port);
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let args = Args::parse();
|
||||
|
||||
let filter = if args.verbose { "debug" } else { "warn" };
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(filter)),
|
||||
)
|
||||
.with_writer(std::io::stderr)
|
||||
.init();
|
||||
|
||||
match args.command {
|
||||
Some(Command::Serve { port, proxy, user_agent, stealth, workers }) => {
|
||||
print_banner(port);
|
||||
if let Some(ref proxy) = proxy {
|
||||
tracing::info!("Using proxy: {}", proxy);
|
||||
}
|
||||
if let Some(ref ua) = user_agent {
|
||||
tracing::info!("User-Agent: {}", ua);
|
||||
}
|
||||
if stealth {
|
||||
tracing::info!("Stealth mode enabled (TLS fingerprint spoofing)");
|
||||
}
|
||||
let _ = stealth;
|
||||
|
||||
if workers > 1 {
|
||||
tracing::info!("{} worker processes", workers);
|
||||
run_multi_worker_serve(port, workers, proxy, stealth).await?;
|
||||
} else {
|
||||
obscura_cdp::start_with_options(port, proxy).await?;
|
||||
}
|
||||
}
|
||||
Some(Command::Fetch { url, dump, selector, wait, wait_until, user_agent, stealth, eval, quiet }) => {
|
||||
run_fetch(&url, dump, selector, wait, &wait_until, user_agent, stealth, eval, quiet).await?;
|
||||
}
|
||||
Some(Command::Scrape { urls, eval, concurrency, format }) => {
|
||||
run_parallel_scrape(urls, eval, concurrency, &format).await?;
|
||||
}
|
||||
None => {
|
||||
print_banner(args.port);
|
||||
if let Some(ref proxy) = args.proxy {
|
||||
tracing::info!("Using proxy: {}", proxy);
|
||||
}
|
||||
obscura_cdp::start_with_options(args.port, args.proxy).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_multi_worker_serve(
|
||||
port: u16,
|
||||
workers: u16,
|
||||
proxy: Option<String>,
|
||||
stealth: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
|
||||
|
||||
let exe = std::env::current_exe()?;
|
||||
let mut children = Vec::new();
|
||||
|
||||
for i in 0..workers {
|
||||
let worker_port = port + 1 + i;
|
||||
let mut cmd = std::process::Command::new(&exe);
|
||||
cmd.arg("serve").arg("--port").arg(worker_port.to_string());
|
||||
if let Some(ref p) = proxy {
|
||||
cmd.arg("--proxy").arg(p);
|
||||
}
|
||||
if stealth {
|
||||
cmd.arg("--stealth");
|
||||
}
|
||||
cmd.stdout(std::process::Stdio::null());
|
||||
cmd.stderr(std::process::Stdio::null());
|
||||
|
||||
let child = cmd.spawn()?;
|
||||
tracing::info!("Worker {} on port {}", i + 1, worker_port);
|
||||
children.push(child);
|
||||
}
|
||||
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
|
||||
|
||||
let addr = std::net::SocketAddr::from(([127, 0, 0, 1], port));
|
||||
let listener = TcpListener::bind(&addr).await?;
|
||||
tracing::info!("Load balancer on port {}, {} workers", port, workers);
|
||||
|
||||
let mut next_worker: u16 = 0;
|
||||
|
||||
loop {
|
||||
let (client_stream, peer_addr) = listener.accept().await?;
|
||||
let worker_port = port + 1 + (next_worker % workers);
|
||||
next_worker = next_worker.wrapping_add(1);
|
||||
|
||||
tracing::debug!("Routing {} to worker port {}", peer_addr, worker_port);
|
||||
|
||||
let mut peek_buf = [0u8; 4];
|
||||
client_stream.peek(&mut peek_buf).await?;
|
||||
|
||||
if &peek_buf == b"GET " {
|
||||
let mut full_peek = [0u8; 256];
|
||||
let n = client_stream.peek(&mut full_peek).await?;
|
||||
let request_line = String::from_utf8_lossy(&full_peek[..n]);
|
||||
|
||||
if request_line.contains("/json") {
|
||||
let worker_addr = format!("127.0.0.1:{}", port + 1);
|
||||
if let Ok(mut worker_stream) = tokio::net::TcpStream::connect(&worker_addr).await {
|
||||
tokio::spawn(async move {
|
||||
let _ = tokio::io::copy_bidirectional(&mut tokio::net::TcpStream::from_std(client_stream.into_std().unwrap()).unwrap(), &mut worker_stream).await;
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let worker_addr = format!("127.0.0.1:{}", worker_port);
|
||||
tokio::spawn(async move {
|
||||
if let Ok(mut worker_stream) = tokio::net::TcpStream::connect(&worker_addr).await {
|
||||
let mut client = client_stream;
|
||||
let _ = tokio::io::copy_bidirectional(&mut client, &mut worker_stream).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_fetch(
|
||||
url_str: &str,
|
||||
dump: DumpFormat,
|
||||
selector: Option<String>,
|
||||
wait_secs: u64,
|
||||
wait_until: &str,
|
||||
user_agent: Option<String>,
|
||||
stealth: bool,
|
||||
eval: Option<String>,
|
||||
quiet: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
let context = Arc::new(BrowserContext::with_options("fetch".to_string(), None, stealth));
|
||||
let mut page = Page::new("fetch-page".to_string(), context);
|
||||
|
||||
if let Some(ref ua) = user_agent {
|
||||
page.http_client.set_user_agent(ua).await;
|
||||
}
|
||||
|
||||
let wait_condition = obscura_browser::lifecycle::WaitUntil::from_str(wait_until);
|
||||
|
||||
if !quiet {
|
||||
eprintln!("Fetching {}...", url_str);
|
||||
}
|
||||
|
||||
page.navigate_with_wait(url_str, wait_condition).await.map_err(|e| {
|
||||
anyhow::anyhow!("Failed to navigate to {}: {}", url_str, e)
|
||||
})?;
|
||||
|
||||
if !quiet {
|
||||
eprintln!("Page loaded: {} - \"{}\"", page.url_string(), page.title);
|
||||
}
|
||||
|
||||
if let Some(ref sel) = selector {
|
||||
let found = wait_for_selector(&mut page, sel, wait_secs).await;
|
||||
if !found {
|
||||
eprintln!("Warning: selector '{}' not found after {}s", sel, wait_secs);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref expr) = eval {
|
||||
let result = page.evaluate(expr);
|
||||
match result {
|
||||
serde_json::Value::String(s) => println!("{}", s),
|
||||
serde_json::Value::Null => println!("null"),
|
||||
other => println!("{}", other),
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match dump {
|
||||
DumpFormat::Html => {
|
||||
dump_html(&page);
|
||||
}
|
||||
DumpFormat::Text => {
|
||||
dump_text(&mut page);
|
||||
}
|
||||
DumpFormat::Links => {
|
||||
dump_links(&page);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_selector(page: &mut Page, selector: &str, timeout_secs: u64) -> bool {
|
||||
let deadline = tokio::time::Instant::now() + tokio::time::Duration::from_secs(timeout_secs);
|
||||
loop {
|
||||
let found = page.with_dom(|dom| {
|
||||
dom.query_selector(selector).ok().flatten().is_some()
|
||||
}).unwrap_or(false);
|
||||
|
||||
if found {
|
||||
return true;
|
||||
}
|
||||
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return false;
|
||||
}
|
||||
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn dump_html(page: &Page) {
|
||||
page.with_dom(|dom| {
|
||||
if let Ok(Some(html_node)) = dom.query_selector("html") {
|
||||
let html = dom.outer_html(html_node);
|
||||
println!("<!DOCTYPE html>");
|
||||
println!("{}", html);
|
||||
} else {
|
||||
let doc = dom.document();
|
||||
let html = dom.inner_html(doc);
|
||||
println!("{}", html);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn dump_text(page: &mut Page) {
|
||||
page.with_dom(|dom| {
|
||||
if let Ok(Some(body)) = dom.query_selector("body") {
|
||||
let text = extract_readable_text(dom, body);
|
||||
println!("{}", text.trim());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn extract_readable_text(dom: &obscura_dom::DomTree, node_id: obscura_dom::NodeId) -> String {
|
||||
use obscura_dom::NodeData;
|
||||
|
||||
let mut result = String::new();
|
||||
let node = match dom.get_node(node_id) {
|
||||
Some(n) => n,
|
||||
None => return result,
|
||||
};
|
||||
|
||||
match &node.data {
|
||||
NodeData::Text { contents } => {
|
||||
let trimmed = contents.trim();
|
||||
if !trimmed.is_empty() {
|
||||
result.push_str(trimmed);
|
||||
}
|
||||
}
|
||||
NodeData::Element { name, .. } => {
|
||||
let tag = name.local.as_ref();
|
||||
let is_block = matches!(
|
||||
tag,
|
||||
"div" | "p" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6"
|
||||
| "li" | "tr" | "br" | "hr" | "blockquote" | "pre"
|
||||
| "section" | "article" | "header" | "footer" | "nav"
|
||||
| "main" | "aside" | "figure" | "figcaption" | "table"
|
||||
| "thead" | "tbody" | "tfoot" | "dl" | "dt" | "dd"
|
||||
| "ul" | "ol"
|
||||
);
|
||||
|
||||
if tag == "script" || tag == "style" {
|
||||
return result;
|
||||
}
|
||||
|
||||
if is_block {
|
||||
result.push('\n');
|
||||
}
|
||||
|
||||
for child_id in dom.children(node_id) {
|
||||
result.push_str(&extract_readable_text(dom, child_id));
|
||||
}
|
||||
|
||||
if is_block {
|
||||
result.push('\n');
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
for child_id in dom.children(node_id) {
|
||||
result.push_str(&extract_readable_text(dom, child_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn run_parallel_scrape(
|
||||
urls: Vec<String>,
|
||||
eval: Option<String>,
|
||||
concurrency: usize,
|
||||
format: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
let total = urls.len();
|
||||
let start = Instant::now();
|
||||
|
||||
eprintln!(
|
||||
"Scraping {} URLs with {} concurrent workers...",
|
||||
total, concurrency
|
||||
);
|
||||
|
||||
let worker_path = std::env::current_exe()
|
||||
.ok()
|
||||
.and_then(|p| p.parent().map(|d| d.join("obscura-worker")))
|
||||
.unwrap_or_else(|| std::path::PathBuf::from("obscura-worker"));
|
||||
|
||||
if !worker_path.exists() {
|
||||
anyhow::bail!(
|
||||
"Worker binary not found at {}. Build with: cargo build --release",
|
||||
worker_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(concurrency));
|
||||
let eval = Arc::new(eval);
|
||||
let worker_path = Arc::new(worker_path);
|
||||
|
||||
let mut handles = Vec::new();
|
||||
|
||||
for (i, url) in urls.into_iter().enumerate() {
|
||||
let sem = semaphore.clone();
|
||||
let eval = eval.clone();
|
||||
let worker_path = worker_path.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let _permit = sem.acquire().await.unwrap();
|
||||
let task_start = Instant::now();
|
||||
|
||||
let mut child = match TokioCommand::new(worker_path.as_ref())
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.spawn()
|
||||
{
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
return serde_json::json!({
|
||||
"url": url,
|
||||
"error": format!("Failed to spawn worker: {}", e),
|
||||
"time_ms": task_start.elapsed().as_millis(),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
let stdin = child.stdin.as_mut().unwrap();
|
||||
let stdout = child.stdout.take().unwrap();
|
||||
let mut reader = BufReader::new(stdout);
|
||||
|
||||
let nav_cmd = serde_json::json!({"cmd": "navigate", "url": url});
|
||||
let mut line = serde_json::to_string(&nav_cmd).unwrap();
|
||||
line.push('\n');
|
||||
if stdin.write_all(line.as_bytes()).await.is_err() {
|
||||
let _ = child.kill().await;
|
||||
return serde_json::json!({"url": url, "error": "Write failed"});
|
||||
}
|
||||
let _ = stdin.flush().await;
|
||||
|
||||
let mut resp_line = String::new();
|
||||
if reader.read_line(&mut resp_line).await.is_err() {
|
||||
let _ = child.kill().await;
|
||||
return serde_json::json!({"url": url, "error": "Read failed"});
|
||||
}
|
||||
|
||||
let nav_resp: serde_json::Value =
|
||||
serde_json::from_str(resp_line.trim()).unwrap_or(serde_json::json!({"ok": false}));
|
||||
|
||||
if !nav_resp["ok"].as_bool().unwrap_or(false) {
|
||||
let _ = child.kill().await;
|
||||
return serde_json::json!({
|
||||
"url": url,
|
||||
"error": nav_resp["error"].as_str().unwrap_or("navigate failed"),
|
||||
"time_ms": task_start.elapsed().as_millis(),
|
||||
});
|
||||
}
|
||||
|
||||
let title = nav_resp["result"]["title"]
|
||||
.as_str()
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let eval_result = if let Some(ref expr) = *eval {
|
||||
let eval_cmd = serde_json::json!({"cmd": "evaluate", "expression": expr});
|
||||
let mut line = serde_json::to_string(&eval_cmd).unwrap();
|
||||
line.push('\n');
|
||||
let _ = stdin.write_all(line.as_bytes()).await;
|
||||
let _ = stdin.flush().await;
|
||||
|
||||
let mut resp_line = String::new();
|
||||
if reader.read_line(&mut resp_line).await.is_ok() {
|
||||
let resp: serde_json::Value = serde_json::from_str(resp_line.trim())
|
||||
.unwrap_or(serde_json::json!({"ok": false}));
|
||||
resp["result"].clone()
|
||||
} else {
|
||||
serde_json::Value::Null
|
||||
}
|
||||
} else {
|
||||
serde_json::Value::Null
|
||||
};
|
||||
|
||||
let shutdown_cmd = serde_json::json!({"cmd": "shutdown"});
|
||||
let mut line = serde_json::to_string(&shutdown_cmd).unwrap();
|
||||
line.push('\n');
|
||||
let _ = stdin.write_all(line.as_bytes()).await;
|
||||
let _ = stdin.flush().await;
|
||||
let _ = child.wait().await;
|
||||
|
||||
let elapsed = task_start.elapsed().as_millis();
|
||||
|
||||
serde_json::json!({
|
||||
"url": url,
|
||||
"title": title,
|
||||
"eval": eval_result,
|
||||
"time_ms": elapsed,
|
||||
"worker": i,
|
||||
})
|
||||
});
|
||||
|
||||
handles.push(handle);
|
||||
}
|
||||
|
||||
let mut results = Vec::new();
|
||||
for handle in handles {
|
||||
match handle.await {
|
||||
Ok(result) => results.push(result),
|
||||
Err(e) => results.push(serde_json::json!({"error": e.to_string()})),
|
||||
}
|
||||
}
|
||||
|
||||
let total_time = start.elapsed();
|
||||
|
||||
if format == "json" {
|
||||
let output = serde_json::json!({
|
||||
"total_urls": total,
|
||||
"concurrency": concurrency,
|
||||
"total_time_ms": total_time.as_millis(),
|
||||
"avg_time_ms": total_time.as_millis() as f64 / total as f64,
|
||||
"results": results,
|
||||
});
|
||||
println!("{}", serde_json::to_string_pretty(&output)?);
|
||||
} else {
|
||||
for r in &results {
|
||||
let url = r["url"].as_str().unwrap_or("?");
|
||||
let title = r["title"].as_str().unwrap_or("");
|
||||
let time = r["time_ms"].as_u64().unwrap_or(0);
|
||||
let eval = &r["eval"];
|
||||
if eval.is_null() {
|
||||
println!("{}ms\t{}\t{}", time, url, title);
|
||||
} else {
|
||||
println!("{}ms\t{}\t{}", time, url, eval);
|
||||
}
|
||||
}
|
||||
eprintln!(
|
||||
"\nTotal: {}ms for {} URLs ({} concurrent)",
|
||||
total_time.as_millis(),
|
||||
total,
|
||||
concurrency
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn dump_links(page: &Page) {
|
||||
let base_url = page.url.clone();
|
||||
page.with_dom(|dom| {
|
||||
let links = dom.query_selector_all("a").unwrap_or_default();
|
||||
for link_id in links {
|
||||
if let Some(node) = dom.get_node(link_id) {
|
||||
let href = node.get_attribute("href").unwrap_or_default().to_string();
|
||||
let text = dom.text_content(link_id);
|
||||
let text = text.trim();
|
||||
|
||||
let full_url = if href.starts_with("http://") || href.starts_with("https://") {
|
||||
href.clone()
|
||||
} else if let Some(ref base) = base_url {
|
||||
base.join(&href).map(|u| u.to_string()).unwrap_or(href.clone())
|
||||
} else {
|
||||
href.clone()
|
||||
};
|
||||
|
||||
if !full_url.is_empty() {
|
||||
if text.is_empty() {
|
||||
println!("{}", full_url);
|
||||
} else {
|
||||
println!("{}\t{}", full_url, text);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
138
crates/obscura-cli/src/worker.rs
Normal file
138
crates/obscura-cli/src/worker.rs
Normal file
@@ -0,0 +1,138 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use obscura_browser::{BrowserContext, Page};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "cmd")]
|
||||
enum WorkerCommand {
|
||||
#[serde(rename = "navigate")]
|
||||
Navigate { url: String },
|
||||
#[serde(rename = "evaluate")]
|
||||
Evaluate { expression: String },
|
||||
#[serde(rename = "title")]
|
||||
Title,
|
||||
#[serde(rename = "dump_html")]
|
||||
DumpHtml,
|
||||
#[serde(rename = "dump_text")]
|
||||
DumpText,
|
||||
#[serde(rename = "shutdown")]
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct WorkerResponse {
|
||||
ok: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
result: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
impl WorkerResponse {
|
||||
fn success(result: serde_json::Value) -> Self {
|
||||
WorkerResponse { ok: true, result: Some(result), error: None }
|
||||
}
|
||||
fn error(msg: String) -> Self {
|
||||
WorkerResponse { ok: false, result: None, error: Some(msg) }
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter("warn")
|
||||
.with_writer(std::io::stderr)
|
||||
.init();
|
||||
|
||||
let context = Arc::new(BrowserContext::new("worker".to_string()));
|
||||
let mut page = Page::new("page-1".to_string(), context);
|
||||
|
||||
let stdin = tokio::io::stdin();
|
||||
let mut stdout = tokio::io::stdout();
|
||||
let mut reader = BufReader::new(stdin);
|
||||
let mut line = String::new();
|
||||
|
||||
loop {
|
||||
line.clear();
|
||||
match reader.read_line(&mut line).await {
|
||||
Ok(0) => break,
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
eprintln!("Worker stdin error: {}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let cmd: WorkerCommand = match serde_json::from_str(trimmed) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
let resp = WorkerResponse::error(format!("Invalid command: {}", e));
|
||||
let mut out = serde_json::to_string(&resp).unwrap();
|
||||
out.push('\n');
|
||||
let _ = stdout.write_all(out.as_bytes()).await;
|
||||
let _ = stdout.flush().await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let resp = match cmd {
|
||||
WorkerCommand::Navigate { url } => {
|
||||
match page.navigate(&url).await {
|
||||
Ok(()) => WorkerResponse::success(serde_json::json!({
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
})),
|
||||
Err(e) => WorkerResponse::error(e.to_string()),
|
||||
}
|
||||
}
|
||||
WorkerCommand::Evaluate { expression } => {
|
||||
let result = page.evaluate(&expression);
|
||||
WorkerResponse::success(result)
|
||||
}
|
||||
WorkerCommand::Title => {
|
||||
WorkerResponse::success(serde_json::json!(page.title))
|
||||
}
|
||||
WorkerCommand::DumpHtml => {
|
||||
let html = page.with_dom(|dom| {
|
||||
if let Ok(Some(html_node)) = dom.query_selector("html") {
|
||||
dom.outer_html(html_node)
|
||||
} else {
|
||||
dom.inner_html(dom.document())
|
||||
}
|
||||
}).unwrap_or_default();
|
||||
WorkerResponse::success(serde_json::json!(html))
|
||||
}
|
||||
WorkerCommand::DumpText => {
|
||||
let text = page.with_dom(|dom| {
|
||||
if let Ok(Some(body)) = dom.query_selector("body") {
|
||||
dom.text_content(body)
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
}).unwrap_or_default();
|
||||
WorkerResponse::success(serde_json::json!(text))
|
||||
}
|
||||
WorkerCommand::Shutdown => {
|
||||
let resp = WorkerResponse::success(serde_json::json!("bye"));
|
||||
let mut out = serde_json::to_string(&resp).unwrap();
|
||||
out.push('\n');
|
||||
let _ = stdout.write_all(out.as_bytes()).await;
|
||||
let _ = stdout.flush().await;
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let mut out = serde_json::to_string(&resp).unwrap();
|
||||
out.push('\n');
|
||||
let _ = stdout.write_all(out.as_bytes()).await;
|
||||
let _ = stdout.flush().await;
|
||||
}
|
||||
}
|
||||
16
crates/obscura-dom/Cargo.toml
Normal file
16
crates/obscura-dom/Cargo.toml
Normal file
@@ -0,0 +1,16 @@
|
||||
[package]
|
||||
name = "obscura-dom"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
html5ever = { workspace = true }
|
||||
markup5ever = { workspace = true }
|
||||
selectors = { workspace = true }
|
||||
servo_arc = { workspace = true }
|
||||
cssparser = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
precomputed-hash = "0.1"
|
||||
|
||||
[dev-dependencies]
|
||||
10
crates/obscura-dom/src/lib.rs
Normal file
10
crates/obscura-dom/src/lib.rs
Normal file
@@ -0,0 +1,10 @@
|
||||
#[macro_use]
|
||||
extern crate html5ever;
|
||||
|
||||
pub mod tree;
|
||||
pub mod tree_sink;
|
||||
pub mod selector;
|
||||
pub mod serialize;
|
||||
|
||||
pub use tree::{Attribute, DomTree, Node, NodeData, NodeId};
|
||||
pub use tree_sink::{parse_html, parse_fragment};
|
||||
641
crates/obscura-dom/src/selector.rs
Normal file
641
crates/obscura-dom/src/selector.rs
Normal file
@@ -0,0 +1,641 @@
|
||||
use cssparser::{CowRcStr, ToCss};
|
||||
use html5ever::{LocalName, Namespace};
|
||||
use precomputed_hash::PrecomputedHash;
|
||||
use selectors::attr::{AttrSelectorOperation, CaseSensitivity, NamespaceConstraint};
|
||||
use selectors::bloom::BloomFilter;
|
||||
use selectors::context::QuirksMode;
|
||||
use selectors::matching::{
|
||||
ElementSelectorFlags, MatchingContext, MatchingForInvalidation, MatchingMode,
|
||||
NeedsSelectorFlags,
|
||||
};
|
||||
use selectors::parser::{self, ParseRelative, SelectorParseErrorKind};
|
||||
use selectors::{Element, OpaqueElement, SelectorList};
|
||||
use selectors::visitor::SelectorVisitor;
|
||||
|
||||
use crate::tree::{DomTree, NodeData, NodeId};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ObscuraSelector;
|
||||
|
||||
impl parser::SelectorImpl for ObscuraSelector {
|
||||
type ExtraMatchingData<'a> = ();
|
||||
type AttrValue = CssString;
|
||||
type Identifier = CssString;
|
||||
type LocalName = CssLocalName;
|
||||
type NamespaceUrl = CssNamespace;
|
||||
type NamespacePrefix = CssString;
|
||||
type BorrowedLocalName = CssLocalName;
|
||||
type BorrowedNamespaceUrl = CssNamespace;
|
||||
type NonTSPseudoClass = PseudoClass;
|
||||
type PseudoElement = PseudoElement;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct CssString(pub String);
|
||||
|
||||
impl<'a> From<&'a str> for CssString {
|
||||
fn from(s: &'a str) -> Self {
|
||||
CssString(s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<str> for CssString {
|
||||
fn as_ref(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCss for CssString {
|
||||
fn to_css<W: std::fmt::Write>(&self, dest: &mut W) -> std::fmt::Result {
|
||||
cssparser::serialize_string(&self.0, dest)
|
||||
}
|
||||
}
|
||||
|
||||
impl PrecomputedHash for CssString {
|
||||
fn precomputed_hash(&self) -> u32 {
|
||||
let mut h: u32 = 5381;
|
||||
for b in self.0.as_bytes() {
|
||||
h = h.wrapping_mul(33).wrapping_add(*b as u32);
|
||||
}
|
||||
h
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CssString {
|
||||
fn default() -> Self {
|
||||
CssString(String::new())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct CssLocalName(pub LocalName);
|
||||
|
||||
impl<'a> From<&'a str> for CssLocalName {
|
||||
fn from(s: &'a str) -> Self {
|
||||
CssLocalName(LocalName::from(s))
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCss for CssLocalName {
|
||||
fn to_css<W: std::fmt::Write>(&self, dest: &mut W) -> std::fmt::Result {
|
||||
dest.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl PrecomputedHash for CssLocalName {
|
||||
fn precomputed_hash(&self) -> u32 {
|
||||
self.0.precomputed_hash()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
|
||||
pub struct CssNamespace(pub Namespace);
|
||||
|
||||
impl PrecomputedHash for CssNamespace {
|
||||
fn precomputed_hash(&self) -> u32 {
|
||||
self.0.precomputed_hash()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum PseudoClass {
|
||||
Hover,
|
||||
Active,
|
||||
Focus,
|
||||
Enabled,
|
||||
Disabled,
|
||||
Checked,
|
||||
}
|
||||
|
||||
impl parser::NonTSPseudoClass for PseudoClass {
|
||||
type Impl = ObscuraSelector;
|
||||
|
||||
fn is_active_or_hover(&self) -> bool {
|
||||
matches!(self, PseudoClass::Hover | PseudoClass::Active)
|
||||
}
|
||||
|
||||
fn is_user_action_state(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
PseudoClass::Hover | PseudoClass::Active | PseudoClass::Focus
|
||||
)
|
||||
}
|
||||
|
||||
fn visit<V>(&self, _visitor: &mut V) -> bool
|
||||
where
|
||||
V: SelectorVisitor<Impl = Self::Impl>,
|
||||
{
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCss for PseudoClass {
|
||||
fn to_css<W: std::fmt::Write>(&self, dest: &mut W) -> std::fmt::Result {
|
||||
match self {
|
||||
PseudoClass::Hover => dest.write_str(":hover"),
|
||||
PseudoClass::Active => dest.write_str(":active"),
|
||||
PseudoClass::Focus => dest.write_str(":focus"),
|
||||
PseudoClass::Enabled => dest.write_str(":enabled"),
|
||||
PseudoClass::Disabled => dest.write_str(":disabled"),
|
||||
PseudoClass::Checked => dest.write_str(":checked"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum PseudoElement {
|
||||
Before,
|
||||
After,
|
||||
}
|
||||
|
||||
impl parser::PseudoElement for PseudoElement {
|
||||
type Impl = ObscuraSelector;
|
||||
}
|
||||
|
||||
impl ToCss for PseudoElement {
|
||||
fn to_css<W: std::fmt::Write>(&self, dest: &mut W) -> std::fmt::Result {
|
||||
match self {
|
||||
PseudoElement::Before => dest.write_str("::before"),
|
||||
PseudoElement::After => dest.write_str("::after"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ObscuraSelectorParser;
|
||||
|
||||
impl<'i> parser::Parser<'i> for ObscuraSelectorParser {
|
||||
type Impl = ObscuraSelector;
|
||||
type Error = SelectorParseErrorKind<'i>;
|
||||
|
||||
fn parse_non_ts_pseudo_class(
|
||||
&self,
|
||||
_location: cssparser::SourceLocation,
|
||||
name: CowRcStr<'i>,
|
||||
) -> Result<PseudoClass, cssparser::ParseError<'i, Self::Error>> {
|
||||
match name.as_ref() {
|
||||
"hover" => Ok(PseudoClass::Hover),
|
||||
"active" => Ok(PseudoClass::Active),
|
||||
"focus" => Ok(PseudoClass::Focus),
|
||||
"enabled" => Ok(PseudoClass::Enabled),
|
||||
"disabled" => Ok(PseudoClass::Disabled),
|
||||
"checked" => Ok(PseudoClass::Checked),
|
||||
_ => Err(cssparser::ParseError {
|
||||
kind: cssparser::ParseErrorKind::Custom(
|
||||
SelectorParseErrorKind::UnsupportedPseudoClassOrElement(name),
|
||||
),
|
||||
location: _location,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct DomElement<'a> {
|
||||
pub tree: &'a DomTree,
|
||||
pub node_id: NodeId,
|
||||
}
|
||||
|
||||
impl<'a> DomElement<'a> {
|
||||
pub fn new(tree: &'a DomTree, node_id: NodeId) -> Self {
|
||||
DomElement { tree, node_id }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> std::fmt::Debug for DomElement<'a> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "DomElement({:?})", self.node_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> PartialEq for DomElement<'a> {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.node_id == other.node_id
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Eq for DomElement<'a> {}
|
||||
|
||||
impl<'a> Element for DomElement<'a> {
|
||||
type Impl = ObscuraSelector;
|
||||
|
||||
fn opaque(&self) -> OpaqueElement {
|
||||
OpaqueElement::new(self)
|
||||
}
|
||||
|
||||
fn parent_element(&self) -> Option<Self> {
|
||||
let node = self.tree.get_node(self.node_id)?;
|
||||
let parent_id = node.parent?;
|
||||
let parent = self.tree.get_node(parent_id)?;
|
||||
if parent.is_element() {
|
||||
Some(DomElement::new(self.tree, parent_id))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn parent_node_is_shadow_root(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn containing_shadow_host(&self) -> Option<Self> {
|
||||
None
|
||||
}
|
||||
|
||||
fn pseudo_element_originating_element(&self) -> Option<Self> {
|
||||
None
|
||||
}
|
||||
|
||||
fn is_pseudo_element(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn prev_sibling_element(&self) -> Option<Self> {
|
||||
let node = self.tree.get_node(self.node_id)?;
|
||||
let mut current = node.prev_sibling;
|
||||
while let Some(sibling_id) = current {
|
||||
let sibling = self.tree.get_node(sibling_id)?;
|
||||
if sibling.is_element() {
|
||||
return Some(DomElement::new(self.tree, sibling_id));
|
||||
}
|
||||
current = sibling.prev_sibling;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn next_sibling_element(&self) -> Option<Self> {
|
||||
let node = self.tree.get_node(self.node_id)?;
|
||||
let mut current = node.next_sibling;
|
||||
while let Some(sibling_id) = current {
|
||||
let sibling = self.tree.get_node(sibling_id)?;
|
||||
if sibling.is_element() {
|
||||
return Some(DomElement::new(self.tree, sibling_id));
|
||||
}
|
||||
current = sibling.next_sibling;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn first_element_child(&self) -> Option<Self> {
|
||||
let node = self.tree.get_node(self.node_id)?;
|
||||
let mut current = node.first_child;
|
||||
while let Some(child_id) = current {
|
||||
let child = self.tree.get_node(child_id)?;
|
||||
if child.is_element() {
|
||||
return Some(DomElement::new(self.tree, child_id));
|
||||
}
|
||||
current = child.next_sibling;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn is_html_element_in_html_document(&self) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.as_element()
|
||||
.map(|name| name.ns == ns!(html))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn has_local_name(&self, local_name: &CssLocalName) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.as_element()
|
||||
.map(|name| name.local == local_name.0)
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn has_namespace(&self, ns: &CssNamespace) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.as_element()
|
||||
.map(|name| name.ns == ns.0)
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn is_same_type(&self, other: &Self) -> bool {
|
||||
let self_name = self.tree.with_node(self.node_id, |n| {
|
||||
n.as_element().map(|name| (name.local.clone(), name.ns.clone()))
|
||||
}).flatten();
|
||||
let other_name = self.tree.with_node(other.node_id, |n| {
|
||||
n.as_element().map(|name| (name.local.clone(), name.ns.clone()))
|
||||
}).flatten();
|
||||
match (self_name, other_name) {
|
||||
(Some((al, ans)), Some((bl, bns))) => al == bl && ans == bns,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn attr_matches(
|
||||
&self,
|
||||
ns: &NamespaceConstraint<&CssNamespace>,
|
||||
local_name: &CssLocalName,
|
||||
operation: &AttrSelectorOperation<&CssString>,
|
||||
) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |node| {
|
||||
let attrs = match node.attrs() {
|
||||
Some(a) => a,
|
||||
None => return false,
|
||||
};
|
||||
attrs.iter().any(|attr| {
|
||||
let ns_match = match ns {
|
||||
NamespaceConstraint::Any => true,
|
||||
NamespaceConstraint::Specific(expected_ns) => attr.name.ns == expected_ns.0,
|
||||
};
|
||||
if !ns_match || attr.name.local != local_name.0 {
|
||||
return false;
|
||||
}
|
||||
operation.eval_str(&attr.value)
|
||||
})
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn has_attr_in_no_namespace(&self, local_name: &CssLocalName) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |node| {
|
||||
node.attrs()
|
||||
.map(|attrs| {
|
||||
attrs.iter().any(|a| {
|
||||
a.name.ns == html5ever::ns!()
|
||||
&& a.name.local == local_name.0
|
||||
})
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn match_non_ts_pseudo_class(
|
||||
&self,
|
||||
_pc: &PseudoClass,
|
||||
_context: &mut MatchingContext<'_, Self::Impl>,
|
||||
) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn match_pseudo_element(
|
||||
&self,
|
||||
_pe: &PseudoElement,
|
||||
_context: &mut MatchingContext<'_, Self::Impl>,
|
||||
) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn apply_selector_flags(&self, _flags: ElementSelectorFlags) {}
|
||||
|
||||
fn is_link(&self) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.as_element()
|
||||
.map(|name| {
|
||||
matches!(name.local.as_ref(), "a" | "area" | "link")
|
||||
&& n.get_attribute("href").is_some()
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn is_html_slot_element(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn assigned_slot(&self) -> Option<Self> {
|
||||
None
|
||||
}
|
||||
|
||||
fn has_id(&self, id: &CssString, case_sensitivity: CaseSensitivity) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.get_attribute("id")
|
||||
.map(|value| case_sensitivity.eq(value.as_bytes(), id.0.as_bytes()))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn has_class(&self, name: &CssString, case_sensitivity: CaseSensitivity) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.get_attribute("class")
|
||||
.map(|class_attr| {
|
||||
class_attr
|
||||
.split_whitespace()
|
||||
.any(|c| case_sensitivity.eq(c.as_bytes(), name.0.as_bytes()))
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn has_custom_state(&self, _name: &CssString) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn imported_part(&self, _name: &CssString) -> Option<CssString> {
|
||||
None
|
||||
}
|
||||
|
||||
fn is_part(&self, _name: &CssString) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |node| {
|
||||
let mut child = node.first_child;
|
||||
while let Some(child_id) = child {
|
||||
if let Some(child_node) = self.tree.get_node(child_id) {
|
||||
match &child_node.data {
|
||||
NodeData::Element { .. } => return false,
|
||||
NodeData::Text { contents } if !contents.is_empty() => return false,
|
||||
_ => {}
|
||||
}
|
||||
child = child_node.next_sibling;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
true
|
||||
})
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
fn is_root(&self) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.parent
|
||||
.map(|parent_id| {
|
||||
self.tree
|
||||
.with_node(parent_id, |p| p.is_document())
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn ignores_nth_child_selectors(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn add_element_unique_hashes(&self, _filter: &mut BloomFilter) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_selector(selector: &str) -> Result<SelectorList<ObscuraSelector>, String> {
|
||||
let mut parser_input = cssparser::ParserInput::new(selector);
|
||||
let mut parser = cssparser::Parser::new(&mut parser_input);
|
||||
SelectorList::parse(&ObscuraSelectorParser, &mut parser, ParseRelative::No)
|
||||
.map_err(|e| format!("Failed to parse selector '{}': {:?}", selector, e))
|
||||
}
|
||||
|
||||
impl DomTree {
|
||||
pub fn query_selector(&self, selector: &str) -> Result<Option<NodeId>, String> {
|
||||
let selector_list = parse_selector(selector)?;
|
||||
let mut caches = selectors::context::SelectorCaches::default();
|
||||
let mut context = MatchingContext::new(
|
||||
MatchingMode::Normal,
|
||||
None,
|
||||
&mut caches,
|
||||
QuirksMode::NoQuirks,
|
||||
NeedsSelectorFlags::No,
|
||||
MatchingForInvalidation::No,
|
||||
);
|
||||
|
||||
let doc = self.document();
|
||||
for desc_id in self.descendants(doc) {
|
||||
let is_element = self.with_node(desc_id, |n| n.is_element()).unwrap_or(false);
|
||||
if is_element {
|
||||
let element = DomElement::new(self, desc_id);
|
||||
if selectors::matching::matches_selector_list(
|
||||
&selector_list,
|
||||
&element,
|
||||
&mut context,
|
||||
) {
|
||||
return Ok(Some(desc_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub fn query_selector_all(&self, selector: &str) -> Result<Vec<NodeId>, String> {
|
||||
let selector_list = parse_selector(selector)?;
|
||||
let mut caches = selectors::context::SelectorCaches::default();
|
||||
let mut context = MatchingContext::new(
|
||||
MatchingMode::Normal,
|
||||
None,
|
||||
&mut caches,
|
||||
QuirksMode::NoQuirks,
|
||||
NeedsSelectorFlags::No,
|
||||
MatchingForInvalidation::No,
|
||||
);
|
||||
let mut results = Vec::new();
|
||||
|
||||
let doc = self.document();
|
||||
for desc_id in self.descendants(doc) {
|
||||
let is_element = self.with_node(desc_id, |n| n.is_element()).unwrap_or(false);
|
||||
if is_element {
|
||||
let element = DomElement::new(self, desc_id);
|
||||
if selectors::matching::matches_selector_list(
|
||||
&selector_list,
|
||||
&element,
|
||||
&mut context,
|
||||
) {
|
||||
results.push(desc_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::tree_sink::parse_html;
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_tag() {
|
||||
let tree = parse_html("<html><body><h1>Title</h1><p>Text</p></body></html>");
|
||||
let result = tree.query_selector("h1").unwrap();
|
||||
assert!(result.is_some());
|
||||
let node = tree.get_node(result.unwrap()).unwrap();
|
||||
assert_eq!(node.as_element().unwrap().local.as_ref(), "h1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_class() {
|
||||
let tree =
|
||||
parse_html(r#"<div class="foo bar">Content</div><div class="baz">Other</div>"#);
|
||||
let result = tree.query_selector(".foo").unwrap();
|
||||
assert!(result.is_some());
|
||||
let node = tree.get_node(result.unwrap()).unwrap();
|
||||
assert_eq!(node.get_attribute("class"), Some("foo bar"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_id() {
|
||||
let tree = parse_html(r#"<div id="main">Content</div>"#);
|
||||
let result = tree.query_selector("#main").unwrap();
|
||||
assert!(result.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_all() {
|
||||
let tree = parse_html("<ul><li>1</li><li>2</li><li>3</li></ul>");
|
||||
let results = tree.query_selector_all("li").unwrap();
|
||||
assert_eq!(results.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_descendant() {
|
||||
let tree =
|
||||
parse_html(r#"<div id="outer"><div id="inner"><span>Target</span></div></div>"#);
|
||||
let result = tree.query_selector("#outer span").unwrap();
|
||||
assert!(result.is_some());
|
||||
let node = tree.get_node(result.unwrap()).unwrap();
|
||||
assert_eq!(node.as_element().unwrap().local.as_ref(), "span");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_attribute() {
|
||||
let tree = parse_html(
|
||||
r#"<input type="text" name="user"><input type="password" name="pass">"#,
|
||||
);
|
||||
let result = tree.query_selector(r#"input[type="password"]"#).unwrap();
|
||||
assert!(result.is_some());
|
||||
let node = tree.get_node(result.unwrap()).unwrap();
|
||||
assert_eq!(node.get_attribute("name"), Some("pass"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_no_match() {
|
||||
let tree = parse_html("<div>Hello</div>");
|
||||
let result = tree.query_selector("span").unwrap();
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_complex() {
|
||||
let tree = parse_html(
|
||||
r#"<div class="container">
|
||||
<ul class="list">
|
||||
<li class="item active">First</li>
|
||||
<li class="item">Second</li>
|
||||
<li class="item active">Third</li>
|
||||
</ul>
|
||||
</div>"#,
|
||||
);
|
||||
let results = tree.query_selector_all(".list .item.active").unwrap();
|
||||
assert_eq!(results.len(), 2);
|
||||
}
|
||||
}
|
||||
178
crates/obscura-dom/src/serialize.rs
Normal file
178
crates/obscura-dom/src/serialize.rs
Normal file
@@ -0,0 +1,178 @@
|
||||
use crate::tree::{DomTree, NodeData, NodeId};
|
||||
|
||||
impl DomTree {
|
||||
pub fn outer_html(&self, node_id: NodeId) -> String {
|
||||
let mut buf = String::new();
|
||||
self.serialize_node(node_id, true, &mut buf);
|
||||
buf
|
||||
}
|
||||
|
||||
pub fn inner_html(&self, node_id: NodeId) -> String {
|
||||
let mut buf = String::new();
|
||||
self.serialize_children(node_id, &mut buf);
|
||||
buf
|
||||
}
|
||||
|
||||
fn serialize_node(&self, node_id: NodeId, include_self: bool, buf: &mut String) {
|
||||
let node = match self.get_node(node_id) {
|
||||
Some(n) => n,
|
||||
None => return,
|
||||
};
|
||||
|
||||
match &node.data {
|
||||
NodeData::Document => {
|
||||
self.serialize_children(node_id, buf);
|
||||
}
|
||||
NodeData::Doctype { name, .. } => {
|
||||
buf.push_str("<!DOCTYPE ");
|
||||
buf.push_str(name);
|
||||
buf.push('>');
|
||||
}
|
||||
NodeData::Element { name, attrs, .. } => {
|
||||
let tag = name.local.as_ref();
|
||||
if include_self {
|
||||
buf.push('<');
|
||||
buf.push_str(tag);
|
||||
for attr in attrs {
|
||||
buf.push(' ');
|
||||
let attr_name = attr.name.local.as_ref();
|
||||
buf.push_str(attr_name);
|
||||
buf.push_str("=\"");
|
||||
escape_attr(&attr.value, buf);
|
||||
buf.push('"');
|
||||
}
|
||||
buf.push('>');
|
||||
}
|
||||
|
||||
if !is_void_element(tag) {
|
||||
self.serialize_children(node_id, buf);
|
||||
if include_self {
|
||||
buf.push_str("</");
|
||||
buf.push_str(tag);
|
||||
buf.push('>');
|
||||
}
|
||||
}
|
||||
}
|
||||
NodeData::Text { contents } => {
|
||||
let parent_is_raw = node.parent
|
||||
.and_then(|pid| {
|
||||
self.with_node(pid, |p| {
|
||||
p.as_element()
|
||||
.map(|name| is_raw_text_element(name.local.as_ref()))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
})
|
||||
.unwrap_or(false);
|
||||
|
||||
if parent_is_raw {
|
||||
buf.push_str(contents);
|
||||
} else {
|
||||
escape_text(contents, buf);
|
||||
}
|
||||
}
|
||||
NodeData::Comment { contents } => {
|
||||
buf.push_str("<!--");
|
||||
buf.push_str(contents);
|
||||
buf.push_str("-->");
|
||||
}
|
||||
NodeData::ProcessingInstruction { target, data } => {
|
||||
buf.push_str("<?");
|
||||
buf.push_str(target);
|
||||
buf.push(' ');
|
||||
buf.push_str(data);
|
||||
buf.push('>');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn serialize_children(&self, node_id: NodeId, buf: &mut String) {
|
||||
for child_id in self.children(node_id) {
|
||||
self.serialize_node(child_id, true, buf);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn escape_text(s: &str, buf: &mut String) {
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'&' => buf.push_str("&"),
|
||||
'<' => buf.push_str("<"),
|
||||
'>' => buf.push_str(">"),
|
||||
_ => buf.push(c),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn escape_attr(s: &str, buf: &mut String) {
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'&' => buf.push_str("&"),
|
||||
'"' => buf.push_str("""),
|
||||
_ => buf.push(c),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_void_element(tag: &str) -> bool {
|
||||
matches!(
|
||||
tag,
|
||||
"area" | "base" | "br" | "col" | "embed" | "hr" | "img" | "input" | "link" | "meta"
|
||||
| "param" | "source" | "track" | "wbr"
|
||||
)
|
||||
}
|
||||
|
||||
fn is_raw_text_element(tag: &str) -> bool {
|
||||
matches!(tag, "script" | "style" | "textarea" | "title")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::tree_sink::parse_html;
|
||||
|
||||
#[test]
|
||||
fn test_outer_html() {
|
||||
let tree = parse_html(r#"<div id="test"><p>Hello</p></div>"#);
|
||||
let div = tree.get_element_by_id("test").unwrap();
|
||||
let html = tree.outer_html(div);
|
||||
assert!(html.contains(r#"<div id="test">"#));
|
||||
assert!(html.contains("<p>Hello</p>"));
|
||||
assert!(html.contains("</div>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inner_html() {
|
||||
let tree = parse_html(r#"<div id="test"><p>Hello</p><p>World</p></div>"#);
|
||||
let div = tree.get_element_by_id("test").unwrap();
|
||||
let html = tree.inner_html(div);
|
||||
assert!(html.contains("<p>Hello</p>"));
|
||||
assert!(html.contains("<p>World</p>"));
|
||||
assert!(!html.contains("<div"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_serialize_attributes() {
|
||||
let tree = parse_html(r#"<a href="https://example.com" class="link">Click</a>"#);
|
||||
let a = tree.query_selector("a").unwrap().unwrap();
|
||||
let html = tree.outer_html(a);
|
||||
assert!(html.contains("href=\"https://example.com\""));
|
||||
assert!(html.contains("class=\"link\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_serialize_special_chars() {
|
||||
let tree = parse_html("<p>Hello & World <3</p>");
|
||||
let p = tree.query_selector("p").unwrap().unwrap();
|
||||
let html = tree.outer_html(p);
|
||||
assert!(html.contains("&"));
|
||||
assert!(html.contains("<"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_void_elements() {
|
||||
let tree = parse_html(r#"<img src="test.png"><br>"#);
|
||||
let img = tree.query_selector("img").unwrap().unwrap();
|
||||
let html = tree.outer_html(img);
|
||||
assert!(html.contains("<img"));
|
||||
assert!(!html.contains("</img>"));
|
||||
}
|
||||
}
|
||||
697
crates/obscura-dom/src/tree.rs
Normal file
697
crates/obscura-dom/src/tree.rs
Normal file
@@ -0,0 +1,697 @@
|
||||
use html5ever::{LocalName, Namespace, QualName};
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct NodeId(pub(crate) u32);
|
||||
|
||||
impl NodeId {
|
||||
pub fn new(val: u32) -> Self {
|
||||
NodeId(val)
|
||||
}
|
||||
|
||||
pub fn index(self) -> usize {
|
||||
self.0 as usize
|
||||
}
|
||||
|
||||
pub fn raw(self) -> u32 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for NodeId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "NodeId({})", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Attribute {
|
||||
pub name: QualName,
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum NodeData {
|
||||
Document,
|
||||
Doctype {
|
||||
name: String,
|
||||
public_id: String,
|
||||
system_id: String,
|
||||
},
|
||||
Element {
|
||||
name: QualName,
|
||||
attrs: Vec<Attribute>,
|
||||
template_contents: Option<NodeId>,
|
||||
mathml_annotation_xml_integration_point: bool,
|
||||
},
|
||||
Text {
|
||||
contents: String,
|
||||
},
|
||||
Comment {
|
||||
contents: String,
|
||||
},
|
||||
ProcessingInstruction {
|
||||
target: String,
|
||||
data: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Node {
|
||||
pub id: NodeId,
|
||||
pub parent: Option<NodeId>,
|
||||
pub first_child: Option<NodeId>,
|
||||
pub last_child: Option<NodeId>,
|
||||
pub prev_sibling: Option<NodeId>,
|
||||
pub next_sibling: Option<NodeId>,
|
||||
pub data: NodeData,
|
||||
}
|
||||
|
||||
impl Node {
|
||||
pub fn is_document(&self) -> bool {
|
||||
matches!(self.data, NodeData::Document)
|
||||
}
|
||||
|
||||
pub fn is_element(&self) -> bool {
|
||||
matches!(self.data, NodeData::Element { .. })
|
||||
}
|
||||
|
||||
pub fn is_text(&self) -> bool {
|
||||
matches!(self.data, NodeData::Text { .. })
|
||||
}
|
||||
|
||||
pub fn as_element(&self) -> Option<&QualName> {
|
||||
match &self.data {
|
||||
NodeData::Element { name, .. } => Some(name),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn attrs(&self) -> Option<&[Attribute]> {
|
||||
match &self.data {
|
||||
NodeData::Element { attrs, .. } => Some(attrs),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn attrs_mut(&mut self) -> Option<&mut Vec<Attribute>> {
|
||||
match &mut self.data {
|
||||
NodeData::Element { attrs, .. } => Some(attrs),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_attribute(&self, name: &str) -> Option<&str> {
|
||||
self.attrs()?.iter().find_map(|a| {
|
||||
if a.name.local.as_ref() == name {
|
||||
Some(a.value.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn set_attribute(&mut self, name: &str, value: String) {
|
||||
if let NodeData::Element { attrs, .. } = &mut self.data {
|
||||
if let Some(attr) = attrs.iter_mut().find(|a| a.name.local.as_ref() == name) {
|
||||
attr.value = value;
|
||||
} else {
|
||||
attrs.push(Attribute {
|
||||
name: QualName::new(None, Namespace::default(), LocalName::from(name)),
|
||||
value,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn text_content_of_text_node(&self) -> Option<&str> {
|
||||
match &self.data {
|
||||
NodeData::Text { contents } => Some(contents),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DomTree {
|
||||
inner: RefCell<DomTreeInner>,
|
||||
}
|
||||
|
||||
pub(crate) struct DomTreeInner {
|
||||
pub(crate) nodes: Vec<Option<Node>>,
|
||||
pub(crate) free_list: Vec<u32>,
|
||||
pub(crate) document: NodeId,
|
||||
pub(crate) id_index: HashMap<String, NodeId>,
|
||||
}
|
||||
|
||||
impl DomTree {
|
||||
pub fn new() -> Self {
|
||||
let doc_node = Node {
|
||||
id: NodeId(0),
|
||||
parent: None,
|
||||
first_child: None,
|
||||
last_child: None,
|
||||
prev_sibling: None,
|
||||
next_sibling: None,
|
||||
data: NodeData::Document,
|
||||
};
|
||||
DomTree {
|
||||
inner: RefCell::new(DomTreeInner {
|
||||
nodes: vec![Some(doc_node)],
|
||||
free_list: Vec::new(),
|
||||
document: NodeId(0),
|
||||
id_index: HashMap::new(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn document(&self) -> NodeId {
|
||||
self.inner.borrow().document
|
||||
}
|
||||
|
||||
pub(crate) fn borrow_inner(&self) -> std::cell::Ref<'_, DomTreeInner> {
|
||||
self.inner.borrow()
|
||||
}
|
||||
|
||||
pub fn new_node(&self, data: NodeData) -> NodeId {
|
||||
let mut inner = self.inner.borrow_mut();
|
||||
let id = if let Some(slot) = inner.free_list.pop() {
|
||||
NodeId(slot)
|
||||
} else {
|
||||
let idx = inner.nodes.len() as u32;
|
||||
inner.nodes.push(None);
|
||||
NodeId(idx)
|
||||
};
|
||||
|
||||
if let NodeData::Element { ref attrs, .. } = data {
|
||||
if let Some(id_attr) = attrs.iter().find(|a| a.name.local.as_ref() == "id") {
|
||||
inner.id_index.insert(id_attr.value.clone(), id);
|
||||
}
|
||||
}
|
||||
|
||||
inner.nodes[id.index()] = Some(Node {
|
||||
id,
|
||||
parent: None,
|
||||
first_child: None,
|
||||
last_child: None,
|
||||
prev_sibling: None,
|
||||
next_sibling: None,
|
||||
data,
|
||||
});
|
||||
id
|
||||
}
|
||||
|
||||
pub fn get_node(&self, id: NodeId) -> Option<Node> {
|
||||
self.inner.borrow().nodes.get(id.index())?.clone()
|
||||
}
|
||||
|
||||
pub fn with_node<F, R>(&self, id: NodeId, f: F) -> Option<R>
|
||||
where
|
||||
F: FnOnce(&Node) -> R,
|
||||
{
|
||||
let inner = self.inner.borrow();
|
||||
inner.nodes.get(id.index())?.as_ref().map(f)
|
||||
}
|
||||
|
||||
pub fn with_node_mut<F, R>(&self, id: NodeId, f: F) -> Option<R>
|
||||
where
|
||||
F: FnOnce(&mut Node) -> R,
|
||||
{
|
||||
let mut inner = self.inner.borrow_mut();
|
||||
inner.nodes.get_mut(id.index())?.as_mut().map(f)
|
||||
}
|
||||
|
||||
pub fn append_child(&self, parent_id: NodeId, child_id: NodeId) {
|
||||
self.detach(child_id);
|
||||
|
||||
let mut inner = self.inner.borrow_mut();
|
||||
|
||||
let old_last = inner.nodes.get(parent_id.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.last_child);
|
||||
|
||||
if let Some(Some(child)) = inner.nodes.get_mut(child_id.index()) {
|
||||
child.parent = Some(parent_id);
|
||||
child.prev_sibling = old_last;
|
||||
child.next_sibling = None;
|
||||
}
|
||||
|
||||
if let Some(old_last_id) = old_last {
|
||||
if let Some(Some(old_last_node)) = inner.nodes.get_mut(old_last_id.index()) {
|
||||
old_last_node.next_sibling = Some(child_id);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(Some(parent)) = inner.nodes.get_mut(parent_id.index()) {
|
||||
if parent.first_child.is_none() {
|
||||
parent.first_child = Some(child_id);
|
||||
}
|
||||
parent.last_child = Some(child_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn insert_before(&self, existing_id: NodeId, new_sibling_id: NodeId) {
|
||||
let (parent_id, prev_id) = {
|
||||
let inner = self.inner.borrow();
|
||||
let node = match inner.nodes.get(existing_id.index()).and_then(|n| n.as_ref()) {
|
||||
Some(n) => n,
|
||||
None => return,
|
||||
};
|
||||
match node.parent {
|
||||
Some(p) => (p, node.prev_sibling),
|
||||
None => return,
|
||||
}
|
||||
};
|
||||
|
||||
self.detach(new_sibling_id);
|
||||
|
||||
let mut inner = self.inner.borrow_mut();
|
||||
|
||||
if let Some(Some(node)) = inner.nodes.get_mut(new_sibling_id.index()) {
|
||||
node.parent = Some(parent_id);
|
||||
node.prev_sibling = prev_id;
|
||||
node.next_sibling = Some(existing_id);
|
||||
}
|
||||
|
||||
if let Some(Some(node)) = inner.nodes.get_mut(existing_id.index()) {
|
||||
node.prev_sibling = Some(new_sibling_id);
|
||||
}
|
||||
|
||||
if let Some(prev) = prev_id {
|
||||
if let Some(Some(node)) = inner.nodes.get_mut(prev.index()) {
|
||||
node.next_sibling = Some(new_sibling_id);
|
||||
}
|
||||
} else if let Some(Some(parent)) = inner.nodes.get_mut(parent_id.index()) {
|
||||
parent.first_child = Some(new_sibling_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn detach(&self, node_id: NodeId) {
|
||||
let mut inner = self.inner.borrow_mut();
|
||||
|
||||
let (parent_id, prev_id, next_id) = match inner.nodes.get(node_id.index()).and_then(|n| n.as_ref()) {
|
||||
Some(node) => (node.parent, node.prev_sibling, node.next_sibling),
|
||||
None => return,
|
||||
};
|
||||
|
||||
if let Some(prev) = prev_id {
|
||||
if let Some(Some(node)) = inner.nodes.get_mut(prev.index()) {
|
||||
node.next_sibling = next_id;
|
||||
}
|
||||
} else if let Some(parent_id) = parent_id {
|
||||
if let Some(Some(parent)) = inner.nodes.get_mut(parent_id.index()) {
|
||||
parent.first_child = next_id;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(next) = next_id {
|
||||
if let Some(Some(node)) = inner.nodes.get_mut(next.index()) {
|
||||
node.prev_sibling = prev_id;
|
||||
}
|
||||
} else if let Some(parent_id) = parent_id {
|
||||
if let Some(Some(parent)) = inner.nodes.get_mut(parent_id.index()) {
|
||||
parent.last_child = prev_id;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(Some(node)) = inner.nodes.get_mut(node_id.index()) {
|
||||
node.parent = None;
|
||||
node.prev_sibling = None;
|
||||
node.next_sibling = None;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove(&self, node_id: NodeId) {
|
||||
self.detach(node_id);
|
||||
let descendants = self.descendants(node_id);
|
||||
let mut inner = self.inner.borrow_mut();
|
||||
|
||||
let mut ids_to_remove = Vec::new();
|
||||
for &desc_id in &descendants {
|
||||
if let Some(Some(node)) = inner.nodes.get(desc_id.index()) {
|
||||
if let Some(id_val) = node.get_attribute("id") {
|
||||
ids_to_remove.push(id_val.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(Some(node)) = inner.nodes.get(node_id.index()) {
|
||||
if let Some(id_val) = node.get_attribute("id") {
|
||||
ids_to_remove.push(id_val.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
for id_str in ids_to_remove {
|
||||
inner.id_index.remove(&id_str);
|
||||
}
|
||||
|
||||
for desc_id in descendants {
|
||||
inner.nodes[desc_id.index()] = None;
|
||||
inner.free_list.push(desc_id.0);
|
||||
}
|
||||
inner.nodes[node_id.index()] = None;
|
||||
inner.free_list.push(node_id.0);
|
||||
}
|
||||
|
||||
pub fn children(&self, node_id: NodeId) -> Vec<NodeId> {
|
||||
let inner = self.inner.borrow();
|
||||
let mut result = Vec::new();
|
||||
let mut current = inner.nodes.get(node_id.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.first_child);
|
||||
while let Some(child_id) = current {
|
||||
result.push(child_id);
|
||||
current = inner.nodes.get(child_id.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.next_sibling);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn descendants(&self, node_id: NodeId) -> Vec<NodeId> {
|
||||
let inner = self.inner.borrow();
|
||||
let mut result = Vec::new();
|
||||
let mut stack = Vec::new();
|
||||
|
||||
let mut first = inner.nodes.get(node_id.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.first_child);
|
||||
let mut children_to_push = Vec::new();
|
||||
while let Some(child_id) = first {
|
||||
children_to_push.push(child_id);
|
||||
first = inner.nodes.get(child_id.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.next_sibling);
|
||||
}
|
||||
for child_id in children_to_push.into_iter().rev() {
|
||||
stack.push(child_id);
|
||||
}
|
||||
|
||||
while let Some(current) = stack.pop() {
|
||||
result.push(current);
|
||||
|
||||
let mut child = inner.nodes.get(current.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.first_child);
|
||||
let mut children_to_push = Vec::new();
|
||||
while let Some(child_id) = child {
|
||||
children_to_push.push(child_id);
|
||||
child = inner.nodes.get(child_id.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.next_sibling);
|
||||
}
|
||||
for child_id in children_to_push.into_iter().rev() {
|
||||
stack.push(child_id);
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
pub fn ancestors(&self, node_id: NodeId) -> Vec<NodeId> {
|
||||
let inner = self.inner.borrow();
|
||||
let mut result = Vec::new();
|
||||
let mut current = inner.nodes.get(node_id.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.parent);
|
||||
while let Some(parent_id) = current {
|
||||
result.push(parent_id);
|
||||
current = inner.nodes.get(parent_id.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.parent);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn get_element_by_id(&self, id: &str) -> Option<NodeId> {
|
||||
self.inner.borrow().id_index.get(id).copied()
|
||||
}
|
||||
|
||||
pub fn text_content(&self, node_id: NodeId) -> String {
|
||||
let inner = self.inner.borrow();
|
||||
let mut result = String::new();
|
||||
collect_text_inner(&inner, node_id, &mut result);
|
||||
result
|
||||
}
|
||||
|
||||
pub fn append_text(&self, parent_id: NodeId, text: &str) {
|
||||
let last_child_is_text = {
|
||||
let inner = self.inner.borrow();
|
||||
inner.nodes.get(parent_id.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.last_child)
|
||||
.and_then(|lc| inner.nodes.get(lc.index()))
|
||||
.and_then(|n| n.as_ref())
|
||||
.map(|n| n.is_text())
|
||||
.unwrap_or(false)
|
||||
};
|
||||
|
||||
if last_child_is_text {
|
||||
let last_child_id = {
|
||||
let inner = self.inner.borrow();
|
||||
inner.nodes.get(parent_id.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.last_child)
|
||||
.unwrap()
|
||||
};
|
||||
let mut inner = self.inner.borrow_mut();
|
||||
if let Some(Some(node)) = inner.nodes.get_mut(last_child_id.index()) {
|
||||
if let NodeData::Text { contents } = &mut node.data {
|
||||
contents.push_str(text);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let text_id = self.new_node(NodeData::Text {
|
||||
contents: text.to_string(),
|
||||
});
|
||||
self.append_child(parent_id, text_id);
|
||||
}
|
||||
|
||||
pub fn find_body_or_root(&self) -> NodeId {
|
||||
let doc = self.document();
|
||||
for child in self.children(doc) {
|
||||
if let Some(n) = self.get_node(child) {
|
||||
if n.as_element().map(|name| name.local.as_ref() == "html").unwrap_or(false) {
|
||||
for html_child in self.children(child) {
|
||||
if let Some(hc) = self.get_node(html_child) {
|
||||
if hc.as_element().map(|name| name.local.as_ref() == "body").unwrap_or(false) {
|
||||
return html_child;
|
||||
}
|
||||
}
|
||||
}
|
||||
return child;
|
||||
}
|
||||
}
|
||||
}
|
||||
doc
|
||||
}
|
||||
|
||||
pub fn import_children_from(&self, parent_id: NodeId, source: &DomTree, source_node: NodeId) {
|
||||
let source_children = source.children(source_node);
|
||||
for source_child_id in source_children {
|
||||
self.import_node_from(parent_id, source, source_child_id);
|
||||
}
|
||||
}
|
||||
|
||||
fn import_node_from(&self, parent_id: NodeId, source: &DomTree, source_node_id: NodeId) {
|
||||
let node_data = {
|
||||
let source_inner = source.inner.borrow();
|
||||
match source_inner.nodes.get(source_node_id.index()) {
|
||||
Some(Some(node)) => node.data.clone(),
|
||||
_ => return,
|
||||
}
|
||||
};
|
||||
|
||||
let new_id = self.new_node(node_data);
|
||||
self.append_child(parent_id, new_id);
|
||||
|
||||
let children = source.children(source_node_id);
|
||||
for child_id in children {
|
||||
self.import_node_from(new_id, source, child_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.inner.borrow().nodes.iter().filter(|n| n.is_some()).count()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() <= 1
|
||||
}
|
||||
|
||||
pub fn update_id_index(&self, node_id: NodeId, old_id: Option<&str>, new_id: Option<&str>) {
|
||||
let mut inner = self.inner.borrow_mut();
|
||||
if let Some(old) = old_id {
|
||||
inner.id_index.remove(old);
|
||||
}
|
||||
if let Some(new) = new_id {
|
||||
inner.id_index.insert(new.to_string(), node_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_text_inner(inner: &DomTreeInner, node_id: NodeId, buf: &mut String) {
|
||||
if let Some(Some(node)) = inner.nodes.get(node_id.index()) {
|
||||
match &node.data {
|
||||
NodeData::Text { contents } => buf.push_str(contents),
|
||||
_ => {
|
||||
let mut child = node.first_child;
|
||||
while let Some(child_id) = child {
|
||||
collect_text_inner(inner, child_id, buf);
|
||||
child = inner.nodes.get(child_id.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.and_then(|n| n.next_sibling);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DomTree {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_new_tree_has_document() {
|
||||
let tree = DomTree::new();
|
||||
assert_eq!(tree.len(), 1);
|
||||
let node = tree.get_node(tree.document()).unwrap();
|
||||
assert!(node.is_document());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_append_child() {
|
||||
let tree = DomTree::new();
|
||||
let child = tree.new_node(NodeData::Text {
|
||||
contents: "hello".into(),
|
||||
});
|
||||
let doc = tree.document();
|
||||
tree.append_child(doc, child);
|
||||
|
||||
assert_eq!(tree.len(), 2);
|
||||
let doc_node = tree.get_node(doc).unwrap();
|
||||
assert_eq!(doc_node.first_child, Some(child));
|
||||
assert_eq!(doc_node.last_child, Some(child));
|
||||
|
||||
let child_node = tree.get_node(child).unwrap();
|
||||
assert_eq!(child_node.parent, Some(doc));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_children() {
|
||||
let tree = DomTree::new();
|
||||
let doc = tree.document();
|
||||
let c1 = tree.new_node(NodeData::Text { contents: "a".into() });
|
||||
let c2 = tree.new_node(NodeData::Text { contents: "b".into() });
|
||||
let c3 = tree.new_node(NodeData::Text { contents: "c".into() });
|
||||
tree.append_child(doc, c1);
|
||||
tree.append_child(doc, c2);
|
||||
tree.append_child(doc, c3);
|
||||
|
||||
assert_eq!(tree.children(doc), vec![c1, c2, c3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detach() {
|
||||
let tree = DomTree::new();
|
||||
let doc = tree.document();
|
||||
let c1 = tree.new_node(NodeData::Text { contents: "a".into() });
|
||||
let c2 = tree.new_node(NodeData::Text { contents: "b".into() });
|
||||
tree.append_child(doc, c1);
|
||||
tree.append_child(doc, c2);
|
||||
|
||||
tree.detach(c1);
|
||||
assert_eq!(tree.children(doc), vec![c2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_insert_before() {
|
||||
let tree = DomTree::new();
|
||||
let doc = tree.document();
|
||||
let c1 = tree.new_node(NodeData::Text { contents: "a".into() });
|
||||
let c2 = tree.new_node(NodeData::Text { contents: "b".into() });
|
||||
let c3 = tree.new_node(NodeData::Text { contents: "c".into() });
|
||||
tree.append_child(doc, c1);
|
||||
tree.append_child(doc, c3);
|
||||
tree.insert_before(c3, c2);
|
||||
|
||||
assert_eq!(tree.children(doc), vec![c1, c2, c3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_text_content() {
|
||||
let tree = DomTree::new();
|
||||
let doc = tree.document();
|
||||
let div = tree.new_node(NodeData::Element {
|
||||
name: QualName::new(None, ns!(html), local_name!("div")),
|
||||
attrs: vec![],
|
||||
template_contents: None,
|
||||
mathml_annotation_xml_integration_point: false,
|
||||
});
|
||||
tree.append_child(doc, div);
|
||||
|
||||
let t1 = tree.new_node(NodeData::Text { contents: "Hello ".into() });
|
||||
let t2 = tree.new_node(NodeData::Text { contents: "World".into() });
|
||||
tree.append_child(div, t1);
|
||||
tree.append_child(div, t2);
|
||||
|
||||
assert_eq!(tree.text_content(div), "Hello World");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_element_by_id() {
|
||||
let tree = DomTree::new();
|
||||
let doc = tree.document();
|
||||
let div = tree.new_node(NodeData::Element {
|
||||
name: QualName::new(None, ns!(html), local_name!("div")),
|
||||
attrs: vec![Attribute {
|
||||
name: QualName::new(None, Namespace::default(), LocalName::from("id")),
|
||||
value: "main".into(),
|
||||
}],
|
||||
template_contents: None,
|
||||
mathml_annotation_xml_integration_point: false,
|
||||
});
|
||||
tree.append_child(doc, div);
|
||||
|
||||
assert_eq!(tree.get_element_by_id("main"), Some(div));
|
||||
assert_eq!(tree.get_element_by_id("nonexistent"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_append_text_merges() {
|
||||
let tree = DomTree::new();
|
||||
let doc = tree.document();
|
||||
tree.append_text(doc, "Hello ");
|
||||
tree.append_text(doc, "World");
|
||||
|
||||
assert_eq!(tree.children(doc).len(), 1);
|
||||
assert_eq!(tree.text_content(doc), "Hello World");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remove_subtree() {
|
||||
let tree = DomTree::new();
|
||||
let doc = tree.document();
|
||||
let div = tree.new_node(NodeData::Element {
|
||||
name: QualName::new(None, ns!(html), local_name!("div")),
|
||||
attrs: vec![],
|
||||
template_contents: None,
|
||||
mathml_annotation_xml_integration_point: false,
|
||||
});
|
||||
tree.append_child(doc, div);
|
||||
let text = tree.new_node(NodeData::Text { contents: "hi".into() });
|
||||
tree.append_child(div, text);
|
||||
|
||||
assert_eq!(tree.len(), 3);
|
||||
tree.remove(div);
|
||||
assert_eq!(tree.len(), 1);
|
||||
}
|
||||
}
|
||||
323
crates/obscura-dom/src/tree_sink.rs
Normal file
323
crates/obscura-dom/src/tree_sink.rs
Normal file
@@ -0,0 +1,323 @@
|
||||
use std::borrow::Cow;
|
||||
use std::cell::Ref;
|
||||
use std::fmt;
|
||||
|
||||
use html5ever::tendril::StrTendril;
|
||||
use html5ever::tree_builder::{ElemName, ElementFlags, NodeOrText, QuirksMode, TreeSink};
|
||||
use html5ever::{Attribute as HtmlAttribute, LocalName, Namespace, QualName};
|
||||
|
||||
use crate::tree::{Attribute, DomTree, NodeData, NodeId};
|
||||
|
||||
pub struct ObscuraElemName<'a> {
|
||||
_ref: Ref<'a, ()>,
|
||||
name: *const QualName,
|
||||
}
|
||||
|
||||
impl<'a> fmt::Debug for ObscuraElemName<'a> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let name = unsafe { &*self.name };
|
||||
write!(f, "{:?}", name)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> ElemName for ObscuraElemName<'a> {
|
||||
fn ns(&self) -> &Namespace {
|
||||
unsafe { &(*self.name).ns }
|
||||
}
|
||||
|
||||
fn local_name(&self) -> &LocalName {
|
||||
unsafe { &(*self.name).local }
|
||||
}
|
||||
}
|
||||
|
||||
impl TreeSink for DomTree {
|
||||
type Handle = NodeId;
|
||||
type Output = Self;
|
||||
type ElemName<'a> = ObscuraElemName<'a>;
|
||||
|
||||
fn finish(self) -> Self::Output {
|
||||
self
|
||||
}
|
||||
|
||||
fn parse_error(&self, _msg: Cow<'static, str>) {}
|
||||
|
||||
fn get_document(&self) -> NodeId {
|
||||
self.document()
|
||||
}
|
||||
|
||||
fn elem_name<'a>(&'a self, target: &'a NodeId) -> ObscuraElemName<'a> {
|
||||
let borrow = self.borrow_inner();
|
||||
let node = borrow.nodes.get(target.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.expect("elem_name called on invalid node");
|
||||
let name_ptr: *const QualName = match &node.data {
|
||||
NodeData::Element { name, .. } => name as *const QualName,
|
||||
_ => panic!("elem_name called on non-element"),
|
||||
};
|
||||
let ref_guard = Ref::map(borrow, |_| &());
|
||||
ObscuraElemName {
|
||||
_ref: ref_guard,
|
||||
name: name_ptr,
|
||||
}
|
||||
}
|
||||
|
||||
fn create_element(
|
||||
&self,
|
||||
name: QualName,
|
||||
attrs: Vec<HtmlAttribute>,
|
||||
flags: ElementFlags,
|
||||
) -> NodeId {
|
||||
let converted_attrs: Vec<Attribute> = attrs
|
||||
.into_iter()
|
||||
.map(|a| Attribute {
|
||||
name: a.name,
|
||||
value: a.value.to_string(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let id = self.new_node(NodeData::Element {
|
||||
name: name.clone(),
|
||||
attrs: converted_attrs,
|
||||
template_contents: None,
|
||||
mathml_annotation_xml_integration_point: flags.mathml_annotation_xml_integration_point,
|
||||
});
|
||||
|
||||
if flags.template {
|
||||
let template_doc = self.new_node(NodeData::Document);
|
||||
self.with_node_mut(id, |node| {
|
||||
if let NodeData::Element { template_contents, .. } = &mut node.data {
|
||||
*template_contents = Some(template_doc);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
id
|
||||
}
|
||||
|
||||
fn create_comment(&self, text: StrTendril) -> NodeId {
|
||||
self.new_node(NodeData::Comment {
|
||||
contents: text.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn create_pi(&self, target: StrTendril, data: StrTendril) -> NodeId {
|
||||
self.new_node(NodeData::ProcessingInstruction {
|
||||
target: target.to_string(),
|
||||
data: data.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn append(&self, parent: &NodeId, child: NodeOrText<NodeId>) {
|
||||
match child {
|
||||
NodeOrText::AppendNode(node_id) => {
|
||||
self.append_child(*parent, node_id);
|
||||
}
|
||||
NodeOrText::AppendText(text) => {
|
||||
self.append_text(*parent, &text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn append_based_on_parent_node(
|
||||
&self,
|
||||
element: &NodeId,
|
||||
prev_element: &NodeId,
|
||||
child: NodeOrText<NodeId>,
|
||||
) {
|
||||
let has_parent = self.with_node(*element, |n| n.parent.is_some()).unwrap_or(false);
|
||||
if has_parent {
|
||||
self.append_before_sibling(element, child);
|
||||
} else {
|
||||
self.append(prev_element, child);
|
||||
}
|
||||
}
|
||||
|
||||
fn append_doctype_to_document(
|
||||
&self,
|
||||
name: StrTendril,
|
||||
public_id: StrTendril,
|
||||
system_id: StrTendril,
|
||||
) {
|
||||
let doctype = self.new_node(NodeData::Doctype {
|
||||
name: name.to_string(),
|
||||
public_id: public_id.to_string(),
|
||||
system_id: system_id.to_string(),
|
||||
});
|
||||
let doc = self.document();
|
||||
self.append_child(doc, doctype);
|
||||
}
|
||||
|
||||
fn add_attrs_if_missing(&self, target: &NodeId, attrs: Vec<HtmlAttribute>) {
|
||||
self.with_node_mut(*target, |node| {
|
||||
if let NodeData::Element { attrs: existing, .. } = &mut node.data {
|
||||
for attr in attrs {
|
||||
let dominated = existing.iter().any(|a| a.name == attr.name);
|
||||
if !dominated {
|
||||
existing.push(Attribute {
|
||||
name: attr.name,
|
||||
value: attr.value.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn remove_from_parent(&self, target: &NodeId) {
|
||||
self.detach(*target);
|
||||
}
|
||||
|
||||
fn reparent_children(&self, node: &NodeId, new_parent: &NodeId) {
|
||||
let children = self.children(*node);
|
||||
for child_id in children {
|
||||
self.append_child(*new_parent, child_id);
|
||||
}
|
||||
}
|
||||
|
||||
fn append_before_sibling(&self, sibling: &NodeId, child: NodeOrText<NodeId>) {
|
||||
match child {
|
||||
NodeOrText::AppendNode(node_id) => {
|
||||
self.insert_before(*sibling, node_id);
|
||||
}
|
||||
NodeOrText::AppendText(text) => {
|
||||
let prev_text_id = {
|
||||
let node = self.get_node(*sibling);
|
||||
node.and_then(|n| n.prev_sibling).and_then(|prev_id| {
|
||||
let prev = self.get_node(prev_id);
|
||||
prev.and_then(|p| if p.is_text() { Some(prev_id) } else { None })
|
||||
})
|
||||
};
|
||||
|
||||
if let Some(prev_text_id) = prev_text_id {
|
||||
self.with_node_mut(prev_text_id, |node| {
|
||||
if let NodeData::Text { contents } = &mut node.data {
|
||||
contents.push_str(&text);
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let text_id = self.new_node(NodeData::Text {
|
||||
contents: text.to_string(),
|
||||
});
|
||||
self.insert_before(*sibling, text_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_template_contents(&self, target: &NodeId) -> NodeId {
|
||||
self.with_node(*target, |n| match &n.data {
|
||||
NodeData::Element { template_contents, .. } => *template_contents,
|
||||
_ => None,
|
||||
})
|
||||
.flatten()
|
||||
.expect("get_template_contents called on non-template element")
|
||||
}
|
||||
|
||||
fn same_node(&self, x: &NodeId, y: &NodeId) -> bool {
|
||||
x == y
|
||||
}
|
||||
|
||||
fn set_quirks_mode(&self, _mode: QuirksMode) {
|
||||
}
|
||||
|
||||
fn is_mathml_annotation_xml_integration_point(&self, target: &NodeId) -> bool {
|
||||
self.with_node(*target, |n| match &n.data {
|
||||
NodeData::Element { mathml_annotation_xml_integration_point, .. } => {
|
||||
*mathml_annotation_xml_integration_point
|
||||
}
|
||||
_ => false,
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_html(html: &str) -> DomTree {
|
||||
use html5ever::tendril::TendrilSink;
|
||||
use html5ever::{parse_document, ParseOpts};
|
||||
|
||||
let tree = DomTree::new();
|
||||
parse_document(tree, ParseOpts::default())
|
||||
.from_utf8()
|
||||
.one(html.as_bytes())
|
||||
}
|
||||
|
||||
pub fn parse_fragment(html: &str) -> DomTree {
|
||||
use html5ever::tendril::TendrilSink;
|
||||
use html5ever::{parse_fragment, ParseOpts, QualName};
|
||||
|
||||
let context_name = QualName::new(None, ns!(html), local_name!("body"));
|
||||
let tree = DomTree::new();
|
||||
parse_fragment(tree, ParseOpts::default(), context_name, vec![])
|
||||
.from_utf8()
|
||||
.one(html.as_bytes())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_simple_html() {
|
||||
let tree = parse_html("<html><head></head><body><h1>Hello</h1></body></html>");
|
||||
assert!(tree.len() > 3);
|
||||
let text = tree.text_content(tree.document());
|
||||
assert!(text.contains("Hello"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_with_attributes() {
|
||||
let tree = parse_html(r#"<div id="main" class="container">Text</div>"#);
|
||||
let main = tree.get_element_by_id("main");
|
||||
assert!(main.is_some());
|
||||
let node = tree.get_node(main.unwrap()).unwrap();
|
||||
assert_eq!(node.get_attribute("class"), Some("container"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_nested_structure() {
|
||||
let tree = parse_html(
|
||||
r#"<html><body>
|
||||
<div id="outer">
|
||||
<p id="para">Hello <strong>World</strong></p>
|
||||
<ul>
|
||||
<li>Item 1</li>
|
||||
<li>Item 2</li>
|
||||
</ul>
|
||||
</div>
|
||||
</body></html>"#,
|
||||
);
|
||||
|
||||
let outer = tree.get_element_by_id("outer").unwrap();
|
||||
let text = tree.text_content(outer);
|
||||
assert!(text.contains("Hello"));
|
||||
assert!(text.contains("World"));
|
||||
assert!(text.contains("Item 1"));
|
||||
assert!(text.contains("Item 2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_malformed_html() {
|
||||
let tree = parse_html("<div><p>Unclosed paragraph<p>Another<div>Nested wrong</div>");
|
||||
assert!(tree.len() > 3);
|
||||
let text = tree.text_content(tree.document());
|
||||
assert!(text.contains("Unclosed paragraph"));
|
||||
assert!(text.contains("Another"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_doctype() {
|
||||
let tree = parse_html("<!DOCTYPE html><html><body>Hello</body></html>");
|
||||
let first_child = tree.children(tree.document())[0];
|
||||
let node = tree.get_node(first_child).unwrap();
|
||||
assert!(matches!(node.data, NodeData::Doctype { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_fragment() {
|
||||
let tree = parse_fragment("<p>Hello</p><p>World</p>");
|
||||
let text = tree.text_content(tree.document());
|
||||
assert!(text.contains("Hello"));
|
||||
assert!(text.contains("World"));
|
||||
}
|
||||
}
|
||||
22
crates/obscura-js/Cargo.toml
Normal file
22
crates/obscura-js/Cargo.toml
Normal file
@@ -0,0 +1,22 @@
|
||||
[package]
|
||||
name = "obscura-js"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
deno_core = "0.350"
|
||||
|
||||
[dependencies]
|
||||
obscura-dom = { workspace = true }
|
||||
obscura-net = { workspace = true }
|
||||
deno_core = "0.350"
|
||||
tokio = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
url = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
html5ever = { workspace = true }
|
||||
deno_error = "0.6"
|
||||
38
crates/obscura-js/build.rs
Normal file
38
crates/obscura-js/build.rs
Normal file
@@ -0,0 +1,38 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-changed=js/bootstrap.js");
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
|
||||
let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
|
||||
let snapshot_path = out_dir.join("OBSCURA_SNAPSHOT.bin");
|
||||
|
||||
let bootstrap_js = include_str!("js/bootstrap.js");
|
||||
|
||||
let output = deno_core::snapshot::create_snapshot(
|
||||
deno_core::snapshot::CreateSnapshotOptions {
|
||||
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
|
||||
startup_snapshot: None,
|
||||
skip_op_registration: true,
|
||||
extensions: vec![],
|
||||
extension_transpiler: None,
|
||||
with_runtime_cb: Some(Box::new(move |runtime| {
|
||||
runtime
|
||||
.execute_script("<obscura:bootstrap>", bootstrap_js.to_string())
|
||||
.expect("bootstrap.js should not fail during snapshot creation");
|
||||
})),
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("Failed to create V8 snapshot");
|
||||
|
||||
std::fs::write(&snapshot_path, &*output.output).expect("Failed to write snapshot");
|
||||
println!(
|
||||
"cargo:rustc-env=OBSCURA_SNAPSHOT_PATH={}",
|
||||
snapshot_path.display()
|
||||
);
|
||||
|
||||
for file in &output.files_loaded_during_snapshot {
|
||||
println!("cargo:rerun-if-changed={}", file.display());
|
||||
}
|
||||
}
|
||||
2925
crates/obscura-js/js/bootstrap.js
vendored
Normal file
2925
crates/obscura-js/js/bootstrap.js
vendored
Normal file
File diff suppressed because it is too large
Load Diff
6
crates/obscura-js/src/lib.rs
Normal file
6
crates/obscura-js/src/lib.rs
Normal file
@@ -0,0 +1,6 @@
|
||||
#[macro_use]
|
||||
extern crate html5ever;
|
||||
|
||||
pub mod module_loader;
|
||||
pub mod runtime;
|
||||
pub mod ops;
|
||||
93
crates/obscura-js/src/module_loader.rs
Normal file
93
crates/obscura-js/src/module_loader.rs
Normal file
@@ -0,0 +1,93 @@
|
||||
use std::pin::Pin;
|
||||
|
||||
use deno_core::error::ModuleLoaderError;
|
||||
use deno_core::ModuleLoadResponse;
|
||||
use deno_core::ModuleLoader;
|
||||
use deno_core::ModuleSource;
|
||||
use deno_core::ModuleSourceCode;
|
||||
use deno_core::ModuleSpecifier;
|
||||
use deno_core::RequestedModuleType;
|
||||
|
||||
pub struct ObscuraModuleLoader {
|
||||
pub base_url: String,
|
||||
}
|
||||
|
||||
impl ObscuraModuleLoader {
|
||||
pub fn new(base_url: &str) -> Self {
|
||||
ObscuraModuleLoader {
|
||||
base_url: base_url.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn io_err(msg: String) -> ModuleLoaderError {
|
||||
std::io::Error::new(std::io::ErrorKind::Other, msg).into()
|
||||
}
|
||||
|
||||
impl ModuleLoader for ObscuraModuleLoader {
|
||||
fn resolve(
|
||||
&self,
|
||||
specifier: &str,
|
||||
referrer: &str,
|
||||
_kind: deno_core::ResolutionKind,
|
||||
) -> Result<ModuleSpecifier, ModuleLoaderError> {
|
||||
let base = if referrer.is_empty()
|
||||
|| referrer.starts_with('<')
|
||||
|| referrer == "."
|
||||
|| referrer == "about:blank"
|
||||
{
|
||||
&self.base_url
|
||||
} else {
|
||||
referrer
|
||||
};
|
||||
|
||||
deno_core::resolve_import(specifier, base).map_err(|e| e.into())
|
||||
}
|
||||
|
||||
fn load(
|
||||
&self,
|
||||
module_specifier: &ModuleSpecifier,
|
||||
_maybe_referrer: Option<&ModuleSpecifier>,
|
||||
_is_dyn_import: bool,
|
||||
_requested_module_type: RequestedModuleType,
|
||||
) -> ModuleLoadResponse {
|
||||
let url = module_specifier.to_string();
|
||||
|
||||
ModuleLoadResponse::Async(Pin::from(Box::new(async move {
|
||||
let client = reqwest::Client::builder()
|
||||
.build()
|
||||
.map_err(|e| io_err(format!("HTTP client error: {}", e)))?;
|
||||
|
||||
tracing::debug!("Loading ES module: {}", url);
|
||||
|
||||
let resp = client
|
||||
.get(&url)
|
||||
.header("Accept", "application/javascript, text/javascript, */*")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| io_err(format!("Failed to fetch module {}: {}", url, e)))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
return Err(io_err(format!(
|
||||
"Module {} returned HTTP {}",
|
||||
url,
|
||||
resp.status()
|
||||
)));
|
||||
}
|
||||
|
||||
let code = resp.text().await.map_err(|e| {
|
||||
io_err(format!("Failed to read module body {}: {}", url, e))
|
||||
})?;
|
||||
|
||||
let specifier = ModuleSpecifier::parse(&url)
|
||||
.map_err(|e| io_err(format!("Invalid module URL {}: {}", url, e)))?;
|
||||
|
||||
Ok(ModuleSource::new(
|
||||
deno_core::ModuleType::JavaScript,
|
||||
ModuleSourceCode::String(code.into()),
|
||||
&specifier,
|
||||
None,
|
||||
))
|
||||
})))
|
||||
}
|
||||
}
|
||||
654
crates/obscura-js/src/ops.rs
Normal file
654
crates/obscura-js/src/ops.rs
Normal file
@@ -0,0 +1,654 @@
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashMap;
|
||||
use std::rc::Rc;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
use deno_core::op2;
|
||||
use deno_core::OpState;
|
||||
use deno_core::Extension;
|
||||
use obscura_dom::{DomTree, NodeData, NodeId};
|
||||
use obscura_net::{CookieJar, ObscuraHttpClient};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
pub type InterceptCallback = Arc<Mutex<Option<Box<dyn Fn(String, String, String) -> Option<(u16, String, String)> + Send + Sync>>>>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum InterceptResolution {
|
||||
Continue {
|
||||
url: Option<String>,
|
||||
method: Option<String>,
|
||||
headers: Option<HashMap<String, String>>,
|
||||
body: Option<String>,
|
||||
},
|
||||
Fulfill {
|
||||
status: u16,
|
||||
headers: HashMap<String, String>,
|
||||
body: String,
|
||||
},
|
||||
Fail { reason: String },
|
||||
}
|
||||
|
||||
pub struct InterceptedRequest {
|
||||
pub request_id: String,
|
||||
pub url: String,
|
||||
pub method: String,
|
||||
pub headers: HashMap<String, String>,
|
||||
pub resource_type: String,
|
||||
pub resolver: tokio::sync::oneshot::Sender<InterceptResolution>,
|
||||
}
|
||||
|
||||
pub struct ObscuraState {
|
||||
pub dom: Option<DomTree>,
|
||||
pub url: String,
|
||||
pub title: String,
|
||||
pub blocked_urls: Vec<String>,
|
||||
pub cookie_jar: Option<Arc<CookieJar>>,
|
||||
pub http_client: Option<Arc<ObscuraHttpClient>>,
|
||||
pub pending_navigation: Option<(String, String, String)>,
|
||||
pub intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<InterceptedRequest>>,
|
||||
pub intercept_counter: u64,
|
||||
pub intercept_enabled: bool,
|
||||
}
|
||||
|
||||
impl ObscuraState {
|
||||
pub fn new() -> Self {
|
||||
ObscuraState {
|
||||
dom: None,
|
||||
url: "about:blank".to_string(),
|
||||
title: String::new(),
|
||||
blocked_urls: Vec::new(),
|
||||
cookie_jar: None,
|
||||
http_client: None,
|
||||
pending_navigation: None,
|
||||
intercept_tx: None,
|
||||
intercept_counter: 0,
|
||||
intercept_enabled: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub type SharedState = Rc<RefCell<ObscuraState>>;
|
||||
|
||||
#[op2]
|
||||
#[string]
|
||||
fn op_dom(state: &OpState, #[string] cmd: String, #[string] arg1: String, #[string] arg2: String) -> String {
|
||||
let gs = state.borrow::<SharedState>().clone();
|
||||
let gs = gs.borrow();
|
||||
let dom = match &gs.dom {
|
||||
Some(d) => d,
|
||||
None => return "null".to_string(),
|
||||
};
|
||||
|
||||
match cmd.as_str() {
|
||||
"document_node_id" => dom.document().index().to_string(),
|
||||
"document_title" => serde_json::to_string(&gs.title).unwrap_or("\"\"".into()),
|
||||
"document_url" => serde_json::to_string(&gs.url).unwrap_or("\"\"".into()),
|
||||
"document_element" => {
|
||||
for cid in dom.children(dom.document()) {
|
||||
if let Some(n) = dom.get_node(cid) {
|
||||
if n.as_element().map(|name| name.local.as_ref() == "html").unwrap_or(false) {
|
||||
return cid.index().to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
"-1".into()
|
||||
}
|
||||
"document_doctype" => {
|
||||
for cid in dom.children(dom.document()) {
|
||||
if let Some(n) = dom.get_node(cid) {
|
||||
if let obscura_dom::NodeData::Doctype { name, public_id, system_id } = &n.data {
|
||||
return serde_json::json!({
|
||||
"name": name,
|
||||
"publicId": public_id,
|
||||
"systemId": system_id,
|
||||
"nodeId": cid.index(),
|
||||
}).to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
"null".into()
|
||||
}
|
||||
"get_element_by_id" => {
|
||||
dom.get_element_by_id(&arg1).map(|id| id.index().to_string()).unwrap_or("-1".into())
|
||||
}
|
||||
"query_selector" => {
|
||||
dom.query_selector(&arg1).ok().flatten().map(|id| id.index().to_string()).unwrap_or("-1".into())
|
||||
}
|
||||
"query_selector_all" => {
|
||||
let ids: Vec<i32> = dom.query_selector_all(&arg1).ok()
|
||||
.map(|ids| ids.iter().map(|id| id.index() as i32).collect()).unwrap_or_default();
|
||||
serde_json::to_string(&ids).unwrap_or("[]".into())
|
||||
}
|
||||
"node_type" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
dom.get_node(NodeId::new(nid)).map(|n| match &n.data {
|
||||
NodeData::Document => "9", NodeData::Element { .. } => "1", NodeData::Text { .. } => "3",
|
||||
NodeData::Comment { .. } => "8", NodeData::Doctype { .. } => "10", NodeData::ProcessingInstruction { .. } => "7",
|
||||
}).unwrap_or("0").into()
|
||||
}
|
||||
"node_name" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
let name: String = dom.get_node(NodeId::new(nid)).map(|n| match &n.data {
|
||||
NodeData::Document => "#document".to_string(), NodeData::Element { name, .. } => name.local.as_ref().to_ascii_uppercase(),
|
||||
NodeData::Text { .. } => "#text".to_string(), NodeData::Comment { .. } => "#comment".to_string(),
|
||||
NodeData::Doctype { name, .. } => name.clone(), NodeData::ProcessingInstruction { target, .. } => target.clone(),
|
||||
}).unwrap_or_default();
|
||||
serde_json::to_string(&name).unwrap_or("\"\"".into())
|
||||
}
|
||||
"text_content" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
serde_json::to_string(&dom.text_content(NodeId::new(nid))).unwrap_or("\"\"".into())
|
||||
}
|
||||
"parent_node" | "first_child" | "last_child" | "next_sibling" | "prev_sibling" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
dom.get_node(NodeId::new(nid)).and_then(|n| match cmd.as_str() {
|
||||
"parent_node" => n.parent, "first_child" => n.first_child,
|
||||
"last_child" => n.last_child, "next_sibling" => n.next_sibling,
|
||||
"prev_sibling" => n.prev_sibling, _ => None,
|
||||
}).map(|id| id.index().to_string()).unwrap_or("-1".into())
|
||||
}
|
||||
"child_nodes" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
let ids: Vec<i32> = dom.children(NodeId::new(nid)).iter().map(|id| id.index() as i32).collect();
|
||||
serde_json::to_string(&ids).unwrap_or("[]".into())
|
||||
}
|
||||
"tag_name" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
let name = dom.get_node(NodeId::new(nid)).and_then(|n| n.as_element().map(|name| name.local.as_ref().to_ascii_uppercase())).unwrap_or_default();
|
||||
serde_json::to_string(&name).unwrap_or("\"\"".into())
|
||||
}
|
||||
"get_attribute" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
let val = dom.get_node(NodeId::new(nid)).and_then(|n| n.get_attribute(&arg2).map(|s| s.to_string()));
|
||||
serde_json::to_string(&val).unwrap_or("null".into())
|
||||
}
|
||||
"set_attribute" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
let node_id = NodeId::new(nid);
|
||||
if let Some((name, value)) = arg2.split_once('\0') {
|
||||
if name == "id" {
|
||||
let old_id = dom.get_node(node_id).and_then(|n| n.get_attribute("id").map(|s| s.to_string()));
|
||||
dom.with_node_mut(node_id, |n| n.set_attribute(name, value.to_string()));
|
||||
dom.update_id_index(node_id, old_id.as_deref(), Some(value));
|
||||
} else {
|
||||
dom.with_node_mut(node_id, |n| n.set_attribute(name, value.to_string()));
|
||||
}
|
||||
}
|
||||
"true".into()
|
||||
}
|
||||
"inner_html" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
serde_json::to_string(&dom.inner_html(NodeId::new(nid))).unwrap_or("\"\"".into())
|
||||
}
|
||||
"outer_html" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
serde_json::to_string(&dom.outer_html(NodeId::new(nid))).unwrap_or("\"\"".into())
|
||||
}
|
||||
"append_child" => {
|
||||
let parent = arg1.parse::<u32>().unwrap_or(0);
|
||||
let child = arg2.parse::<u32>().unwrap_or(0);
|
||||
dom.append_child(NodeId::new(parent), NodeId::new(child));
|
||||
"true".into()
|
||||
}
|
||||
"remove_child" => {
|
||||
let child = arg1.parse::<u32>().unwrap_or(0);
|
||||
dom.detach(NodeId::new(child));
|
||||
"true".into()
|
||||
}
|
||||
"insert_before" => {
|
||||
let new_node = arg1.parse::<u32>().unwrap_or(0);
|
||||
let ref_node = arg2.parse::<u32>().unwrap_or(0);
|
||||
dom.insert_before(NodeId::new(ref_node), NodeId::new(new_node));
|
||||
"true".into()
|
||||
}
|
||||
"remove_attribute" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
dom.with_node_mut(NodeId::new(nid), |n| {
|
||||
if let NodeData::Element { attrs, .. } = &mut n.data {
|
||||
attrs.retain(|a| a.name.local.as_ref() != arg2.as_str());
|
||||
}
|
||||
});
|
||||
"true".into()
|
||||
}
|
||||
"set_inner_html" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
let target = NodeId::new(nid);
|
||||
let children = dom.children(target);
|
||||
for child in children {
|
||||
dom.detach(child);
|
||||
}
|
||||
if !arg2.is_empty() {
|
||||
let fragment = obscura_dom::parse_fragment(&arg2);
|
||||
let import_root = fragment.find_body_or_root();
|
||||
dom.import_children_from(target, &fragment, import_root);
|
||||
}
|
||||
"true".into()
|
||||
}
|
||||
"set_text_content" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
dom.with_node_mut(NodeId::new(nid), |n| {
|
||||
if let NodeData::Text { contents } = &mut n.data {
|
||||
*contents = arg2.clone();
|
||||
}
|
||||
});
|
||||
"true".into()
|
||||
}
|
||||
"create_document_fragment" => {
|
||||
dom.new_node(NodeData::Document).index().to_string()
|
||||
}
|
||||
"create_element" => {
|
||||
dom.new_node(NodeData::Element {
|
||||
name: html5ever::QualName::new(None, html5ever::ns!(html), html5ever::LocalName::from(arg1.as_str())),
|
||||
attrs: vec![], template_contents: None, mathml_annotation_xml_integration_point: false,
|
||||
}).index().to_string()
|
||||
}
|
||||
"create_text_node" => {
|
||||
dom.new_node(NodeData::Text { contents: arg1.clone() }).index().to_string()
|
||||
}
|
||||
"element_children" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
let ids: Vec<i32> = dom.children(NodeId::new(nid)).iter()
|
||||
.filter(|&&id| dom.get_node(id).map(|n| n.is_element()).unwrap_or(false))
|
||||
.map(|id| id.index() as i32).collect();
|
||||
serde_json::to_string(&ids).unwrap_or("[]".into())
|
||||
}
|
||||
"has_child_nodes" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
dom.get_node(NodeId::new(nid)).map(|n| n.first_child.is_some()).unwrap_or(false).to_string()
|
||||
}
|
||||
"contains" => {
|
||||
let nid = arg1.parse::<u32>().unwrap_or(0);
|
||||
let other = arg2.parse::<u32>().unwrap_or(0);
|
||||
dom.descendants(NodeId::new(nid)).contains(&NodeId::new(other)).to_string()
|
||||
}
|
||||
_ => "null".into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[op2(fast)]
|
||||
fn op_console_msg(state: &OpState, #[string] level: &str, #[string] msg: &str) {
|
||||
let _ = state;
|
||||
match level {
|
||||
"warn" => tracing::warn!(target: "obscura::console", "{}", msg),
|
||||
"error" => tracing::error!(target: "obscura::console", "{}", msg),
|
||||
_ => tracing::info!(target: "obscura::console", "{}", msg),
|
||||
}
|
||||
}
|
||||
|
||||
static SHARED_HTTP_CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
|
||||
|
||||
fn get_shared_client() -> &'static reqwest::Client {
|
||||
SHARED_HTTP_CLIENT.get_or_init(|| {
|
||||
reqwest::Client::builder()
|
||||
.build()
|
||||
.expect("failed to build shared reqwest::Client")
|
||||
})
|
||||
}
|
||||
|
||||
#[op2(async)]
|
||||
#[string]
|
||||
async fn op_fetch_url(
|
||||
state: Rc<RefCell<OpState>>,
|
||||
#[string] url: String,
|
||||
#[string] method: String,
|
||||
#[string] headers_json: String,
|
||||
#[string] body: String,
|
||||
#[string] origin: String,
|
||||
#[string] mode: String,
|
||||
) -> Result<String, deno_error::JsErrorBox> {
|
||||
tracing::debug!("op_fetch_url called: {} {} (intercept check pending)", method, url);
|
||||
|
||||
if let Ok(parsed_url) = url::Url::parse(&url) {
|
||||
if let Err(e) = validate_fetch_url(&parsed_url) {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0,
|
||||
"body": "",
|
||||
"url": url,
|
||||
"headers": {},
|
||||
"blocked": true,
|
||||
"error": e,
|
||||
}).to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let (cookie_jar, in_flight, intercept_tx) = {
|
||||
let state_borrow = state.borrow();
|
||||
let gs = state_borrow.borrow::<SharedState>().clone();
|
||||
let mut gs = gs.borrow_mut();
|
||||
for pattern in &gs.blocked_urls {
|
||||
if pattern == "*" || url.contains(pattern) || glob_match(pattern, &url) {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0,
|
||||
"body": "",
|
||||
"url": url,
|
||||
"headers": {},
|
||||
"blocked": true,
|
||||
}).to_string());
|
||||
}
|
||||
}
|
||||
let jar = gs.cookie_jar.clone();
|
||||
let in_flight = gs.http_client.as_ref().map(|c| c.in_flight.clone());
|
||||
tracing::debug!("op_fetch_url: intercept_enabled={}, has_tx={}", gs.intercept_enabled, gs.intercept_tx.is_some());
|
||||
let itx = if gs.intercept_enabled {
|
||||
gs.intercept_counter += 1;
|
||||
gs.intercept_tx.clone().map(|tx| (tx, format!("intercept-{}", gs.intercept_counter)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
(jar, in_flight, itx)
|
||||
};
|
||||
|
||||
if let Some((tx, request_id)) = intercept_tx {
|
||||
let custom_headers: HashMap<String, String> = serde_json::from_str(&headers_json).unwrap_or_default();
|
||||
let (resolve_tx, resolve_rx) = tokio::sync::oneshot::channel();
|
||||
let intercepted = InterceptedRequest {
|
||||
request_id: request_id.clone(),
|
||||
url: url.clone(),
|
||||
method: method.clone(),
|
||||
headers: custom_headers.clone(),
|
||||
resource_type: "Fetch".to_string(),
|
||||
resolver: resolve_tx,
|
||||
};
|
||||
if tx.send(intercepted).is_ok() {
|
||||
match resolve_rx.await {
|
||||
Ok(InterceptResolution::Fulfill { status, headers: h, body: b }) => {
|
||||
let resp_headers: HashMap<String, String> = h;
|
||||
return Ok(serde_json::json!({
|
||||
"status": status,
|
||||
"body": b,
|
||||
"url": url,
|
||||
"headers": resp_headers,
|
||||
}).to_string());
|
||||
}
|
||||
Ok(InterceptResolution::Fail { reason }) => {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0,
|
||||
"body": "",
|
||||
"url": url,
|
||||
"headers": {},
|
||||
"blocked": true,
|
||||
"error": reason,
|
||||
}).to_string());
|
||||
}
|
||||
Ok(InterceptResolution::Continue { url: new_url, method: new_method, headers: new_headers, body: new_body }) => {
|
||||
tracing::debug!("Interception: continue request {}", url);
|
||||
}
|
||||
Err(_) => {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let client = get_shared_client();
|
||||
|
||||
let request_origin = url::Url::parse(&url)
|
||||
.ok()
|
||||
.map(|u| {
|
||||
let host = u.host_str().unwrap_or("");
|
||||
match u.port() {
|
||||
Some(p) => format!("{}://{}:{}", u.scheme(), host, p),
|
||||
None => format!("{}://{}", u.scheme(), host),
|
||||
}
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let page_origin = if origin.is_empty() { request_origin.clone() } else { origin.clone() };
|
||||
let is_cross_origin = !page_origin.is_empty() && request_origin != page_origin;
|
||||
|
||||
let req_method: reqwest::Method = method.parse().unwrap_or(reqwest::Method::GET);
|
||||
|
||||
let custom_headers: std::collections::HashMap<String, String> =
|
||||
serde_json::from_str(&headers_json).unwrap_or_default();
|
||||
|
||||
let needs_preflight = is_cross_origin
|
||||
&& mode == "cors"
|
||||
&& (req_method != reqwest::Method::GET
|
||||
&& req_method != reqwest::Method::HEAD
|
||||
&& req_method != reqwest::Method::POST
|
||||
|| custom_headers.keys().any(|k| {
|
||||
let kl = k.to_lowercase();
|
||||
kl != "accept" && kl != "accept-language" && kl != "content-language"
|
||||
&& kl != "content-type"
|
||||
}));
|
||||
|
||||
if needs_preflight {
|
||||
let preflight = client
|
||||
.request(reqwest::Method::OPTIONS, &url)
|
||||
.header("Origin", &page_origin)
|
||||
.header("Access-Control-Request-Method", method.as_str())
|
||||
.header(
|
||||
"Access-Control-Request-Headers",
|
||||
custom_headers.keys().cloned().collect::<Vec<_>>().join(", "),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| deno_error::JsErrorBox::generic(format!("CORS preflight failed: {}", e)))?;
|
||||
|
||||
let allowed_origin = preflight
|
||||
.headers()
|
||||
.get("access-control-allow-origin")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("");
|
||||
|
||||
if allowed_origin != "*" && allowed_origin != page_origin {
|
||||
return Err(deno_error::JsErrorBox::generic(format!(
|
||||
"CORS preflight: Origin '{}' not allowed by Access-Control-Allow-Origin '{}'",
|
||||
page_origin, allowed_origin
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
let mut req = client.request(req_method, &url);
|
||||
|
||||
if is_cross_origin {
|
||||
req = req.header("Origin", &page_origin);
|
||||
}
|
||||
|
||||
if !is_cross_origin {
|
||||
if let Some(ref jar) = cookie_jar {
|
||||
if let Ok(parsed_url) = url::Url::parse(&url) {
|
||||
let cookie_header = jar.get_cookie_header(&parsed_url);
|
||||
if !cookie_header.is_empty() {
|
||||
req = req.header("Cookie", &cookie_header);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (k, v) in &custom_headers {
|
||||
req = req.header(k.as_str(), v.as_str());
|
||||
}
|
||||
|
||||
if !body.is_empty() {
|
||||
req = req.body(body);
|
||||
}
|
||||
|
||||
if let Some(ref counter) = in_flight {
|
||||
counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
|
||||
let response = req
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if let Some(ref counter) = in_flight {
|
||||
counter.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
deno_error::JsErrorBox::generic(e.to_string())
|
||||
})?;
|
||||
|
||||
if let Some(ref counter) = in_flight {
|
||||
counter.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
|
||||
let status = response.status().as_u16();
|
||||
|
||||
if let Some(ref jar) = cookie_jar {
|
||||
if let Ok(parsed_url) = url::Url::parse(&url) {
|
||||
for val in response.headers().get_all(reqwest::header::SET_COOKIE) {
|
||||
if let Ok(s) = val.to_str() {
|
||||
jar.set_cookie(s, &parsed_url);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let resp_headers: std::collections::HashMap<String, String> = response
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
|
||||
if is_cross_origin && mode == "cors" {
|
||||
let allowed = resp_headers
|
||||
.get("access-control-allow-origin")
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or("");
|
||||
|
||||
if allowed != "*" && allowed != page_origin {
|
||||
return Ok(serde_json::json!({
|
||||
"status": 0,
|
||||
"body": "",
|
||||
"url": url,
|
||||
"headers": {},
|
||||
"corsBlocked": true,
|
||||
"corsError": format!("CORS error: Origin '{}' not in Access-Control-Allow-Origin '{}'", page_origin, allowed),
|
||||
})
|
||||
.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let resp_body = response
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?;
|
||||
|
||||
tracing::debug!("op_fetch_url completed: {} {} ({} bytes)", method, url, resp_body.len());
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"status": status,
|
||||
"body": resp_body,
|
||||
"url": url,
|
||||
"headers": resp_headers,
|
||||
})
|
||||
.to_string())
|
||||
}
|
||||
|
||||
fn glob_match(pattern: &str, url: &str) -> bool {
|
||||
if pattern == "*" {
|
||||
return true;
|
||||
}
|
||||
if pattern.starts_with('*') && pattern.ends_with('*') {
|
||||
return url.contains(&pattern[1..pattern.len() - 1]);
|
||||
}
|
||||
if pattern.starts_with('*') {
|
||||
return url.ends_with(&pattern[1..]);
|
||||
}
|
||||
if pattern.ends_with('*') {
|
||||
return url.starts_with(&pattern[..pattern.len() - 1]);
|
||||
}
|
||||
url == pattern
|
||||
}
|
||||
|
||||
fn validate_fetch_url(url: &url::Url) -> Result<(), String> {
|
||||
let scheme = url.scheme();
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return Err(format!(
|
||||
"Forbidden URL scheme '{}' - only http and https are allowed",
|
||||
scheme
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(host) = url.host() {
|
||||
match host {
|
||||
url::Host::Ipv4(ip) => {
|
||||
if ip.is_loopback()
|
||||
|| ip.is_private()
|
||||
|| ip.is_link_local()
|
||||
|| ip.is_broadcast()
|
||||
|| ip.is_documentation()
|
||||
{
|
||||
return Err(format!(
|
||||
"Access to private/internal IP address {} is not allowed",
|
||||
ip
|
||||
));
|
||||
}
|
||||
}
|
||||
url::Host::Ipv6(ip) => {
|
||||
if ip.is_loopback() || ip.is_unicast_link_local() {
|
||||
return Err(format!(
|
||||
"Access to private/internal IPv6 address {} is not allowed",
|
||||
ip
|
||||
));
|
||||
}
|
||||
}
|
||||
url::Host::Domain(domain) => {
|
||||
let lower_domain = domain.to_lowercase();
|
||||
if lower_domain == "localhost"
|
||||
|| lower_domain.ends_with(".localhost")
|
||||
|| lower_domain == "127.0.0.1"
|
||||
|| lower_domain == "::1"
|
||||
{
|
||||
return Err(format!(
|
||||
"Access to localhost domain '{}' is not allowed",
|
||||
domain
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[op2]
|
||||
#[string]
|
||||
fn op_get_cookies(state: &OpState) -> String {
|
||||
let gs = state.borrow::<SharedState>().clone();
|
||||
let gs = gs.borrow();
|
||||
let jar = match &gs.cookie_jar {
|
||||
Some(j) => j,
|
||||
None => return String::new(),
|
||||
};
|
||||
let url = match url::Url::parse(&gs.url) {
|
||||
Ok(u) => u,
|
||||
Err(_) => return String::new(),
|
||||
};
|
||||
jar.get_js_visible_cookies(&url)
|
||||
}
|
||||
|
||||
#[op2(fast)]
|
||||
fn op_set_cookie(state: &OpState, #[string] cookie_str: &str) {
|
||||
let gs = state.borrow::<SharedState>().clone();
|
||||
let gs = gs.borrow();
|
||||
let jar = match &gs.cookie_jar {
|
||||
Some(j) => j,
|
||||
None => return,
|
||||
};
|
||||
let url = match url::Url::parse(&gs.url) {
|
||||
Ok(u) => u,
|
||||
Err(_) => return,
|
||||
};
|
||||
jar.set_cookie_from_js(cookie_str, &url);
|
||||
}
|
||||
|
||||
#[op2(fast)]
|
||||
fn op_navigate(state: &OpState, #[string] url: &str, #[string] method: &str, #[string] body: &str) {
|
||||
let gs = state.borrow::<SharedState>().clone();
|
||||
let mut gs = gs.borrow_mut();
|
||||
gs.pending_navigation = Some((url.to_string(), method.to_string(), body.to_string()));
|
||||
}
|
||||
|
||||
pub fn build_extension() -> Extension {
|
||||
Extension {
|
||||
name: "obscura_dom",
|
||||
ops: std::borrow::Cow::Owned(vec![
|
||||
op_dom(),
|
||||
op_console_msg(),
|
||||
op_fetch_url(),
|
||||
op_get_cookies(),
|
||||
op_set_cookie(),
|
||||
op_navigate(),
|
||||
]),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
1322
crates/obscura-js/src/runtime.rs
Normal file
1322
crates/obscura-js/src/runtime.rs
Normal file
File diff suppressed because it is too large
Load Diff
20
crates/obscura-net/Cargo.toml
Normal file
20
crates/obscura-net/Cargo.toml
Normal file
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "obscura-net"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
stealth = ["wreq", "wreq-util"]
|
||||
|
||||
[dependencies]
|
||||
reqwest = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
url = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
async-trait = "0.1"
|
||||
wreq = { version = "6.0.0-rc.28", optional = true, features = ["prefix-symbols"] }
|
||||
wreq-util = { version = "3.0.0-rc.10", optional = true }
|
||||
77
crates/obscura-net/src/blocklist.rs
Normal file
77
crates/obscura-net/src/blocklist.rs
Normal file
@@ -0,0 +1,77 @@
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const PGL_LIST: &str = include_str!("pgl_domains.txt");
|
||||
|
||||
fn blocklist() -> &'static HashSet<&'static str> {
|
||||
static BLOCKLIST: OnceLock<HashSet<&str>> = OnceLock::new();
|
||||
BLOCKLIST.get_or_init(|| {
|
||||
let mut set = HashSet::with_capacity(4000);
|
||||
for line in PGL_LIST.lines() {
|
||||
let domain = line.trim();
|
||||
if !domain.is_empty() && !domain.starts_with('#') {
|
||||
set.insert(domain);
|
||||
}
|
||||
}
|
||||
for domain in EXTRA_DOMAINS {
|
||||
set.insert(*domain);
|
||||
}
|
||||
set
|
||||
})
|
||||
}
|
||||
|
||||
pub fn is_blocked(host: &str) -> bool {
|
||||
let bl = blocklist();
|
||||
|
||||
if bl.contains(host) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let mut domain = host;
|
||||
while let Some(pos) = domain.find('.') {
|
||||
domain = &domain[pos + 1..];
|
||||
if bl.contains(domain) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
static EXTRA_DOMAINS: &[&str] = &[];
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_exact_match() {
|
||||
assert!(is_blocked("google-analytics.com"));
|
||||
assert!(is_blocked("doubleclick.net"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_subdomain_match() {
|
||||
assert!(is_blocked("www.google-analytics.com"));
|
||||
assert!(is_blocked("ssl.google-analytics.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_blocked() {
|
||||
assert!(!is_blocked("google.com"));
|
||||
assert!(!is_blocked("example.com"));
|
||||
assert!(!is_blocked("github.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pgl_domains() {
|
||||
assert!(is_blocked("adnxs.com"));
|
||||
assert!(is_blocked("criteo.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blocklist_size() {
|
||||
assert!(blocklist().len() > 3500);
|
||||
}
|
||||
}
|
||||
415
crates/obscura-net/src/client.rs
Normal file
415
crates/obscura-net/src/client.rs
Normal file
@@ -0,0 +1,415 @@
|
||||
use std::collections::HashMap;
|
||||
use std::net::IpAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::header::{HeaderMap, HeaderName, HeaderValue, USER_AGENT};
|
||||
use reqwest::redirect::Policy;
|
||||
use reqwest::{Client, Method};
|
||||
use tokio::sync::RwLock;
|
||||
use url::Url;
|
||||
|
||||
use crate::cookies::CookieJar;
|
||||
use crate::interceptor::{InterceptAction, RequestInterceptor};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Response {
|
||||
pub url: Url,
|
||||
pub status: u16,
|
||||
pub headers: HashMap<String, String>,
|
||||
pub body: Vec<u8>,
|
||||
pub redirected_from: Vec<Url>,
|
||||
}
|
||||
|
||||
impl Response {
|
||||
pub fn text(&self) -> Result<String, std::string::FromUtf8Error> {
|
||||
String::from_utf8(self.body.clone())
|
||||
}
|
||||
|
||||
pub fn header(&self, name: &str) -> Option<&str> {
|
||||
self.headers.get(&name.to_lowercase()).map(|s| s.as_str())
|
||||
}
|
||||
|
||||
pub fn content_type(&self) -> Option<&str> {
|
||||
self.header("content-type")
|
||||
}
|
||||
|
||||
pub fn is_html(&self) -> bool {
|
||||
self.content_type()
|
||||
.map(|ct| ct.contains("text/html"))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RequestInfo {
|
||||
pub url: Url,
|
||||
pub method: String,
|
||||
pub headers: HashMap<String, String>,
|
||||
pub resource_type: ResourceType,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ResourceType {
|
||||
Document,
|
||||
Script,
|
||||
Stylesheet,
|
||||
Image,
|
||||
Font,
|
||||
Xhr,
|
||||
Fetch,
|
||||
Other,
|
||||
}
|
||||
|
||||
pub type RequestCallback = Arc<dyn Fn(&RequestInfo) + Send + Sync>;
|
||||
pub type ResponseCallback = Arc<dyn Fn(&RequestInfo, &Response) + Send + Sync>;
|
||||
|
||||
fn validate_url(url: &Url) -> Result<(), ObscuraNetError> {
|
||||
let scheme = url.scheme();
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Forbidden URL scheme '{}' - only http and https are allowed",
|
||||
scheme
|
||||
)));
|
||||
}
|
||||
|
||||
if let Some(host) = url.host() {
|
||||
match host {
|
||||
url::Host::Ipv4(ip) => {
|
||||
if ip.is_loopback()
|
||||
|| ip.is_private()
|
||||
|| ip.is_link_local()
|
||||
|| ip.is_broadcast()
|
||||
|| ip.is_documentation()
|
||||
{
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Access to private/internal IP address {} is not allowed",
|
||||
ip
|
||||
)));
|
||||
}
|
||||
}
|
||||
url::Host::Ipv6(ip) => {
|
||||
if ip.is_loopback() || ip.is_unicast_link_local() {
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Access to private/internal IPv6 address {} is not allowed",
|
||||
ip
|
||||
)));
|
||||
}
|
||||
}
|
||||
url::Host::Domain(domain) => {
|
||||
let lower_domain = domain.to_lowercase();
|
||||
if lower_domain == "localhost"
|
||||
|| lower_domain.ends_with(".localhost")
|
||||
|| lower_domain == "127.0.0.1"
|
||||
|| lower_domain == "::1"
|
||||
{
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Access to localhost domain '{}' is not allowed",
|
||||
domain
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct ObscuraHttpClient {
|
||||
client: tokio::sync::OnceCell<Client>,
|
||||
proxy_url: Option<String>,
|
||||
pub cookie_jar: Arc<CookieJar>,
|
||||
pub user_agent: RwLock<String>,
|
||||
pub extra_headers: RwLock<HashMap<String, String>>,
|
||||
pub interceptor: RwLock<Option<Box<dyn RequestInterceptor + Send + Sync>>>,
|
||||
pub on_request: RwLock<Vec<RequestCallback>>,
|
||||
pub on_response: RwLock<Vec<ResponseCallback>>,
|
||||
pub timeout: Duration,
|
||||
pub in_flight: Arc<std::sync::atomic::AtomicU32>,
|
||||
pub block_trackers: bool,
|
||||
}
|
||||
|
||||
impl ObscuraHttpClient {
|
||||
pub fn new() -> Self {
|
||||
Self::with_cookie_jar(Arc::new(CookieJar::new()))
|
||||
}
|
||||
|
||||
pub fn with_cookie_jar(cookie_jar: Arc<CookieJar>) -> Self {
|
||||
Self::with_options(cookie_jar, None)
|
||||
}
|
||||
|
||||
pub fn with_options(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
|
||||
ObscuraHttpClient {
|
||||
client: tokio::sync::OnceCell::new(),
|
||||
proxy_url: proxy_url.map(|s| s.to_string()),
|
||||
cookie_jar,
|
||||
user_agent: RwLock::new(
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36".to_string(),
|
||||
),
|
||||
extra_headers: RwLock::new(HashMap::new()),
|
||||
interceptor: RwLock::new(None),
|
||||
on_request: RwLock::new(Vec::new()),
|
||||
on_response: RwLock::new(Vec::new()),
|
||||
in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),
|
||||
timeout: Duration::from_secs(30),
|
||||
block_trackers: false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_client(&self) -> &Client {
|
||||
self.client.get_or_init(|| async {
|
||||
let mut builder = Client::builder()
|
||||
.redirect(Policy::none())
|
||||
.timeout(Duration::from_secs(30))
|
||||
.danger_accept_invalid_certs(false)
|
||||
;
|
||||
|
||||
if let Some(ref proxy) = self.proxy_url {
|
||||
if let Ok(p) = reqwest::Proxy::all(proxy.as_str()) {
|
||||
builder = builder.proxy(p);
|
||||
}
|
||||
}
|
||||
|
||||
builder.build().expect("failed to build HTTP client")
|
||||
}).await
|
||||
}
|
||||
|
||||
pub async fn fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
||||
self.fetch_with_method(Method::GET, url, None).await
|
||||
}
|
||||
|
||||
pub async fn post_form(&self, url: &Url, body: &str) -> Result<Response, ObscuraNetError> {
|
||||
self.fetch_with_method(Method::POST, url, Some(body.as_bytes().to_vec())).await
|
||||
}
|
||||
|
||||
pub async fn fetch_with_method(
|
||||
&self,
|
||||
initial_method: Method,
|
||||
url: &Url,
|
||||
initial_body: Option<Vec<u8>>,
|
||||
) -> Result<Response, ObscuraNetError> {
|
||||
validate_url(url)?;
|
||||
|
||||
let mut method = initial_method;
|
||||
let mut body = initial_body;
|
||||
if self.block_trackers {
|
||||
if let Some(host) = url.host_str() {
|
||||
if crate::blocklist::is_blocked(host) {
|
||||
tracing::debug!("Blocked tracker: {}", url);
|
||||
return Ok(Response {
|
||||
status: 0,
|
||||
url: url.clone(),
|
||||
headers: HashMap::new(),
|
||||
body: Vec::new(),
|
||||
redirected_from: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut current_url = url.clone();
|
||||
let mut redirects = Vec::new();
|
||||
let max_redirects = 20;
|
||||
|
||||
for _redirect_count in 0..max_redirects {
|
||||
let request_info = RequestInfo {
|
||||
url: current_url.clone(),
|
||||
method: method.to_string(),
|
||||
headers: self.extra_headers.read().await.clone(),
|
||||
resource_type: ResourceType::Document,
|
||||
};
|
||||
|
||||
if let Some(interceptor) = self.interceptor.read().await.as_ref() {
|
||||
match interceptor.intercept(&request_info).await {
|
||||
InterceptAction::Continue => {}
|
||||
InterceptAction::Block => {
|
||||
return Err(ObscuraNetError::Blocked(current_url.to_string()));
|
||||
}
|
||||
InterceptAction::Fulfill(response) => {
|
||||
return Ok(response);
|
||||
}
|
||||
InterceptAction::ModifyHeaders(headers) => {
|
||||
let mut extra = self.extra_headers.write().await;
|
||||
extra.extend(headers);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for cb in self.on_request.read().await.iter() {
|
||||
cb(&request_info);
|
||||
}
|
||||
|
||||
let ua = self.user_agent.read().await.clone();
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap_or_else(|_| {
|
||||
HeaderValue::from_static("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36")
|
||||
}));
|
||||
headers.insert(
|
||||
reqwest::header::ACCEPT,
|
||||
HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"),
|
||||
);
|
||||
headers.insert(
|
||||
reqwest::header::ACCEPT_LANGUAGE,
|
||||
HeaderValue::from_static("en-US,en;q=0.9"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua"),
|
||||
HeaderValue::from_static("\"Chromium\";v=\"145\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"145\""),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua-mobile"),
|
||||
HeaderValue::from_static("?0"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua-platform"),
|
||||
HeaderValue::from_static("\"Linux\""),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-dest"),
|
||||
HeaderValue::from_static("document"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-mode"),
|
||||
HeaderValue::from_static("navigate"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-site"),
|
||||
HeaderValue::from_static("none"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-fetch-user"),
|
||||
HeaderValue::from_static("?1"),
|
||||
);
|
||||
headers.insert(
|
||||
HeaderName::from_static("upgrade-insecure-requests"),
|
||||
HeaderValue::from_static("1"),
|
||||
);
|
||||
|
||||
let cookie_header = self.cookie_jar.get_cookie_header(¤t_url);
|
||||
if !cookie_header.is_empty() {
|
||||
if let Ok(val) = HeaderValue::from_str(&cookie_header) {
|
||||
headers.insert(reqwest::header::COOKIE, val);
|
||||
}
|
||||
}
|
||||
|
||||
for (k, v) in self.extra_headers.read().await.iter() {
|
||||
if let (Ok(name), Ok(val)) = (
|
||||
HeaderName::from_bytes(k.as_bytes()),
|
||||
HeaderValue::from_str(v),
|
||||
) {
|
||||
headers.insert(name, val);
|
||||
}
|
||||
}
|
||||
|
||||
let mut req_builder = self.get_client().await.request(method.clone(), current_url.as_str())
|
||||
.headers(headers);
|
||||
|
||||
if let Some(ref b) = body {
|
||||
if method == Method::POST {
|
||||
req_builder = req_builder.header(
|
||||
reqwest::header::CONTENT_TYPE,
|
||||
"application/x-www-form-urlencoded",
|
||||
);
|
||||
}
|
||||
req_builder = req_builder.body(b.clone());
|
||||
}
|
||||
|
||||
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let resp = req_builder.send().await.map_err(|e| {
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
ObscuraNetError::Network(format!("{}: {}", current_url, e))
|
||||
})?;
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let status = resp.status();
|
||||
|
||||
for val in resp.headers().get_all(reqwest::header::SET_COOKIE) {
|
||||
if let Ok(s) = val.to_str() {
|
||||
self.cookie_jar.set_cookie(s, ¤t_url);
|
||||
}
|
||||
}
|
||||
|
||||
let response_headers: HashMap<String, String> = resp
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
|
||||
if status.is_redirection() {
|
||||
if let Some(location) = resp.headers().get(reqwest::header::LOCATION) {
|
||||
let location_str = location.to_str().map_err(|_| {
|
||||
ObscuraNetError::Network("Invalid redirect Location header".into())
|
||||
})?;
|
||||
let next_url = current_url.join(location_str).map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Invalid redirect URL: {}", e))
|
||||
})?;
|
||||
validate_url(&next_url)?;
|
||||
redirects.push(current_url.clone());
|
||||
current_url = next_url;
|
||||
if status == reqwest::StatusCode::MOVED_PERMANENTLY
|
||||
|| status == reqwest::StatusCode::FOUND
|
||||
|| status == reqwest::StatusCode::SEE_OTHER
|
||||
{
|
||||
method = Method::GET;
|
||||
body = None;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let body_bytes = resp.bytes().await.map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Failed to read body: {}", e))
|
||||
})?.to_vec();
|
||||
|
||||
let response = Response {
|
||||
url: current_url,
|
||||
status: status.as_u16(),
|
||||
headers: response_headers,
|
||||
body: body_bytes,
|
||||
redirected_from: redirects,
|
||||
};
|
||||
|
||||
for cb in self.on_response.read().await.iter() {
|
||||
cb(&request_info, &response);
|
||||
}
|
||||
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
Err(ObscuraNetError::TooManyRedirects(current_url.to_string()))
|
||||
}
|
||||
|
||||
pub async fn set_user_agent(&self, ua: &str) {
|
||||
*self.user_agent.write().await = ua.to_string();
|
||||
}
|
||||
|
||||
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
|
||||
*self.extra_headers.write().await = headers;
|
||||
}
|
||||
|
||||
pub fn active_requests(&self) -> u32 {
|
||||
self.in_flight.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn is_network_idle(&self) -> bool {
|
||||
self.active_requests() == 0
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ObscuraHttpClient {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ObscuraNetError {
|
||||
#[error("Network error: {0}")]
|
||||
Network(String),
|
||||
|
||||
#[error("Too many redirects: {0}")]
|
||||
TooManyRedirects(String),
|
||||
|
||||
#[error("Request blocked: {0}")]
|
||||
Blocked(String),
|
||||
}
|
||||
486
crates/obscura-net/src/cookies.rs
Normal file
486
crates/obscura-net/src/cookies.rs
Normal file
@@ -0,0 +1,486 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::RwLock;
|
||||
use url::Url;
|
||||
|
||||
pub struct CookieJar {
|
||||
cookies: RwLock<HashMap<String, HashMap<String, CookieEntry>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct CookieEntry {
|
||||
name: String,
|
||||
value: String,
|
||||
path: String,
|
||||
domain: String,
|
||||
secure: bool,
|
||||
http_only: bool,
|
||||
expires: Option<u64>,
|
||||
same_site: String,
|
||||
}
|
||||
|
||||
impl CookieJar {
|
||||
pub fn new() -> Self {
|
||||
CookieJar {
|
||||
cookies: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_cookie(&self, set_cookie_str: &str, url: &Url) {
|
||||
let parts: Vec<&str> = set_cookie_str.splitn(2, ';').collect();
|
||||
let name_value = parts[0].trim();
|
||||
let (name, value) = match name_value.split_once('=') {
|
||||
Some((n, v)) => (n.trim().to_string(), v.trim().to_string()),
|
||||
None => return,
|
||||
};
|
||||
|
||||
let mut domain = url.host_str().unwrap_or("").to_lowercase();
|
||||
let mut path = url.path().to_string();
|
||||
let mut secure = false;
|
||||
let mut http_only = false;
|
||||
let mut expires: Option<u64> = None;
|
||||
let mut same_site = "Lax".to_string();
|
||||
|
||||
if parts.len() > 1 {
|
||||
for attr in parts[1].split(';') {
|
||||
let attr = attr.trim();
|
||||
if let Some((key, val)) = attr.split_once('=') {
|
||||
match key.trim().to_lowercase().as_str() {
|
||||
"domain" => {
|
||||
domain = val.trim().trim_start_matches('.').to_lowercase();
|
||||
}
|
||||
"path" => {
|
||||
path = val.trim().to_string();
|
||||
}
|
||||
"expires" => {
|
||||
if let Ok(ts) = parse_http_date(val.trim()) {
|
||||
expires = Some(ts);
|
||||
}
|
||||
}
|
||||
"max-age" => {
|
||||
if let Ok(secs) = val.trim().parse::<i64>() {
|
||||
if secs <= 0 {
|
||||
expires = Some(0);
|
||||
} else {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
expires = Some(now + secs as u64);
|
||||
}
|
||||
}
|
||||
}
|
||||
"samesite" => {
|
||||
same_site = val.trim().to_string();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
} else {
|
||||
match attr.to_lowercase().as_str() {
|
||||
"secure" => secure = true,
|
||||
"httponly" => http_only = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(exp) = expires {
|
||||
if exp == 0 {
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
if let Some(domain_cookies) = cookies.get_mut(&domain) {
|
||||
domain_cookies.remove(&name);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
if exp < now {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let entry = CookieEntry {
|
||||
name: name.clone(),
|
||||
value,
|
||||
path,
|
||||
domain: domain.clone(),
|
||||
secure,
|
||||
http_only,
|
||||
expires,
|
||||
same_site,
|
||||
};
|
||||
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
cookies.entry(domain).or_default().insert(name, entry);
|
||||
}
|
||||
|
||||
pub fn get_cookie_header(&self, url: &Url) -> String {
|
||||
let host = url.host_str().unwrap_or("");
|
||||
let path = url.path();
|
||||
let is_secure = url.scheme() == "https";
|
||||
let cookies = self.cookies.read().unwrap();
|
||||
|
||||
let mut matching: Vec<String> = Vec::new();
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
for (domain, domain_cookies) in cookies.iter() {
|
||||
if !domain_matches(host, domain) {
|
||||
continue;
|
||||
}
|
||||
for entry in domain_cookies.values() {
|
||||
if let Some(exp) = entry.expires {
|
||||
if exp < now {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if entry.secure && !is_secure {
|
||||
continue;
|
||||
}
|
||||
if !path.starts_with(&entry.path) {
|
||||
continue;
|
||||
}
|
||||
matching.push(format!("{}={}", entry.name, entry.value));
|
||||
}
|
||||
}
|
||||
|
||||
matching.join("; ")
|
||||
}
|
||||
|
||||
pub fn get_all_cookies(&self) -> Vec<CookieInfo> {
|
||||
let cookies = self.cookies.read().unwrap();
|
||||
let mut result = Vec::new();
|
||||
for domain_cookies in cookies.values() {
|
||||
for entry in domain_cookies.values() {
|
||||
result.push(CookieInfo {
|
||||
name: entry.name.clone(),
|
||||
value: entry.value.clone(),
|
||||
domain: entry.domain.clone(),
|
||||
path: entry.path.clone(),
|
||||
secure: entry.secure,
|
||||
http_only: entry.http_only,
|
||||
});
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn set_cookies_from_cdp(&self, cookies: Vec<CookieInfo>) {
|
||||
let mut jar = self.cookies.write().unwrap();
|
||||
for cookie in cookies {
|
||||
let entry = CookieEntry {
|
||||
name: cookie.name.clone(),
|
||||
value: cookie.value,
|
||||
path: cookie.path,
|
||||
domain: cookie.domain.clone(),
|
||||
secure: cookie.secure,
|
||||
http_only: cookie.http_only,
|
||||
expires: None,
|
||||
same_site: "Lax".to_string(),
|
||||
};
|
||||
jar.entry(cookie.domain).or_default().insert(cookie.name, entry);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_js_visible_cookies(&self, url: &Url) -> String {
|
||||
let host = url.host_str().unwrap_or("");
|
||||
let path = url.path();
|
||||
let is_secure = url.scheme() == "https";
|
||||
let cookies = self.cookies.read().unwrap();
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
let mut matching: Vec<String> = Vec::new();
|
||||
|
||||
for (domain, domain_cookies) in cookies.iter() {
|
||||
if !domain_matches(host, domain) {
|
||||
continue;
|
||||
}
|
||||
for entry in domain_cookies.values() {
|
||||
if entry.http_only {
|
||||
continue;
|
||||
}
|
||||
if let Some(exp) = entry.expires {
|
||||
if exp < now {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if entry.secure && !is_secure {
|
||||
continue;
|
||||
}
|
||||
if !path.starts_with(&entry.path) {
|
||||
continue;
|
||||
}
|
||||
matching.push(format!("{}={}", entry.name, entry.value));
|
||||
}
|
||||
}
|
||||
|
||||
matching.join("; ")
|
||||
}
|
||||
|
||||
pub fn set_cookie_from_js(&self, cookie_str: &str, url: &Url) {
|
||||
let parts: Vec<&str> = cookie_str.splitn(2, ';').collect();
|
||||
let name_value = parts[0].trim();
|
||||
let (name, value) = match name_value.split_once('=') {
|
||||
Some((n, v)) => (n.trim().to_string(), v.trim().to_string()),
|
||||
None => return,
|
||||
};
|
||||
|
||||
let mut domain = url.host_str().unwrap_or("").to_lowercase();
|
||||
let mut path = url.path().to_string();
|
||||
let mut secure = false;
|
||||
let mut expires: Option<u64> = None;
|
||||
let mut same_site = "Lax".to_string();
|
||||
|
||||
if parts.len() > 1 {
|
||||
for attr in parts[1].split(';') {
|
||||
let attr = attr.trim();
|
||||
if let Some((key, val)) = attr.split_once('=') {
|
||||
match key.trim().to_lowercase().as_str() {
|
||||
"domain" => {
|
||||
domain = val.trim().trim_start_matches('.').to_lowercase();
|
||||
}
|
||||
"path" => {
|
||||
path = val.trim().to_string();
|
||||
}
|
||||
"expires" => {
|
||||
if let Ok(ts) = parse_http_date(val.trim()) {
|
||||
expires = Some(ts);
|
||||
}
|
||||
}
|
||||
"max-age" => {
|
||||
if let Ok(secs) = val.trim().parse::<i64>() {
|
||||
if secs <= 0 {
|
||||
expires = Some(0);
|
||||
} else {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
expires = Some(now + secs as u64);
|
||||
}
|
||||
}
|
||||
}
|
||||
"samesite" => {
|
||||
same_site = val.trim().to_string();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
} else {
|
||||
match attr.to_lowercase().as_str() {
|
||||
"secure" => secure = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(exp) = expires {
|
||||
if exp == 0 {
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
if let Some(domain_cookies) = cookies.get_mut(&domain) {
|
||||
domain_cookies.remove(&name);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
if exp < now {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let entry = CookieEntry {
|
||||
name: name.clone(),
|
||||
value,
|
||||
path,
|
||||
domain: domain.clone(),
|
||||
secure,
|
||||
http_only: false,
|
||||
expires,
|
||||
same_site,
|
||||
};
|
||||
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
cookies.entry(domain).or_default().insert(name, entry);
|
||||
}
|
||||
|
||||
pub fn delete_cookie(&self, name: &str, domain: &str) {
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
if domain.is_empty() {
|
||||
for domain_cookies in cookies.values_mut() {
|
||||
domain_cookies.remove(name);
|
||||
}
|
||||
} else {
|
||||
let domains_to_try = [
|
||||
domain.to_string(),
|
||||
format!(".{}", domain.trim_start_matches('.')),
|
||||
domain.trim_start_matches('.').to_string(),
|
||||
];
|
||||
for d in &domains_to_try {
|
||||
if let Some(domain_cookies) = cookies.get_mut(d.as_str()) {
|
||||
domain_cookies.remove(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear(&self) {
|
||||
self.cookies.write().unwrap().clear();
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CookieJar {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct CookieInfo {
|
||||
pub name: String,
|
||||
pub value: String,
|
||||
pub domain: String,
|
||||
pub path: String,
|
||||
pub secure: bool,
|
||||
#[serde(rename = "httpOnly")]
|
||||
pub http_only: bool,
|
||||
}
|
||||
|
||||
fn parse_http_date(s: &str) -> Result<u64, ()> {
|
||||
let months = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"];
|
||||
|
||||
let s = s.replace('-', " ");
|
||||
let parts: Vec<&str> = s.split_whitespace().collect();
|
||||
|
||||
if parts.len() < 5 { return Err(()); }
|
||||
|
||||
let day: u64 = parts[1].parse().map_err(|_| ())?;
|
||||
let month = months.iter().position(|m| parts[2].to_lowercase().starts_with(m))
|
||||
.ok_or(())? as u64 + 1;
|
||||
let year: u64 = parts[3].parse().map_err(|_| ())?;
|
||||
|
||||
let time_parts: Vec<&str> = parts[4].split(':').collect();
|
||||
let hour: u64 = time_parts.first().and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||
let minute: u64 = time_parts.get(1).and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||
let second: u64 = time_parts.get(2).and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||
|
||||
let mut days_total: u64 = 0;
|
||||
for y in 1970..year {
|
||||
days_total += if y % 4 == 0 && (y % 100 != 0 || y % 400 == 0) { 366 } else { 365 };
|
||||
}
|
||||
let days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
||||
let is_leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0);
|
||||
for m in 1..month {
|
||||
days_total += days_in_month[m as usize] + if m == 2 && is_leap { 1 } else { 0 };
|
||||
}
|
||||
days_total += day - 1;
|
||||
|
||||
Ok(days_total * 86400 + hour * 3600 + minute * 60 + second)
|
||||
}
|
||||
|
||||
fn domain_matches(host: &str, domain: &str) -> bool {
|
||||
let host = host.to_lowercase();
|
||||
let domain = domain.to_lowercase();
|
||||
host == domain || host.ends_with(&format!(".{}", domain))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_set_and_get_cookie() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/path").unwrap();
|
||||
jar.set_cookie("session=abc123; Path=/; Secure; HttpOnly", &url);
|
||||
|
||||
let header = jar.get_cookie_header(&url);
|
||||
assert!(header.contains("session=abc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cookie_domain_matching() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://www.example.com/").unwrap();
|
||||
jar.set_cookie("token=xyz; Domain=example.com", &url);
|
||||
|
||||
let header = jar.get_cookie_header(&url);
|
||||
assert!(header.contains("token=xyz"));
|
||||
|
||||
let sub_url = Url::parse("https://api.example.com/").unwrap();
|
||||
let header2 = jar.get_cookie_header(&sub_url);
|
||||
assert!(header2.contains("token=xyz"));
|
||||
|
||||
let other_url = Url::parse("https://other.com/").unwrap();
|
||||
let header3 = jar.get_cookie_header(&other_url);
|
||||
assert!(header3.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_secure_cookie_not_sent_over_http() {
|
||||
let jar = CookieJar::new();
|
||||
let https_url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("secure_token=secret; Secure", &https_url);
|
||||
|
||||
let http_url = Url::parse("http://example.com/").unwrap();
|
||||
let header = jar.get_cookie_header(&http_url);
|
||||
assert!(header.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_age_zero_deletes_cookie() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("session=abc", &url);
|
||||
assert!(jar.get_cookie_header(&url).contains("session=abc"));
|
||||
|
||||
jar.set_cookie("session=abc; Max-Age=0", &url);
|
||||
assert!(jar.get_cookie_header(&url).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_age_sets_expiry() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("token=xyz; Max-Age=3600", &url);
|
||||
assert!(jar.get_cookie_header(&url).contains("token=xyz"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_expired_cookie_not_sent() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("old=gone; Expires=Thu, 01 Jan 2020 00:00:00 GMT", &url);
|
||||
assert!(jar.get_cookie_header(&url).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_samesite_parsed() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("strict_cookie=val; SameSite=Strict", &url);
|
||||
assert!(jar.get_cookie_header(&url).contains("strict_cookie=val"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clear_cookies() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("a=1", &url);
|
||||
assert!(!jar.get_cookie_header(&url).is_empty());
|
||||
|
||||
jar.clear();
|
||||
assert!(jar.get_cookie_header(&url).is_empty());
|
||||
}
|
||||
}
|
||||
15
crates/obscura-net/src/interceptor.rs
Normal file
15
crates/obscura-net/src/interceptor.rs
Normal file
@@ -0,0 +1,15 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::client::{RequestInfo, Response};
|
||||
|
||||
pub enum InterceptAction {
|
||||
Continue,
|
||||
Block,
|
||||
Fulfill(Response),
|
||||
ModifyHeaders(HashMap<String, String>),
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait RequestInterceptor {
|
||||
async fn intercept(&self, request: &RequestInfo) -> InterceptAction;
|
||||
}
|
||||
14
crates/obscura-net/src/lib.rs
Normal file
14
crates/obscura-net/src/lib.rs
Normal file
@@ -0,0 +1,14 @@
|
||||
pub mod client;
|
||||
pub mod cookies;
|
||||
pub mod interceptor;
|
||||
pub mod robots;
|
||||
pub mod blocklist;
|
||||
#[cfg(feature = "stealth")]
|
||||
pub mod wreq_client;
|
||||
|
||||
pub use client::{ObscuraHttpClient, ObscuraNetError, RequestInfo, ResourceType, Response};
|
||||
pub use cookies::{CookieInfo, CookieJar};
|
||||
pub use robots::RobotsCache;
|
||||
pub use blocklist::is_blocked as is_tracker_blocked;
|
||||
#[cfg(feature = "stealth")]
|
||||
pub use wreq_client::{StealthHttpClient, STEALTH_USER_AGENT};
|
||||
3520
crates/obscura-net/src/pgl_domains.txt
Normal file
3520
crates/obscura-net/src/pgl_domains.txt
Normal file
File diff suppressed because it is too large
Load Diff
163
crates/obscura-net/src/robots.rs
Normal file
163
crates/obscura-net/src/robots.rs
Normal file
@@ -0,0 +1,163 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::RwLock;
|
||||
|
||||
pub struct RobotsCache {
|
||||
cache: RwLock<HashMap<String, RobotsRules>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct RobotsRules {
|
||||
disallowed: Vec<String>,
|
||||
allowed: Vec<String>,
|
||||
}
|
||||
|
||||
impl RobotsCache {
|
||||
pub fn new() -> Self {
|
||||
RobotsCache {
|
||||
cache: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_and_store(&self, domain: &str, body: &str, our_agent: &str) {
|
||||
let rules = parse_robots_txt(body, our_agent);
|
||||
self.cache.write().unwrap().insert(domain.to_string(), rules);
|
||||
}
|
||||
|
||||
pub fn is_allowed(&self, domain: &str, path: &str) -> bool {
|
||||
let cache = self.cache.read().unwrap();
|
||||
let rules = match cache.get(domain) {
|
||||
Some(r) => r,
|
||||
None => return true,
|
||||
};
|
||||
|
||||
for pattern in &rules.allowed {
|
||||
if path_matches(path, pattern) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
for pattern in &rules.disallowed {
|
||||
if path_matches(path, pattern) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RobotsCache {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_robots_txt(body: &str, our_agent: &str) -> RobotsRules {
|
||||
let our_agent_lower = our_agent.to_lowercase();
|
||||
let mut disallowed = Vec::new();
|
||||
let mut allowed = Vec::new();
|
||||
let mut in_matching_section = false;
|
||||
let mut found_specific = false;
|
||||
|
||||
for line in body.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some((key, value)) = line.split_once(':') {
|
||||
let key = key.trim().to_lowercase();
|
||||
let value = value.trim();
|
||||
|
||||
match key.as_str() {
|
||||
"user-agent" => {
|
||||
let agent = value.to_lowercase();
|
||||
in_matching_section = agent == "*"
|
||||
|| our_agent_lower.contains(&agent)
|
||||
|| agent.contains(&our_agent_lower);
|
||||
if agent != "*" && in_matching_section {
|
||||
found_specific = true;
|
||||
}
|
||||
}
|
||||
"disallow" if in_matching_section && !value.is_empty() => {
|
||||
disallowed.push(value.to_string());
|
||||
}
|
||||
"allow" if in_matching_section && !value.is_empty() => {
|
||||
allowed.push(value.to_string());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !found_specific {
|
||||
disallowed.clear();
|
||||
allowed.clear();
|
||||
in_matching_section = false;
|
||||
|
||||
for line in body.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') { continue; }
|
||||
if let Some((key, value)) = line.split_once(':') {
|
||||
let key = key.trim().to_lowercase();
|
||||
let value = value.trim();
|
||||
match key.as_str() {
|
||||
"user-agent" => {
|
||||
in_matching_section = value.trim() == "*";
|
||||
}
|
||||
"disallow" if in_matching_section && !value.is_empty() => {
|
||||
disallowed.push(value.to_string());
|
||||
}
|
||||
"allow" if in_matching_section && !value.is_empty() => {
|
||||
allowed.push(value.to_string());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
RobotsRules { disallowed, allowed }
|
||||
}
|
||||
|
||||
fn path_matches(path: &str, pattern: &str) -> bool {
|
||||
if pattern.ends_with('*') {
|
||||
path.starts_with(&pattern[..pattern.len() - 1])
|
||||
} else if pattern.ends_with('$') {
|
||||
path == &pattern[..pattern.len() - 1]
|
||||
} else {
|
||||
path.starts_with(pattern)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_basic_robots() {
|
||||
let body = "User-agent: *\nDisallow: /private/\nDisallow: /admin\nAllow: /admin/public\n";
|
||||
let cache = RobotsCache::new();
|
||||
cache.parse_and_store("example.com", body, "Obscura");
|
||||
assert!(cache.is_allowed("example.com", "/"));
|
||||
assert!(cache.is_allowed("example.com", "/page"));
|
||||
assert!(!cache.is_allowed("example.com", "/private/secret"));
|
||||
assert!(!cache.is_allowed("example.com", "/admin"));
|
||||
assert!(cache.is_allowed("example.com", "/admin/public"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_rules_means_allowed() {
|
||||
let cache = RobotsCache::new();
|
||||
assert!(cache.is_allowed("unknown.com", "/anything"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_disallow_all() {
|
||||
let body = "User-agent: *\nDisallow: /\n";
|
||||
let cache = RobotsCache::new();
|
||||
cache.parse_and_store("blocked.com", body, "Obscura");
|
||||
assert!(!cache.is_allowed("blocked.com", "/"));
|
||||
assert!(!cache.is_allowed("blocked.com", "/page"));
|
||||
}
|
||||
}
|
||||
149
crates/obscura-net/src/wreq_client.rs
Normal file
149
crates/obscura-net/src/wreq_client.rs
Normal file
@@ -0,0 +1,149 @@
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::collections::HashMap;
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::error::Error;
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::sync::Arc;
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::time::Duration;
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
use tokio::sync::RwLock;
|
||||
#[cfg(feature = "stealth")]
|
||||
use url::Url;
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
use crate::cookies::CookieJar;
|
||||
#[cfg(feature = "stealth")]
|
||||
use crate::client::{Response, ObscuraNetError};
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
pub const STEALTH_USER_AGENT: &str =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
pub struct StealthHttpClient {
|
||||
client: wreq::Client,
|
||||
pub cookie_jar: Arc<CookieJar>,
|
||||
pub extra_headers: RwLock<HashMap<String, String>>,
|
||||
pub in_flight: Arc<std::sync::atomic::AtomicU32>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
impl StealthHttpClient {
|
||||
pub fn new(cookie_jar: Arc<CookieJar>) -> Self {
|
||||
Self::with_proxy(cookie_jar, None)
|
||||
}
|
||||
|
||||
pub fn with_proxy(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
|
||||
let cert_store = wreq::tls::CertStore::builder()
|
||||
.set_default_paths()
|
||||
.build()
|
||||
.expect("Failed to load system CA certificates");
|
||||
|
||||
let emulation_opts = wreq_util::EmulationOption::builder()
|
||||
.emulation(wreq_util::Emulation::Chrome145)
|
||||
.emulation_os(wreq_util::EmulationOS::Linux)
|
||||
.build();
|
||||
|
||||
let mut builder = wreq::Client::builder()
|
||||
.emulation(emulation_opts)
|
||||
.cert_store(cert_store)
|
||||
.timeout(Duration::from_secs(30))
|
||||
.redirect(wreq::redirect::Policy::none());
|
||||
|
||||
if let Some(proxy) = proxy_url {
|
||||
if let Ok(p) = wreq::Proxy::all(proxy) {
|
||||
builder = builder.proxy(p);
|
||||
}
|
||||
}
|
||||
|
||||
let client = builder.build().expect("failed to build wreq stealth client");
|
||||
|
||||
StealthHttpClient {
|
||||
client,
|
||||
cookie_jar,
|
||||
extra_headers: RwLock::new(HashMap::new()),
|
||||
in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
||||
let mut current_url = url.clone();
|
||||
let mut redirects = Vec::new();
|
||||
|
||||
for _ in 0..20 {
|
||||
let mut req = self.client.get(current_url.as_str());
|
||||
|
||||
let cookie_header = self.cookie_jar.get_cookie_header(¤t_url);
|
||||
if !cookie_header.is_empty() {
|
||||
req = req.header("Cookie", &cookie_header);
|
||||
}
|
||||
|
||||
for (k, v) in self.extra_headers.read().await.iter() {
|
||||
req = req.header(k.as_str(), v.as_str());
|
||||
}
|
||||
|
||||
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let resp = req.send().await.map_err(|e| {
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
ObscuraNetError::Network(format!("{}: {} (source: {:?})", current_url, e, e.source()))
|
||||
})?;
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let status = resp.status();
|
||||
|
||||
for val in resp.headers().get_all("set-cookie") {
|
||||
if let Ok(s) = val.to_str() {
|
||||
self.cookie_jar.set_cookie(s, ¤t_url);
|
||||
}
|
||||
}
|
||||
|
||||
let response_headers: HashMap<String, String> = resp
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
|
||||
if status.is_redirection() {
|
||||
if let Some(location) = resp.headers().get("location") {
|
||||
let location_str = location.to_str().map_err(|_| {
|
||||
ObscuraNetError::Network("Invalid redirect Location".into())
|
||||
})?;
|
||||
let next_url = current_url.join(location_str).map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Invalid redirect URL: {}", e))
|
||||
})?;
|
||||
redirects.push(current_url.clone());
|
||||
current_url = next_url;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let body = resp.bytes().await.map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Failed to read body: {}", e))
|
||||
})?.to_vec();
|
||||
|
||||
return Ok(Response {
|
||||
url: current_url,
|
||||
status: status.as_u16(),
|
||||
headers: response_headers,
|
||||
body,
|
||||
redirected_from: redirects,
|
||||
});
|
||||
}
|
||||
|
||||
Err(ObscuraNetError::TooManyRedirects(url.to_string()))
|
||||
}
|
||||
|
||||
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
|
||||
*self.extra_headers.write().await = headers;
|
||||
}
|
||||
|
||||
pub fn active_requests(&self) -> u32 {
|
||||
self.in_flight.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn is_network_idle(&self) -> bool {
|
||||
self.active_requests() == 0
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user