Add fetch navigation timeout (#92)
This commit is contained in:
1 parent
a7b1acfa18
commit
9d8d3a02a9
2 files changed
+18
-5
No files matched your search
@@ -87,6 +87,9 @@ obscura fetch https://news.ycombinator.com --dump html
|
|||||||
|
|
||||||
# Wait for dynamic content
|
# Wait for dynamic content
|
||||||
obscura fetch https://example.com --wait-until networkidle0
|
obscura fetch https://example.com --wait-until networkidle0
|
||||||
|
|
||||||
|
# Bound navigation time for slow or broken pages
|
||||||
|
obscura fetch https://example.com --timeout 10
|
||||||
```
|
```
|
||||||
|
|
||||||
### Start the CDP server
|
### Start the CDP server
|
||||||
@@ -232,6 +235,7 @@ Fetch and render a single page.
|
|||||||
| `--dump` | `html` | Output: `html`, `text`, or `links` |
|
| `--dump` | `html` | Output: `html`, `text`, or `links` |
|
||||||
| `--eval` | — | JavaScript expression to evaluate |
|
| `--eval` | — | JavaScript expression to evaluate |
|
||||||
| `--wait-until` | `load` | Wait: `load`, `domcontentloaded`, `networkidle0` |
|
| `--wait-until` | `load` | Wait: `load`, `domcontentloaded`, `networkidle0` |
|
||||||
|
| `--timeout` | `30` | Maximum navigation time in seconds |
|
||||||
| `--selector` | — | Wait for CSS selector |
|
| `--selector` | — | Wait for CSS selector |
|
||||||
| `--stealth` | off | Anti-detection mode |
|
| `--stealth` | off | Anti-detection mode |
|
||||||
| `--quiet` | off | Suppress banner |
|
| `--quiet` | off | Suppress banner |
|
||||||
|
|||||||
@@ -60,6 +60,9 @@ enum Command {
|
|||||||
#[arg(long, default_value_t = 5)]
|
#[arg(long, default_value_t = 5)]
|
||||||
wait: u64,
|
wait: u64,
|
||||||
|
|
||||||
|
#[arg(long, default_value_t = 30, value_parser = clap::value_parser!(u64).range(1..))]
|
||||||
|
timeout: u64,
|
||||||
|
|
||||||
#[arg(long, default_value = "load")]
|
#[arg(long, default_value = "load")]
|
||||||
wait_until: String,
|
wait_until: String,
|
||||||
|
|
||||||
@@ -154,8 +157,8 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
obscura_cdp::start_with_options(port, proxy, stealth).await?;
|
obscura_cdp::start_with_options(port, proxy, stealth).await?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Some(Command::Fetch { url, dump, selector, wait, wait_until, user_agent, stealth, eval, quiet }) => {
|
Some(Command::Fetch { url, dump, selector, wait, timeout, wait_until, user_agent, stealth, eval, quiet }) => {
|
||||||
run_fetch(&url, dump, selector, wait, &wait_until, user_agent, stealth, eval, quiet).await?;
|
run_fetch(&url, dump, selector, wait, timeout, &wait_until, user_agent, stealth, eval, quiet).await?;
|
||||||
}
|
}
|
||||||
Some(Command::Scrape { urls, eval, concurrency, format, timeout }) => {
|
Some(Command::Scrape { urls, eval, concurrency, format, timeout }) => {
|
||||||
run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout).await?;
|
run_parallel_scrape(urls, eval, concurrency.get(), &format, timeout).await?;
|
||||||
@@ -300,6 +303,7 @@ async fn run_fetch(
|
|||||||
dump: DumpFormat,
|
dump: DumpFormat,
|
||||||
selector: Option<String>,
|
selector: Option<String>,
|
||||||
wait_secs: u64,
|
wait_secs: u64,
|
||||||
|
timeout_secs: u64,
|
||||||
wait_until: &str,
|
wait_until: &str,
|
||||||
user_agent: Option<String>,
|
user_agent: Option<String>,
|
||||||
stealth: bool,
|
stealth: bool,
|
||||||
@@ -319,9 +323,14 @@ async fn run_fetch(
|
|||||||
eprintln!("Fetching {}...", url_str);
|
eprintln!("Fetching {}...", url_str);
|
||||||
}
|
}
|
||||||
|
|
||||||
page.navigate_with_wait(url_str, wait_condition).await.map_err(|e| {
|
match timeout(Duration::from_secs(timeout_secs), page.navigate_with_wait(url_str, wait_condition)).await {
|
||||||
anyhow::anyhow!("Failed to navigate to {}: {}", url_str, e)
|
Ok(result) => result.map_err(|e| anyhow::anyhow!("Failed to navigate to {}: {}", url_str, e))?,
|
||||||
})?;
|
Err(_) => anyhow::bail!(
|
||||||
|
"Timed out navigating to {} after {}s",
|
||||||
|
url_str,
|
||||||
|
timeout_secs
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
if !quiet {
|
if !quiet {
|
||||||
eprintln!("Page loaded: {} - \"{}\"", page.url_string(), page.title);
|
eprintln!("Page loaded: {} - \"{}\"", page.url_string(), page.title);
|
||||||
|
|||||||
Reference in new issue
Block a user