Examples
Browser crawl
Drive a local catalog through headless Chromium and store page titles.
This crawler launches Chromium against a local catalog containing a home page, three categories, and six products. Up to three handlers evaluate document.title, save URL-title records in an in-memory dataset, and enqueue same-hostname links, while the browser pool permits four open pages per browser.
A local Chrome or Chromium installation is required. Set MILLIPEDE_CHROME to the browser executable to override automatic discovery.
cargo run -p millipede --features browser-chromiumoxide,storage-memory --example browser_crawl//! Crawls a local category-and-product site with headless Chromium and stores page titles.
//!
//! Run with:
//! `cargo run -p millipede --features browser-chromiumoxide,storage-memory --example browser_crawl`
use std::{sync::Arc, time::Duration};
use millipede::DatasetExt;
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{method, path},
};
fn html(body: impl Into<Vec<u8>>) -> ResponseTemplate {
ResponseTemplate::new(200).set_body_raw(body, "text/html")
}
async fn mount_page(server: &MockServer, route: &str, body: impl Into<Vec<u8>>) {
Mock::given(method("GET"))
.and(path(route))
.respond_with(html(body))
.mount(server)
.await;
}
async fn local_site() -> MockServer {
let server = MockServer::start().await;
mount_page(
&server,
"/",
r#"<!doctype html><html><head><title>Catalog</title></head><body>
<a href="/category/1">Category 1</a>
<a href="/category/2">Category 2</a>
<a href="/category/3">Category 3</a>
</body></html>"#,
)
.await;
for category in 1..=3 {
mount_page(
&server,
&format!("/category/{category}"),
format!(
r#"<!doctype html><html><head><title>Category {category}</title></head><body>
<a href="/product/{category}-1">Product {category}-1</a>
<a href="/product/{category}-2">Product {category}-2</a>
</body></html>"#
),
)
.await;
for product in 1..=2 {
mount_page(
&server,
&format!("/product/{category}-{product}"),
format!(
r#"<!doctype html><html><head><title>Product {category}-{product}</title></head>
<body><a href="/">Catalog</a></body></html>"#
),
)
.await;
}
}
server
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let Some(exe) = millipede::find_browser() else {
eprintln!(
"browser_crawl: no Chromium/Chrome found; set MILLIPEDE_CHROME to your browser binary"
);
return Ok(());
};
let server = local_site().await;
let kind = millipede::BrowserKind::builder(millipede::ChromiumoxideProvider)
.launch_options(millipede::ChromiumLaunchOptions::default().with_executable(exe))
.max_open_pages_per_browser(4)
.navigation_timeout(Duration::from_secs(20))
.build()?;
let crawler = millipede::Crawler::builder(kind)
.max_concurrency(3)
.storage_client(Arc::new(millipede::MemoryStorageClient::new()))
.request_handler(|ctx: millipede::BrowserContext| async move {
let title = ctx.page.evaluate_js("document.title").await.ok();
ctx.storage
.dataset()
.push(&serde_json::json!({
"url": ctx.request.url.as_str(),
"title": title,
}))
.await?;
let _ = ctx.enqueue.same_hostname().await?;
Ok(())
})
.build()
.await?;
let stats = crawler.run([server.uri()]).await?;
println!(
"browser crawl complete: finished={}, failed={}",
stats.requests_finished, stats.requests_failed
);
anyhow::ensure!(
stats.requests_finished >= 9,
"expected at least 9 finished requests, got {}",
stats.requests_finished
);
Ok(())
}