Quick start
Smart crawler
Keep ordinary pages on HTTP and promote browser-only responses to Chromium.
Smart crawler
SmartKind<ChromiumoxideProvider> begins on the HTTP-and-HTML path. Its promotion detector examines the response and sends flagged pages—such as a JavaScript shell or another browser-only response—to a real Chromium browser. This preserves the lower cost of HTTP for contentful documents while still handling the pages that require rendering.
SmartContext tells the handler which path completed the request. Its HTTP variant exposes parsed HTML for static pages; its browser variant exposes the live page for browser work such as waiting on a selector or evaluating JavaScript. The example below discovers same-host links through the shared smart context, records which variant handled each URL, keeps the home and static pages on HTTP, and promotes only the application shell.
Enable browser-chromiumoxide; it also enables browser. The html feature is already part of the default feature set and supplies the HTTP-side parsed document used by smart crawling.
//! Demonstrates HTTP-first crawling that promotes only a JavaScript shell to headless Chromium.
//!
//! Run with:
//! `cargo run -p millipede --features browser-chromiumoxide,html,storage-memory --example smart_crawl`
use std::{
sync::{Arc, Mutex},
time::Duration,
};
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{method, path},
};
fn html(body: impl Into<Vec<u8>>) -> ResponseTemplate {
ResponseTemplate::new(200).set_body_raw(body, "text/html")
}
async fn mount_page(server: &MockServer, route: &str, body: impl Into<Vec<u8>>) {
Mock::given(method("GET"))
.and(path(route))
.respond_with(html(body))
.mount(server)
.await;
}
async fn local_site() -> MockServer {
let server = MockServer::start().await;
mount_page(
&server,
"/",
r#"<!doctype html><html><head><title>HTTP home</title></head><body>
<p>This contentful home page is intentionally long enough to stay on the fast HTTP path
while linking to two more static documents and one application that needs JavaScript.</p>
<a href="/static/1">Static 1</a><a href="/static/2">Static 2</a><a href="/app">App</a>
</body></html>"#,
)
.await;
for page in 1..=2 {
mount_page(
&server,
&format!("/static/{page}"),
format!(
r#"<!doctype html><html><head><title>Static {page}</title></head><body>
<p>This is static page {page}, with enough meaningful server-rendered content to
remain on the HTTP path instead of consuming an expensive browser page.</p>
</body></html>"#
),
)
.await;
}
mount_page(
&server,
"/app",
r#"<html><body><div id="root"></div><script>document.getElementById('root').innerHTML = '<h1 id="ready">Rendered by JS</h1>';</script></body></html>"#,
)
.await;
server
}
fn contains_path(urls: &[String], expected: &str) -> bool {
urls.iter().any(|value| {
url::Url::parse(value)
.map(|url| url.path() == expected)
.unwrap_or(false)
})
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let Some(exe) = millipede::find_browser() else {
eprintln!(
"smart_crawl: no Chromium/Chrome found; set MILLIPEDE_CHROME to your browser binary"
);
return Ok(());
};
let server = local_site().await;
let kind = millipede::SmartKind::builder(millipede::ChromiumoxideProvider)
.browser_kind(
millipede::BrowserKind::builder(millipede::ChromiumoxideProvider)
.launch_options(millipede::ChromiumLaunchOptions::default().with_executable(exe))
.navigation_timeout(Duration::from_secs(20)),
)
.detector(millipede::DefaultPromotionDetector::default().with_min_visible_text(120))
.sticky_promotion(false)
.build()?;
let paths = Arc::new(Mutex::new((Vec::<String>::new(), Vec::<String>::new())));
let handler_paths = Arc::clone(&paths);
let crawler = millipede::Crawler::builder(kind)
.max_concurrency(3)
.storage_client(Arc::new(millipede::MemoryStorageClient::new()))
.request_handler(move |ctx: millipede::SmartContext| {
let handler_paths = Arc::clone(&handler_paths);
async move {
let _ = ctx.enqueue().same_hostname().await?;
match ctx {
millipede::SmartContext::Http(http) => {
let selector = millipede::html::scraper::Selector::parse("title")
.expect("static title selector is valid");
let title = http
.html
.select_first(&selector, |element| element.text().collect::<String>());
println!("HTTP {}: {title:?}", http.request.url);
handler_paths
.lock()
.unwrap_or_else(|error| error.into_inner())
.0
.push(http.request.url.to_string());
}
millipede::SmartContext::Browser(browser) => {
browser
.page
.wait_for_selector("#ready", Duration::from_secs(10))
.await?;
let heading = browser
.page
.evaluate_js("document.querySelector('#ready').textContent")
.await?;
println!("Browser {}: {heading}", browser.request.url);
handler_paths
.lock()
.unwrap_or_else(|error| error.into_inner())
.1
.push(browser.request.url.to_string());
}
_ => unreachable!("unsupported smart crawler context variant"),
}
Ok(())
}
})
.build()
.await?;
let stats = crawler.run([server.uri()]).await?;
let (http_urls, browser_urls) = paths
.lock()
.unwrap_or_else(|error| error.into_inner())
.clone();
println!("HTTP URLs: {http_urls:#?}");
println!("Browser URLs: {browser_urls:#?}");
println!(
"smart crawl complete: finished={}, failed={}",
stats.requests_finished, stats.requests_failed
);
anyhow::ensure!(contains_path(&http_urls, "/"), "root did not stay on HTTP");
anyhow::ensure!(
contains_path(&http_urls, "/static/1") && contains_path(&http_urls, "/static/2"),
"static pages did not stay on HTTP"
);
anyhow::ensure!(
browser_urls.len() == 1 && contains_path(&browser_urls, "/app"),
"expected only /app in the browser list"
);
Ok(())
}Run the complete example with:
cargo run -p millipede --features browser-chromiumoxide,html,storage-memory --example smart_crawl