Examples
Smart crawl
Keep static documents on HTTP and promote only a JavaScript shell to Chromium.
This local four-page crawl uses a visible-text threshold to leave the content-rich home and two static pages on the fast HTML path. The sparse /app shell alone is promoted to Chromium, where the handler waits for #ready and reads its JavaScript-rendered heading; non-sticky promotion prevents that decision from automatically spreading to other requests.
A local Chrome or Chromium installation is required. Set MILLIPEDE_CHROME to the browser executable to override automatic discovery.
cargo run -p millipede --features browser-chromiumoxide,html,storage-memory --example smart_crawl//! Demonstrates HTTP-first crawling that promotes only a JavaScript shell to headless Chromium.
//!
//! Run with:
//! `cargo run -p millipede --features browser-chromiumoxide,html,storage-memory --example smart_crawl`
use std::{
sync::{Arc, Mutex},
time::Duration,
};
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{method, path},
};
fn html(body: impl Into<Vec<u8>>) -> ResponseTemplate {
ResponseTemplate::new(200).set_body_raw(body, "text/html")
}
async fn mount_page(server: &MockServer, route: &str, body: impl Into<Vec<u8>>) {
Mock::given(method("GET"))
.and(path(route))
.respond_with(html(body))
.mount(server)
.await;
}
async fn local_site() -> MockServer {
let server = MockServer::start().await;
mount_page(
&server,
"/",
r#"<!doctype html><html><head><title>HTTP home</title></head><body>
<p>This contentful home page is intentionally long enough to stay on the fast HTTP path
while linking to two more static documents and one application that needs JavaScript.</p>
<a href="/static/1">Static 1</a><a href="/static/2">Static 2</a><a href="/app">App</a>
</body></html>"#,
)
.await;
for page in 1..=2 {
mount_page(
&server,
&format!("/static/{page}"),
format!(
r#"<!doctype html><html><head><title>Static {page}</title></head><body>
<p>This is static page {page}, with enough meaningful server-rendered content to
remain on the HTTP path instead of consuming an expensive browser page.</p>
</body></html>"#
),
)
.await;
}
mount_page(
&server,
"/app",
r#"<html><body><div id="root"></div><script>document.getElementById('root').innerHTML = '<h1 id="ready">Rendered by JS</h1>';</script></body></html>"#,
)
.await;
server
}
fn contains_path(urls: &[String], expected: &str) -> bool {
urls.iter().any(|value| {
url::Url::parse(value)
.map(|url| url.path() == expected)
.unwrap_or(false)
})
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let Some(exe) = millipede::find_browser() else {
eprintln!(
"smart_crawl: no Chromium/Chrome found; set MILLIPEDE_CHROME to your browser binary"
);
return Ok(());
};
let server = local_site().await;
let kind = millipede::SmartKind::builder(millipede::ChromiumoxideProvider)
.browser_kind(
millipede::BrowserKind::builder(millipede::ChromiumoxideProvider)
.launch_options(millipede::ChromiumLaunchOptions::default().with_executable(exe))
.navigation_timeout(Duration::from_secs(20)),
)
.detector(millipede::DefaultPromotionDetector::default().with_min_visible_text(120))
.sticky_promotion(false)
.build()?;
let paths = Arc::new(Mutex::new((Vec::<String>::new(), Vec::<String>::new())));
let handler_paths = Arc::clone(&paths);
let crawler = millipede::Crawler::builder(kind)
.max_concurrency(3)
.storage_client(Arc::new(millipede::MemoryStorageClient::new()))
.request_handler(move |ctx: millipede::SmartContext| {
let handler_paths = Arc::clone(&handler_paths);
async move {
let _ = ctx.enqueue().same_hostname().await?;
match ctx {
millipede::SmartContext::Http(http) => {
let selector = millipede::html::scraper::Selector::parse("title")
.expect("static title selector is valid");
let title = http
.html
.select_first(&selector, |element| element.text().collect::<String>());
println!("HTTP {}: {title:?}", http.request.url);
handler_paths
.lock()
.unwrap_or_else(|error| error.into_inner())
.0
.push(http.request.url.to_string());
}
millipede::SmartContext::Browser(browser) => {
browser
.page
.wait_for_selector("#ready", Duration::from_secs(10))
.await?;
let heading = browser
.page
.evaluate_js("document.querySelector('#ready').textContent")
.await?;
println!("Browser {}: {heading}", browser.request.url);
handler_paths
.lock()
.unwrap_or_else(|error| error.into_inner())
.1
.push(browser.request.url.to_string());
}
_ => unreachable!("unsupported smart crawler context variant"),
}
Ok(())
}
})
.build()
.await?;
let stats = crawler.run([server.uri()]).await?;
let (http_urls, browser_urls) = paths
.lock()
.unwrap_or_else(|error| error.into_inner())
.clone();
println!("HTTP URLs: {http_urls:#?}");
println!("Browser URLs: {browser_urls:#?}");
println!(
"smart crawl complete: finished={}, failed={}",
stats.requests_finished, stats.requests_failed
);
anyhow::ensure!(contains_path(&http_urls, "/"), "root did not stay on HTTP");
anyhow::ensure!(
contains_path(&http_urls, "/static/1") && contains_path(&http_urls, "/static/2"),
"static pages did not stay on HTTP"
);
anyhow::ensure!(
browser_urls.len() == 1 && contains_path(&browser_urls, "/app"),
"expected only /app in the browser list"
);
Ok(())
}