Examples
Scrape books
Route category and detail pages into a persistent structured dataset.
This HtmlCrawler routes listing pages and labeled detail pages separately: listings enqueue pagination plus product links, while detail handlers extract title, price, availability, and URL into JSON records. It restricts discovery to the same hostname, caps the crawl at 200 requests with concurrency five, purges storage at startup, and writes a Crawlee-compatible dataset beneath ./storage.
This example hits the real books.toscrape.com open practice site. Set MILLIPEDE_BOOKS_BASE_URL or pass the first CLI argument to override the base URL; it is deliberately excluded from CI's examples job, where a wiremock twin runs in the test suite instead.
cargo run -p millipede --example scrape_books --features storage-fs//! Crawls category and detail pages on books.toscrape.com and stores a structured book dataset.
//!
//! Run with:
//! `cargo run -p millipede --example scrape_books --features storage-fs`
//!
//! This example HITS THE REAL NETWORK. books.toscrape.com is an open-source practice site for
//! scraping. CI instead runs the wiremock version in `millipede/tests/scrape_books_mock.rs`, which
//! is why this example is deliberately not in the `.github/workflows/ci.yml` examples job.
//! `MILLIPEDE_BOOKS_BASE_URL` or the first command-line argument overrides the base URL.
use std::sync::Arc;
use millipede::{
Configuration, CrawlPolicy, Crawler, DatasetExt, EnqueueStrategy, FailedRequestContext,
FsStorageClient, HtmlContext, HtmlKind, Router,
};
use serde_json::json;
millipede_html::selectors! {
title_selector = "h1";
price_selector = "p.price_color";
availability_selector = "p.instock.availability";
}
fn text(ctx: &HtmlContext, selector: &millipede_html::scraper::Selector) -> String {
ctx.html
.select_first(selector, |element| element.text().collect::<String>())
.unwrap_or_default()
.trim()
.to_owned()
}
fn base_url() -> String {
std::env::args()
.nth(1)
.or_else(|| std::env::var("MILLIPEDE_BOOKS_BASE_URL").ok())
.unwrap_or_else(|| "https://books.toscrape.com/".to_owned())
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let storage_client = Arc::new(FsStorageClient::new("./storage"));
let configuration = Configuration::builder().purge_on_start(true).build()?;
let router = Router::<HtmlContext>::new()
.default(|ctx: HtmlContext| async move {
let _ = ctx.enqueue.options().selector("ul.pager a").send().await?;
let _ = ctx
.enqueue
.options()
.selector("article.product_pod h3 a")
.globs(["**/catalogue/**"])
.label("detail")
.send()
.await?;
Ok(())
})
.route("detail", |ctx: HtmlContext| async move {
let book = json!({
"url": ctx.request.url,
"title": text(&ctx, title_selector()),
"price": text(&ctx, price_selector()),
"availability": text(&ctx, availability_selector()),
});
ctx.storage.dataset().push(&book).await?;
Ok(())
});
let crawler = Crawler::builder(HtmlKind::new()?)
.configuration(configuration)
.storage_client(storage_client)
.crawl_policy(
CrawlPolicy::new()
.strategy(EnqueueStrategy::SameHostname)
.max_requests_per_crawl(200),
)
.max_concurrency(5)
.request_handler(router)
.failed_request_handler(|ctx: FailedRequestContext| async move {
eprintln!("book request failed for {}: {}", ctx.request.url, ctx.error);
Ok(())
})
.build()
.await?;
let stats = crawler.run(base_url()).await?;
println!("crawl complete: {stats:#?}");
println!(
"books are in ./storage/datasets/default/ using a Crawlee-compatible layout; see \
docs/guide/crawlee-storage-migration.md"
);
Ok(())
}