Examples
Fingerprint crawl
Exercise deterministic headers, challenge recovery, error grouping, and snapshots offline.
A custom offline HTTP client verifies that header generation and session tokens are enabled, then returns a challenge body on the first visit to one route and a recovered body after session rotation. The recovered handler deliberately fails non-retryably so the crawler persists its HTML response snapshot; the example then checks one success, one failure, one anti-bot retry, normalized error groups, and stable user agents for the same session token.
cargo run -p millipede --features http,fingerprint,storage-memory --example fingerprint_crawl//! Demonstrates deterministic browser-like headers, anti-bot detection, normalized error
//! statistics, and failure snapshots against a fully offline mock site. Run it with
//! `cargo run -p millipede --features http,fingerprint,storage-memory --example fingerprint_crawl`.
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use millipede::StorageClient;
const DETECTED_CHALLENGE_BODY: &str = "<html><title>Just a moment...</title><body>Checking your browser before accessing the site.</body></html>";
const CHALLENGE_BODY: &str =
"<html><body>The Cloudflare challenge route recovered after session rotation.</body></html>";
struct OfflineClient {
attempts: AtomicUsize,
}
#[async_trait::async_trait]
impl millipede::HttpClient for OfflineClient {
async fn send(
&self,
request: millipede::HttpRequest,
) -> Result<millipede::HttpResponse, millipede::HttpClientError> {
if !request.use_header_generator {
return Err(millipede::HttpClientError::other(anyhow::anyhow!(
"header generation was not enabled"
)));
}
let token = request.session_token.as_ref().ok_or_else(|| {
millipede::HttpClientError::other(anyhow::anyhow!("session token was not supplied"))
})?;
let profile = millipede::HeaderGenerator::new().generate(token.as_str());
if profile.user_agent.is_empty() || profile.headers.is_empty() {
return Err(millipede::HttpClientError::other(anyhow::anyhow!(
"generated header profile was empty"
)));
}
let body = if request.url.path() == "/challenge"
&& self.attempts.fetch_add(1, Ordering::SeqCst) == 0
{
DETECTED_CHALLENGE_BODY
} else if request.url.path() == "/challenge" {
CHALLENGE_BODY
} else {
"<html><body>Ready to crawl.</body></html>"
};
let mut headers = millipede::HeaderMap::new();
headers.insert(
"content-type",
"text/html"
.parse()
.expect("static content type should parse"),
);
Ok(millipede::HttpResponse::new(
request.url,
200_u16.try_into().expect("static status code should parse"),
headers,
body.into(),
))
}
async fn stream(
&self,
_request: millipede::HttpRequest,
) -> Result<millipede::StreamingResponse, millipede::HttpClientError> {
Err(millipede::HttpClientError::other(anyhow::anyhow!(
"streaming is not used by this example"
)))
}
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let normal_url = "https://offline.example/normal";
let challenge_url = "https://offline.example/challenge";
let normal_request = millipede::Request::get(normal_url).build()?;
let challenge_request = millipede::Request::get(challenge_url).build()?;
let snapshot_key = format!(
"{}.body",
millipede::ErrorSnapshotter::base_key(&challenge_request)
);
let storage = Arc::new(millipede::MemoryStorageClient::new());
let crawler = millipede::Crawler::builder(
millipede::HttpKind::builder()
.http_client(Arc::new(OfflineClient {
attempts: AtomicUsize::new(0),
}))
.header_generator(true)
.detect_anti_bot_default()
.snapshot_errors_on_failure(true)
.build()?,
)
.max_request_retries(0)
.max_session_rotations(1)
.storage_client(storage.clone())
.request_handler(|ctx: millipede::HttpContext| async move {
if ctx.request.url.path() == "/challenge" {
return Err(millipede::CrawlError::non_retryable(anyhow::anyhow!(
"intentional handler failure after recovering {}",
ctx.request.url
)));
}
Ok(())
})
.failed_request_handler(|ctx: millipede::FailedRequestContext| async move {
eprintln!("failed to crawl {}: {}", ctx.request.url, ctx.error);
Ok(())
})
.build()
.await?;
let stats = crawler.run([normal_request, challenge_request]).await?;
let kvs = storage.open_key_value_store(Some("default")).await?;
let snapshot = millipede::ErrorSnapshotter::new(kvs)
.load(&snapshot_key)
.await?
.ok_or_else(|| anyhow::anyhow!("challenge response snapshot was not persisted"))?;
anyhow::ensure!(
snapshot.content_type == "text/html",
"unexpected snapshot content type: {}",
snapshot.content_type
);
anyhow::ensure!(
snapshot.bytes.as_ref() == CHALLENGE_BODY.as_bytes(),
"snapshot did not match the recovered challenge-route response body"
);
anyhow::ensure!(
stats.requests_finished == 1 && stats.requests_failed == 1 && stats.requests_retries == 1,
"unexpected final request counts: {stats:#?}"
);
anyhow::ensure!(
stats
.errors
.keys()
.any(|error| error.contains("intentional handler failure")),
"terminal error groups did not include the handler failure: {:#?}",
stats.errors
);
anyhow::ensure!(
stats
.retry_errors
.keys()
.any(|error| error.contains("anti-bot detected: Cloudflare")),
"retry error groups did not include the anti-bot retry: {:#?}",
stats.retry_errors
);
println!(
"recovered snapshot: content_type={} bytes={}",
snapshot.content_type,
snapshot.bytes.len()
);
println!(
"FinalStatistics: requests_finished={} requests_failed={} requests_retries={} errors={:#?} retry_errors={:#?}",
stats.requests_finished,
stats.requests_failed,
stats.requests_retries,
stats.errors,
stats.retry_errors
);
let generator = millipede::HeaderGenerator::new();
let first_headers = generator.generate("demo-session");
let second_headers = generator.generate("demo-session");
anyhow::ensure!(
first_headers.user_agent == second_headers.user_agent,
"the same session token produced different user agents"
);
println!("{}", first_headers.user_agent);
println!("{}", second_headers.user_agent);
Ok(())
}