Rust SDK

An async, strongly typed client for the CrawlKit API. Built on reqwest and serde with full tokio support. Build high-performance data pipelines and ship code that Google actually indexes.

Installation

# Future official crate
cargo add crawlkit-client

# For now, add these dependencies to Cargo.toml
[dependencies]
reqwest = { version = "0.12", features = ["json"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
tokio = { version = "1", features = ["full"] }
thiserror = "2"

Type Definitions

use serde::{Deserialize, Serialize};

#[derive(Debug, Serialize)]
pub struct CreateDatasetRequest {
    pub name: String,
    pub description: Option<String>,
    pub columns: Vec<Column>,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct Column {
    pub name: String,
    pub data_type: ColumnType,
}

#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ColumnType {
    Text,
    Integer,
    Float,
    Boolean,
    Jsonb,
    Timestamp,
}

#[derive(Debug, Deserialize)]
pub struct Dataset {
    pub id: String,
    pub name: String,
    pub description: Option<String>,
    pub columns: Vec<Column>,
    pub row_count: u64,
    pub created_at: String,
}

#[derive(Debug, Serialize)]
pub struct SpiderConfig {
    pub name: String,
    pub scope: SpiderScope,
    pub definition: SpiderDefinition,
    pub dataset_id: Option<String>,
}

#[derive(Debug, Serialize)]
pub struct SpiderScope {
    pub start_urls: Vec<String>,
}

#[derive(Debug, Serialize)]
pub struct SpiderDefinition {
    pub handlers: Vec<SpiderHandler>,
    pub navigation: Option<SpiderNavigation>,
}

#[derive(Debug, Serialize)]
pub struct SpiderHandler {
    pub url_pattern: String,
    pub fields: Vec<SpiderField>,
}

#[derive(Debug, Serialize)]
pub struct SpiderField {
    pub name: String,
    pub selector: String,
    #[serde(rename = "type")]
    pub field_type: String,
    pub attribute: Option<String>,
}

#[derive(Debug, Serialize)]
pub struct SpiderNavigation {
    pub follow_links: Option<String>,
    pub max_depth: Option<u32>,
}

#[derive(Debug, Deserialize)]
pub struct Spider {
    pub id: String,
    pub name: String,
    pub status: String,
    pub scope: serde_json::Value,
    pub dataset_id: Option<String>,
    pub created_at: String,
}

#[derive(Debug, Deserialize)]
pub struct EnrichmentJob {
    pub job_id: String,
    pub status: String,
    pub source: String,
    pub dataset_id: String,
    pub total_rows: u64,
    pub processed_rows: u64,
    pub created_at: String,
}

#[derive(Debug, Deserialize)]
pub struct PipelineRun {
    pub run_id: String,
    pub status: String,
}

#[derive(Debug, Deserialize)]
pub struct SeoCheckResult {
    pub url: String,
    pub score: u32,
    pub issues: Vec<SeoIssue>,
    pub metadata: serde_json::Value,
    pub remediations: Option<Vec<serde_json::Value>>,
}

#[derive(Debug, Deserialize)]
pub struct SeoIssue {
    pub category: String,
    pub severity: String,
    pub message: String,
    pub affected_urls: Vec<String>,
    pub evidence: Option<Vec<String>>,
}

#[derive(Debug, Deserialize)]
pub struct AuditResult {
    pub audit_id: String,
    pub score: u32,
    pub crawl_stats: serde_json::Value,
    pub issues: Vec<SeoIssue>,
}

Error Types

use thiserror::Error;

#[derive(Debug, Error)]
pub enum CrawlKitError {
    #[error("API error {status}: {message}")]
    Api {
        message: String,
        status: u16,
        retryable: bool,
        retry_after: Option<u64>,
    },

    #[error("HTTP error: {0}")]
    Http(#[from] reqwest::Error),

    #[error("Deserialization error: {0}")]
    Deserialize(#[from] serde_json::Error),
}

impl CrawlKitError {
    pub fn is_retryable(&self) -> bool {
        match self {
            CrawlKitError::Api { retryable, .. } => *retryable,
            CrawlKitError::Http(e) => e.is_timeout() || e.is_connect(),
            CrawlKitError::Deserialize(_) => false,
        }
    }

    pub fn retry_after(&self) -> Option<u64> {
        match self {
            CrawlKitError::Api { retry_after, .. } => *retry_after,
            _ => None,
        }
    }
}

Client Implementation

pub struct CrawlKitClient {
    client: reqwest::Client,
    base_url: String,
}

impl CrawlKitClient {
    pub fn new(api_key: &str) -> Self {
        Self::with_base_url(api_key, "https://api.crawlkit.app/api/v1")
    }

    pub fn with_base_url(api_key: &str, base_url: &str) -> Self {
        use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE};

        let mut headers = HeaderMap::new();
        headers.insert(
            AUTHORIZATION,
            HeaderValue::from_str(&format!("Bearer {api_key}")).unwrap(),
        );
        headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));

        let client = reqwest::Client::builder()
            .default_headers(headers)
            .timeout(std::time::Duration::from_secs(30))
            .build()
            .expect("failed to build HTTP client");

        Self {
            client,
            base_url: base_url.to_string(),
        }
    }

    async fn request<T: serde::de::DeserializeOwned>(
        &self,
        method: reqwest::Method,
        path: &str,
        body: Option<&impl Serialize>,
    ) -> Result<T, CrawlKitError> {
        let url = format!("{}{}", self.base_url, path);

        let mut req = self.client.request(method, &url);
        if let Some(body) = body {
            req = req.json(body);
        }

        let response = req.send().await?;
        let status = response.status().as_u16();

        if status >= 400 {
            let error_body: serde_json::Value = response.json().await?;
            return Err(CrawlKitError::Api {
                message: error_body["message"]
                    .as_str()
                    .unwrap_or("Unknown error")
                    .to_string(),
                status,
                retryable: error_body["retryable"].as_bool().unwrap_or(false),
                retry_after: error_body["retry_after"].as_u64(),
            });
        }

        Ok(response.json().await?)
    }

    // ── Data Platform ──

    pub async fn create_dataset(
        &self,
        req: &CreateDatasetRequest,
    ) -> Result<Dataset, CrawlKitError> {
        self.request(reqwest::Method::POST, "/datasets", Some(req)).await
    }

    pub async fn add_rows(
        &self,
        dataset_id: &str,
        rows: &[serde_json::Value],
    ) -> Result<serde_json::Value, CrawlKitError> {
        let body = serde_json::json!({
            "format": "json",
            "data": rows,
        });
        self.request(
            reqwest::Method::POST,
            &format!("/datasets/{dataset_id}/rows"),
            Some(&body),
        )
        .await
    }

    pub async fn create_spider(
        &self,
        config: &SpiderConfig,
    ) -> Result<Spider, CrawlKitError> {
        self.request(reqwest::Method::POST, "/spiders", Some(config)).await
    }

    pub async fn enrich(
        &self,
        dataset_id: &str,
        source_name: &str,
        input_mapping: &serde_json::Value,
        output_mapping: &serde_json::Value,
    ) -> Result<EnrichmentJob, CrawlKitError> {
        let body = serde_json::json!({
            "dataset_id": dataset_id,
            "source_name": source_name,
            "input_mapping": input_mapping,
            "output_mapping": output_mapping,
        });
        self.request(reqwest::Method::POST, "/enrichment/enrich", Some(&body)).await
    }

    pub async fn run_pipeline(
        &self,
        pipeline_id: &str,
    ) -> Result<PipelineRun, CrawlKitError> {
        self.request::<PipelineRun>(
            reqwest::Method::POST,
            &format!("/pipelines/{pipeline_id}/run"),
            None::<&serde_json::Value>.as_ref(),
        )
        .await
    }

    // ── SEO Analysis ──

    pub async fn check_url(
        &self,
        url: &str,
        js_rendering: bool,
    ) -> Result<SeoCheckResult, CrawlKitError> {
        let body = serde_json::json!({
            "url": url,
            "js_rendering": js_rendering,
        });
        self.request(reqwest::Method::POST, "/check-url", Some(&body)).await
    }

    pub async fn audit_site(
        &self,
        site_url: &str,
        max_pages: u32,
    ) -> Result<AuditResult, CrawlKitError> {
        let body = serde_json::json!({
            "site_url": site_url,
            "max_pages": max_pages,
        });
        self.request(reqwest::Method::POST, "/audit/run", Some(&body)).await
    }

    pub async fn dns_audit(
        &self,
        domain: &str,
    ) -> Result<serde_json::Value, CrawlKitError> {
        let body = serde_json::json!({"domain": domain});
        self.request(reqwest::Method::POST, "/dns/audit", Some(&body)).await
    }
}

Examples

Create a Dataset and Add Rows

#[tokio::main]
async fn main() -> Result<(), CrawlKitError> {
    let client = CrawlKitClient::new("ck_live_YOUR_API_KEY");

    // Create a dataset for web scraping results
    let dataset = client
        .create_dataset(&CreateDatasetRequest {
            name: "ecommerce_products".into(),
            description: Some("Product catalog from competitor sites".into()),
            columns: vec![
                Column { name: "product_name".into(), data_type: ColumnType::Text },
                Column { name: "price".into(), data_type: ColumnType::Float },
                Column { name: "category".into(), data_type: ColumnType::Text },
                Column { name: "in_stock".into(), data_type: ColumnType::Boolean },
                Column { name: "specs".into(), data_type: ColumnType::Jsonb },
            ],
        })
        .await?;

    println!("Dataset created: {}", dataset.id);

    // Add rows
    let rows = vec![
        serde_json::json!({
            "product_name": "Mechanical Keyboard",
            "price": 129.99,
            "category": "Electronics",
            "in_stock": true,
            "specs": {"switches": "Cherry MX Brown", "layout": "TKL"},
        }),
        serde_json::json!({
            "product_name": "4K Monitor",
            "price": 449.99,
            "category": "Electronics",
            "in_stock": true,
            "specs": {"resolution": "3840x2160", "panel": "IPS", "refresh": "60Hz"},
        }),
    ];

    let result = client.add_rows(&dataset.id, &rows).await?;
    println!("Inserted: {:?}", result);

    Ok(())
}

Deploy a Spider and Run Enrichment

#[tokio::main]
async fn main() -> Result<(), CrawlKitError> {
    let client = CrawlKitClient::new("ck_live_YOUR_API_KEY");

    // Create a spider for job listings
    let spider = client
        .create_spider(&SpiderConfig {
            name: "job_scraper".into(),
            scope: SpiderScope {
                start_urls: vec!["https://www.ycombinator.com/jobs".into()],
            },
            definition: SpiderDefinition {
                handlers: vec![SpiderHandler {
                    url_pattern: "/jobs/*".into(),
                    fields: vec![
                        SpiderField {
                            name: "title".into(),
                            selector: "h1.job-title".into(),
                            field_type: "text".into(),
                            attribute: None,
                        },
                        SpiderField {
                            name: "company".into(),
                            selector: ".company-name".into(),
                            field_type: "text".into(),
                            attribute: None,
                        },
                        SpiderField {
                            name: "apply_url".into(),
                            selector: "a.apply-btn".into(),
                            field_type: "attribute".into(),
                            attribute: Some("href".into()),
                        },
                    ],
                }],
                navigation: Some(SpiderNavigation {
                    follow_links: Some(".pagination a".into()),
                    max_depth: Some(5),
                }),
            },
            dataset_id: Some("ds_a1b2c3d4e5f6".into()),
        })
        .await?;

    println!("Spider {}: {}", spider.id, spider.status);

    // Enrich dataset with company data
    let job = client
        .enrich(
            "ds_a1b2c3d4e5f6",
            "company_data",
            &serde_json::json!({"url": "domain"}),
            &serde_json::json!({"company_name": "company", "employee_count": "employees"}),
        )
        .await?;

    println!("Enrichment job {}: {}", job.job_id, job.status);

    Ok(())
}

Check a URL for SEO Issues

#[tokio::main]
async fn main() -> Result<(), CrawlKitError> {
    let client = CrawlKitClient::new("ck_live_YOUR_API_KEY");

    // Ship code that Google actually indexes
    let result = client.check_url("https://crawlkit.app", false).await?;

    println!("SEO Score: {}/100", result.score);
    println!("Issues: {}", result.issues.len());

    for issue in &result.issues {
        if issue.severity == "critical" || issue.severity == "high" {
            println!("  [{}] {}", issue.severity.to_uppercase(), issue.message);
        }
    }

    Ok(())
}

Error Handling with Retry

use std::time::Duration;
use tokio::time::sleep;

async fn with_retry<T, F, Fut>(f: F, max_retries: u32) -> Result<T, CrawlKitError>
where
    F: Fn() -> Fut,
    Fut: std::future::Future<Output = Result<T, CrawlKitError>>,
{
    let mut attempt = 0;
    loop {
        match f().await {
            Ok(result) => return Ok(result),
            Err(e) if e.is_retryable() && attempt < max_retries => {
                let delay = e.retry_after().unwrap_or(2u64.pow(attempt));
                eprintln!("Retry {}/{max_retries} in {delay}s: {e}");
                sleep(Duration::from_secs(delay)).await;
                attempt += 1;
            }
            Err(e) => return Err(e),
        }
    }
}

// Usage
#[tokio::main]
async fn main() -> Result<(), CrawlKitError> {
    let client = CrawlKitClient::new("ck_live_YOUR_API_KEY");

    let result = with_retry(
        || client.check_url("https://crawlkit.app", false),
        3,
    )
    .await?;

    println!("Score: {}", result.score);
    Ok(())
}