curl Examples

Copy-paste curl commands for every major CrawlKit API endpoint. No SDK required — just curl, an API key, and optionally jq for parsing responses.

Setup

Set your API key as an environment variable so you do not have to paste it into every command:

# Add to your .bashrc, .zshrc, or .env file
export CRAWLKIT_API_KEY="ck_live_YOUR_API_KEY"

# Verify it works
curl https://api.crawlkit.app/api/v1/auth/validate \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .
Install jq for readable JSON output: brew install jq (macOS), apt install jq (Ubuntu), or download from jqlang.github.io.

Data Platform

Datasets

# Create a dataset
curl -X POST https://api.crawlkit.app/api/v1/datasets \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "competitor_analysis",
    "description": "Competitor domains with traffic and tech data",
    "columns": [
      {"name": "domain", "data_type": "text"},
      {"name": "monthly_traffic", "data_type": "integer"},
      {"name": "domain_rating", "data_type": "float"},
      {"name": "tech_stack", "data_type": "jsonb"},
      {"name": "last_crawled", "data_type": "timestamp"}
    ]
  }' | jq .

# List all datasets
curl https://api.crawlkit.app/api/v1/datasets \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '.[] | {id, name, row_count}'

# Get dataset details
curl https://api.crawlkit.app/api/v1/datasets/ds_a1b2c3d4e5f6 \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Add rows to a dataset
curl -X POST https://api.crawlkit.app/api/v1/datasets/ds_a1b2c3d4e5f6/rows \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "format": "json",
    "data": [
      {
        "domain": "stripe.com",
        "monthly_traffic": 45000000,
        "domain_rating": 92.5,
        "tech_stack": {"frameworks": ["React", "Ruby on Rails"]},
        "last_crawled": "2026-03-13T10:00:00Z"
      },
      {
        "domain": "linear.app",
        "monthly_traffic": 2500000,
        "domain_rating": 78.3,
        "tech_stack": {"frameworks": ["React", "Next.js"]},
        "last_crawled": "2026-03-13T10:00:00Z"
      }
    ]
  }' | jq .

# List rows with pagination
curl "https://api.crawlkit.app/api/v1/datasets/ds_a1b2c3d4e5f6/rows?limit=20&offset=0" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '.rows[] | .data'

# Export dataset as CSV
curl "https://api.crawlkit.app/api/v1/datasets/ds_a1b2c3d4e5f6/export?format=csv" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Get dataset statistics
curl https://api.crawlkit.app/api/v1/datasets/ds_a1b2c3d4e5f6/stats \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

Spiders

# Create a spider to extract product data
curl -X POST https://api.crawlkit.app/api/v1/spiders \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "product_scraper",
    "scope": {
      "start_urls": ["https://books.toscrape.com/catalogue/category/books/travel_2/index.html"]
    },
    "definition": {
      "handlers": [
        {
          "url_pattern": "/products/*",
          "fields": [
            {"name": "title", "selector": "h1.product-title", "type": "text"},
            {"name": "price", "selector": ".price-current", "type": "text"},
            {"name": "description", "selector": ".product-desc", "type": "text"},
            {"name": "image_url", "selector": "img.product-image", "type": "attribute", "attribute": "src"}
          ]
        }
      ],
      "navigation": {
        "follow_links": ".pagination a",
        "max_depth": 3
      }
    },
    "dataset_id": "ds_a1b2c3d4e5f6"
  }' | jq .

# List all spiders
curl https://api.crawlkit.app/api/v1/spiders \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '.[] | {id, name, status}'

# Get spider details
curl https://api.crawlkit.app/api/v1/spiders/sp_x7y8z9w0 \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Test a spider (dry run)
curl -X POST https://api.crawlkit.app/api/v1/spiders/sp_x7y8z9w0/test \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html"}' | jq .

# Check spider health
curl https://api.crawlkit.app/api/v1/spiders/sp_x7y8z9w0/health \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

Enrichment

# List available enrichment sources
curl https://api.crawlkit.app/api/v1/enrichment/sources \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '.[] | {name, description}'

# Run enrichment on a dataset
curl -X POST https://api.crawlkit.app/api/v1/enrichment/enrich \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "dataset_id": "ds_a1b2c3d4e5f6",
    "source_name": "tech_stack_detector",
    "input_mapping": {"url": "domain"},
    "output_mapping": {"technologies": "tech_stack"}
  }' | jq .

# Check enrichment job status
curl https://api.crawlkit.app/api/v1/enrichment/jobs/enr_m1n2o3p4 \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '{status, processed_rows, total_rows}'

Pipelines

# Create a multi-step pipeline
curl -X POST https://api.crawlkit.app/api/v1/pipelines \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "lead_generation",
    "steps": [
      {
        "type": "crawl",
        "config": {"spider_id": "sp_x7y8z9w0", "max_pages": 500}
      },
      {
        "type": "enrich",
        "config": {
          "source_name": "company_data",
          "input_mapping": {"url": "domain"},
          "output_mapping": {"revenue": "estimated_revenue", "employees": "employee_count"}
        }
      },
      {
        "type": "filter",
        "config": {"conditions": [{"field": "employee_count", "operator": "gte", "value": 50}]}
      },
      {
        "type": "export",
        "config": {"format": "csv", "destination": "webhook", "webhook_url": "https://your-app.com/api/leads"}
      }
    ]
  }' | jq .

# List pipelines
curl https://api.crawlkit.app/api/v1/pipelines \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '.[] | {id, name, status}'

# Run a pipeline
curl -X POST https://api.crawlkit.app/api/v1/pipelines/pl_abc123/run \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Check pipeline run status
curl https://api.crawlkit.app/api/v1/pipelines/pl_abc123/runs/run_xyz789 \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

Crawl Jobs

# Start a crawl job
curl -X POST https://api.crawlkit.app/api/v1/crawl/jobs \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "site_url": "https://crawlkit.app",
    "spider_id": "sp_x7y8z9w0",
    "budget": {"max_pages": 100}
  }' | jq .

# List crawl jobs for a site
curl "https://api.crawlkit.app/api/v1/crawl/jobs?site_url=https://crawlkit.app" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '.[] | {job_id, status, pages_crawled}'

# Get crawl job status
curl https://api.crawlkit.app/api/v1/crawl/jobs/cj_abc123 \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Pause a crawl
curl -X POST https://api.crawlkit.app/api/v1/crawl/jobs/cj_abc123/pause \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Resume a crawl
curl -X POST https://api.crawlkit.app/api/v1/crawl/jobs/cj_abc123/resume \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Cancel a crawl
curl -X POST https://api.crawlkit.app/api/v1/crawl/jobs/cj_abc123/cancel \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# View crawl frontier
curl https://api.crawlkit.app/api/v1/crawl/frontier/cj_abc123 \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Create a crawl strategy
curl -X POST https://api.crawlkit.app/api/v1/crawl/strategies \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "deep_crawl",
    "config": {
      "base_strategy": "breadth_first",
      "max_depth": 10
    }
  }' | jq .

SEO Analysis

Ship code that Google actually indexes.

Check a URL

# Quick SEO check on a single URL
curl -X POST https://api.crawlkit.app/api/v1/check-url \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://crawlkit.app",
    "js_rendering": false
  }' | jq '{score, issues: [.issues[] | {severity, category, message}]}'

# Check with JS rendering enabled (for SPAs)
curl -X POST https://api.crawlkit.app/api/v1/check-url \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://vercel.com", "js_rendering": true}' | jq .

# Analyze raw HTML content
curl -X POST https://api.crawlkit.app/api/v1/check \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://crawlkit.app",
    "content": "<html><head><title>Test</title></head><body><h1>Hello</h1></body></html>"
  }' | jq .

Run a Full Site Audit

# Run an authenticated site audit (persists results)
curl -X POST https://api.crawlkit.app/api/v1/audit/run \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "site_url": "https://crawlkit.app",
    "max_pages": 100,
    "js_rendering": false,
    "include_link_analysis": true
  }' | jq '{audit_id, score, pages: .crawl_stats.pages_crawled, issues: (.issues | length)}'

# Get site health status
curl "https://api.crawlkit.app/api/v1/status?site_url=https://crawlkit.app" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

URL Inspection

# Inspect a URL for crawlability
curl -X POST https://api.crawlkit.app/api/v1/inspection/url \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://crawlkit.app"}' | jq .

# Inspect HTML content
curl -X POST https://api.crawlkit.app/api/v1/inspection/html \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://crawlkit.app", "html": "<html>...</html>"}' | jq .

# Test CSS/XPath selectors against a URL
curl -X POST https://api.crawlkit.app/api/v1/inspection/selectors \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://crawlkit.app",
    "selectors": [
      {"name": "title", "selector": "h1", "type": "css"},
      {"name": "links", "selector": "//a/@href", "type": "xpath"}
    ]
  }' | jq .

Google Search Console

# Get OAuth authorization URL
curl -X POST https://api.crawlkit.app/api/v1/gsc/auth/url \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "redirect_uri": "https://your-app.com/callback",
    "scopes": ["webmasters.readonly", "indexing"]
  }' | jq .authorization_url

# Exchange OAuth code
curl -X POST https://api.crawlkit.app/api/v1/gsc/auth/callback \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"code": "4/0OAUTH_CODE", "redirect_uri": "https://your-app.com/callback"}' | jq .

# List GSC properties
curl https://api.crawlkit.app/api/v1/gsc/properties \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '.[] | {property, permission_level}'

# URL inspection via GSC
curl -X POST https://api.crawlkit.app/api/v1/gsc/inspect \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "site_url": "https://crawlkit.app",
    "url": "https://docs.crawlkit.app/guides/spiders.html"
  }' | jq '{coverage: .index_status.coverage_state, last_crawl: .index_status.last_crawl_time}'

# Batch URL inspection (up to 50 URLs)
curl -X POST https://api.crawlkit.app/api/v1/gsc/inspect/batch \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "site_url": "https://crawlkit.app",
    "urls": [
      "https://crawlkit.app/",
      "https://docs.crawlkit.app/guides/",
      "https://crawlkit.app"
    ]
  }' | jq '.[] | {url, coverage: .index_status.coverage_state}'

# Search analytics
curl -X POST https://api.crawlkit.app/api/v1/gsc/analytics \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "site_url": "https://crawlkit.app",
    "start_date": "2026-02-01",
    "end_date": "2026-03-01",
    "dimensions": ["query"],
    "row_limit": 25
  }' | jq '.rows[:10] | .[] | {query: .keys[0], clicks, impressions, ctr, position}'

# List sitemaps
curl "https://api.crawlkit.app/api/v1/gsc/sitemaps?site_url=https://crawlkit.app" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Submit URL for indexing
curl -X POST https://api.crawlkit.app/api/v1/gsc/index \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://docs.crawlkit.app/platform.html"}' | jq .

# Get property snapshot
curl "https://api.crawlkit.app/api/v1/gsc/snapshot?site_url=https://crawlkit.app" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

Leads & Campaigns

# Score a lead by domain
curl -X POST https://api.crawlkit.app/api/v1/leads/score \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "domain": "www.ycombinator.com/companies/stripe",
    "signals": ["high_traffic", "uses_react", "hiring_engineers"]
  }' | jq .

# List leads with filters
curl "https://api.crawlkit.app/api/v1/leads?score_min=70&limit=20" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '.[] | {domain, score, signals}'

# Create a campaign
curl -X POST https://api.crawlkit.app/api/v1/campaigns \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "React SaaS Outreach",
    "filters": {"tech_stack_contains": "React", "employee_min": 50},
    "dataset_id": "ds_a1b2c3d4e5f6"
  }' | jq .

Content Workspace

# Generate a content brief
curl -X POST https://api.crawlkit.app/api/v1/brief \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "keyword": "web scraping best practices",
    "site_url": "https://crawlkit.app"
  }' | jq .

# Generate JSON-LD structured data
curl -X POST https://api.crawlkit.app/api/v1/schema \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://docs.crawlkit.app/guides//web-scraping",
    "site_url": "https://crawlkit.app"
  }' | jq .

# Get internal link suggestions
curl -X POST https://api.crawlkit.app/api/v1/links \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://docs.crawlkit.app/guides//web-scraping",
    "site_url": "https://crawlkit.app"
  }' | jq '.suggestions[] | {target: .target_url, anchor: .anchor_text, reason}'

# SERP analysis for a keyword
curl "https://api.crawlkit.app/api/v1/competitors?keyword=web+scraping+api&site_url=https://crawlkit.app" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

Social Monitoring

# Create a Reddit monitor
curl -X POST https://api.crawlkit.app/api/v1/social/reddit/monitors \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "Brand Mentions",
    "keywords": ["crawlkit", "crawl kit"],
    "subreddits": ["webdev", "SEO", "datascience"],
    "site_url": "https://crawlkit.app"
  }' | jq .

# List Reddit monitors
curl "https://api.crawlkit.app/api/v1/social/reddit/monitors?site_url=https://crawlkit.app" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Poll a monitor for new mentions
curl -X POST https://api.crawlkit.app/api/v1/social/reddit/monitors/mon_abc123/poll \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# List mentions with sentiment
curl "https://api.crawlkit.app/api/v1/social/reddit/mentions?monitor_id=mon_abc123" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '.[] | {title, sentiment, score, subreddit}'

# Create a LinkedIn monitor
curl -X POST https://api.crawlkit.app/api/v1/social/linkedin/monitors \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "Industry Signals",
    "keywords": ["data enrichment", "web intelligence"],
    "site_url": "https://crawlkit.app"
  }' | jq .

DNS & Infrastructure

# Full DNS audit
curl -X POST https://api.crawlkit.app/api/v1/dns/audit \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"domain": "crawlkit.app"}' | jq .

# SSL/TLS validation
curl -X POST https://api.crawlkit.app/api/v1/dns/ssl \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"domain": "crawlkit.app"}' | jq '{valid: .certificate.valid, issuer: .certificate.issuer, expires: .certificate.not_after}'

# Trace redirect chain
curl -X POST https://api.crawlkit.app/api/v1/dns/redirects \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://crawlkit.app"}' | jq '.chain[] | {url, status_code}'

# Email authentication check (SPF/DKIM/DMARC)
curl -X POST https://api.crawlkit.app/api/v1/dns/email-auth \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"domain": "crawlkit.app"}' | jq '{spf: .spf.valid, dkim: .dkim.valid, dmarc: .dmarc.valid}'

Keywords & Feedback

# Import keywords from Google Search Console
curl -X POST https://api.crawlkit.app/api/v1/keywords/import/gsc \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "site_url": "https://crawlkit.app",
    "days": 28
  }' | jq .

# List keywords with filters
curl "https://api.crawlkit.app/api/v1/keywords?site_url=https://crawlkit.app&min_clicks=10" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '.[] | {query, clicks, impressions, position}'

# Record a fix for feedback loop
curl -X POST https://api.crawlkit.app/api/v1/feedback/record \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "site_url": "https://crawlkit.app",
    "event_type": "fix_applied",
    "loop_type": "meta_description",
    "url": "https://docs.crawlkit.app/guides/spiders.html",
    "details": {"fix": "Added meta description", "issue_category": "missing_meta_description"}
  }' | jq .

# Check feedback verification status
curl "https://api.crawlkit.app/api/v1/feedback/status?site_url=https://crawlkit.app" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# View feedback stats
curl "https://api.crawlkit.app/api/v1/feedback/stats?site_url=https://crawlkit.app" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '{success_rate, resolved, pending, avg_resolution_hours}'

IndexNow

# Generate an IndexNow key
curl -X POST https://api.crawlkit.app/api/v1/indexnow/key/generate \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"site_url": "https://crawlkit.app"}' | jq .

# Verify key accessibility
curl "https://api.crawlkit.app/api/v1/indexnow/key/verify?site_url=https://crawlkit.app&key=YOUR_KEY" \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Submit URLs to IndexNow (Bing, Yandex, Naver, Seznam)
curl -X POST https://api.crawlkit.app/api/v1/indexnow/submit \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "site_url": "https://crawlkit.app",
    "urls": [
      "https://docs.crawlkit.app/platform.html",
      "https://docs.crawlkit.app/capabilities.html"
    ],
    "key": "YOUR_INDEXNOW_KEY"
  }' | jq .

Authentication & Usage

# Validate API key and check tier
curl -X POST https://api.crawlkit.app/api/v1/auth/validate \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq '{tier, usage: .usage}'

# Check current usage
curl https://api.crawlkit.app/api/v1/auth/usage \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY" | jq .

# Health check (no auth required)
curl https://api.crawlkit.app/health | jq .

Shell Script Patterns

Batch URL Check

#!/bin/bash
# Check multiple URLs and output a summary

URLS=(
  "https://crawlkit.app"
  "https://crawlkit.app"
  "https://crawlkit.app"
  "https://docs.crawlkit.app/guides/"
)

echo "URL,Score,Issues"

for url in "${URLS[@]}"; do
  result=$(curl -s -X POST https://api.crawlkit.app/api/v1/check-url \
    -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
    -H "Content-Type: application/json" \
    -d "{\"url\": \"$url\"}")

  score=$(echo "$result" | jq -r '.score')
  issues=$(echo "$result" | jq -r '.issues | length')
  echo "$url,$score,$issues"

  # Rate limiting: 1 request per second
  sleep 1
done

Dataset Backup Script

#!/bin/bash
# Export all datasets to CSV files

datasets=$(curl -s https://api.crawlkit.app/api/v1/datasets \
  -H "Authorization: Bearer $CRAWLKIT_API_KEY")

echo "$datasets" | jq -r '.[] | .id' | while read -r id; do
  name=$(echo "$datasets" | jq -r ".[] | select(.id == \"$id\") | .name")
  echo "Exporting $name ($id)..."

  curl -s "https://api.crawlkit.app/api/v1/datasets/$id/export?format=csv" \
    -H "Authorization: Bearer $CRAWLKIT_API_KEY" \
    -o "${name}_$(date +%Y%m%d).csv"
done

echo "Done. All datasets exported."

Monitor Enrichment Job

#!/bin/bash
# Poll an enrichment job until completion

JOB_ID="$1"
if [ -z "$JOB_ID" ]; then
  echo "Usage: $0 <job_id>"
  exit 1
fi

while true; do
  status=$(curl -s "https://api.crawlkit.app/api/v1/enrichment/jobs/$JOB_ID" \
    -H "Authorization: Bearer $CRAWLKIT_API_KEY")

  state=$(echo "$status" | jq -r '.status')
  processed=$(echo "$status" | jq -r '.processed_rows')
  total=$(echo "$status" | jq -r '.total_rows')

  echo "[$state] $processed/$total rows"

  if [ "$state" = "completed" ] || [ "$state" = "failed" ]; then
    echo "Final status: $state"
    echo "$status" | jq .
    break
  fi

  sleep 5
done
PreviousRust SDK NextAPI Reference