JavaScript SDK

A lightweight, zero-dependency client for the CrawlKit API. Uses the standard fetch API — works in Node.js 18+, Deno, Bun, and modern browsers.

Client Setup

const CRAWLKIT_API_KEY = "ck_live_YOUR_API_KEY";
const BASE_URL = "https://api.crawlkit.app/api/v1";

async function crawlkit(method, path, body = null, params = {}) {
  const url = new URL(`${BASE_URL}${path}`);
  Object.entries(params).forEach(([k, v]) => url.searchParams.set(k, v));

  const res = await fetch(url.toString(), {
    method,
    headers: {
      "Authorization": `Bearer ${CRAWLKIT_API_KEY}`,
      "Content-Type": "application/json",
    },
    body: body ? JSON.stringify(body) : undefined,
  });

  if (!res.ok) {
    const err = await res.json();
    const error = new Error(err.message || `HTTP ${res.status}`);
    error.status = res.status;
    error.retryable = err.retryable || false;
    error.retryAfter = err.retry_after;
    throw error;
  }

  return res.json();
}

Data Platform Examples

Create a Dataset

// Create a dataset for product catalog data
const dataset = await crawlkit("POST", "/datasets", {
  name: "product_catalog",
  description: "E-commerce product data scraped from competitor sites",
  columns: [
    { name: "product_name", data_type: "text" },
    { name: "price", data_type: "float" },
    { name: "category", data_type: "text" },
    { name: "in_stock", data_type: "boolean" },
    { name: "specs", data_type: "jsonb" },
    { name: "scraped_at", data_type: "timestamp" },
  ],
});

console.log(`Dataset created: ${dataset.id}`);

Add Rows to a Dataset

// Insert rows from your application
const result = await crawlkit("POST", `/datasets/${dataset.id}/rows`, {
  format: "json",
  data: [
    {
      product_name: "Wireless Keyboard",
      price: 79.99,
      category: "Electronics",
      in_stock: true,
      specs: { connectivity: "Bluetooth 5.0", battery: "6 months" },
    },
    {
      product_name: "USB-C Hub",
      price: 49.99,
      category: "Electronics",
      in_stock: true,
      specs: { ports: 7, power_delivery: "100W" },
    },
  ],
});

console.log(`Inserted ${result.inserted} rows`);

Deploy a Spider

// Create a spider to extract pricing data from a competitor
const spider = await crawlkit("POST", "/spiders", {
  name: "competitor_pricing",
  scope: {
    start_urls: ["https://books.toscrape.com/catalogue/category/books/travel_2/index.html"],
  },
  definition: {
    handlers: [
      {
        url_pattern: "/products/*",
        fields: [
          { name: "product_name", selector: "h1.product-title", type: "text" },
          { name: "price", selector: ".current-price", type: "text" },
          { name: "category", selector: ".breadcrumb li:last-child", type: "text" },
          { name: "in_stock", selector: ".stock-status", type: "text" },
          { name: "image_url", selector: "img.product-hero", type: "attribute", attribute: "src" },
        ],
      },
    ],
    navigation: {
      follow_links: ".pagination a",
      max_depth: 3,
    },
  },
  dataset_id: dataset.id,
});

console.log(`Spider ${spider.id}: ${spider.status}`);

Run Enrichment

// Enrich a dataset with company intelligence
const job = await crawlkit("POST", "/enrichment/enrich", {
  dataset_id: dataset.id,
  source_name: "company_data",
  input_mapping: { url: "domain" },
  output_mapping: {
    company_name: "company",
    employee_count: "employees",
    industry: "industry",
  },
});

console.log(`Enrichment ${job.job_id}: ${job.status}`);

// Poll for completion
let status = job;
while (status.status === "running" || status.status === "pending") {
  await new Promise((r) => setTimeout(r, 5000));
  status = await crawlkit("GET", `/enrichment/jobs/${job.job_id}`);
  console.log(`Progress: ${status.processed_rows}/${status.total_rows}`);
}

console.log(`Enrichment complete: ${status.status}`);

Create and Run a Pipeline

// Build a multi-step data pipeline
const pipeline = await crawlkit("POST", "/pipelines", {
  name: "lead_generation_pipeline",
  steps: [
    {
      type: "crawl",
      config: { spider_id: spider.id, max_pages: 1000 },
    },
    {
      type: "enrich",
      config: {
        source_name: "tech_stack_detector",
        input_mapping: { url: "domain" },
        output_mapping: { technologies: "tech_stack" },
      },
    },
    {
      type: "filter",
      config: {
        conditions: [
          { field: "tech_stack", operator: "contains", value: "React" },
        ],
      },
    },
    {
      type: "export",
      config: { format: "json", destination: "webhook", webhook_url: "https://your-app.com/api/leads" },
    },
  ],
});

// Execute the pipeline
const run = await crawlkit("POST", `/pipelines/${pipeline.id}/run`);
console.log(`Pipeline run started: ${run.run_id}`);

SEO Analysis Examples

Check a URL

// Ship code that Google actually indexes
const check = await crawlkit("POST", "/check-url", {
  url: "https://crawlkit.app",
  js_rendering: false,
});

console.log(`Score: ${check.score}/100`);
console.log(`Issues found: ${check.issues.length}`);

// Show critical and high severity issues
check.issues
  .filter((i) => i.severity === "critical" || i.severity === "high")
  .forEach((issue) => {
    console.log(`[${issue.severity.toUpperCase()}] ${issue.message}`);
  });

Run a Site Audit

const audit = await crawlkit("POST", "/audit/run", {
  site_url: "https://crawlkit.app",
  max_pages: 50,
  js_rendering: false,
  include_link_analysis: true,
});

console.log(`Audit ${audit.audit_id}`);
console.log(`Score: ${audit.score}/100`);
console.log(`Pages crawled: ${audit.crawl_stats.pages_crawled}`);
console.log(`Issues: ${audit.issues.length}`);

Error Handling with Retry Logic

async function crawlkitWithRetry(method, path, body = null, options = {}) {
  const maxRetries = options.maxRetries || 3;
  const baseDelay = options.baseDelay || 1000;

  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      return await crawlkit(method, path, body);
    } catch (err) {
      const isLast = attempt === maxRetries;
      const isRetryable = err.retryable && !isLast;

      if (!isRetryable) {
        throw err;
      }

      // Use server-provided delay or exponential backoff
      const delay = err.retryAfter
        ? err.retryAfter * 1000
        : baseDelay * Math.pow(2, attempt) + Math.random() * 500;

      console.warn(`Retry ${attempt + 1}/${maxRetries} in ${Math.round(delay)}ms`);
      await new Promise((r) => setTimeout(r, delay));
    }
  }
}

// Usage
const result = await crawlkitWithRetry("POST", "/check-url", {
  url: "https://crawlkit.app",
});

Browser Usage

// Proxy through your backend to protect the API key
async function checkUrl(url) {
  const res = await fetch("/api/crawlkit/check-url", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ url }),
  });
  return res.json();
}

// Your backend (e.g., Express or Next.js API route)
app.post("/api/crawlkit/check-url", async (req, res) => {
  const result = await crawlkit("POST", "/check-url", {
    url: req.body.url,
  });
  res.json(result);
});