TypeScript SDK

A fully typed client for the CrawlKit API. Build data pipelines, deploy spiders, enrich datasets, and ship code that Google actually indexes — all with complete type safety.

Coming soon: npm install @crawlkit/sdk. The examples below show the planned API surface using a typed wrapper around fetch. You can use these patterns today with zero dependencies.

Installation

npm install @crawlkit/sdk

Quick Start

import { CrawlKit } from "@crawlkit/sdk";

const ck = new CrawlKit({ apiKey: "ck_live_YOUR_API_KEY" });

// Create a dataset
const dataset = await ck.datasets.create({
  name: "competitor_data",
  columns: [
    { name: "domain", data_type: "text" },
    { name: "traffic", data_type: "integer" },
    { name: "tech_stack", data_type: "jsonb" },
  ],
});
console.log(`Dataset created: ${dataset.id}`);

Type Definitions

Core types used throughout the SDK. These give you full IntelliSense and compile-time validation.

interface CrawlKitConfig {
  apiKey: string;
  baseUrl?: string;  // defaults to "https://api.crawlkit.app/api/v1"
  timeout?: number;  // defaults to 30000ms
  retries?: number;  // defaults to 3
}

interface Dataset {
  id: string;
  name: string;
  description?: string;
  columns: Column[];
  row_count: number;
  created_at: string;
}

interface Column {
  name: string;
  data_type: "text" | "integer" | "float" | "boolean" | "jsonb" | "timestamp";
}

interface DatasetRow {
  id: string;
  data: Record<string, unknown>;
  created_at: string;
}

interface SpiderConfig {
  name: string;
  scope: { start_urls: string[] };
  definition: {
    handlers: Array<{
      url_pattern: string;
      fields: Array<{
        name: string;
        selector: string;
        type: "text" | "html" | "attribute";
        attribute?: string;
      }>;
    }>;
    navigation?: { follow_links?: string; max_depth?: number };
  };
  dataset_id?: string;
}

interface Spider {
  id: string;
  name: string;
  status: "ready" | "running" | "paused" | "completed" | "failed";
  scope: { start_urls: string[] };
  dataset_id?: string;
  created_at: string;
}

interface EnrichmentJob {
  job_id: string;
  status: "pending" | "running" | "completed" | "failed";
  source: string;
  dataset_id: string;
  total_rows: number;
  processed_rows: number;
  created_at: string;
}

interface Pipeline {
  id: string;
  name: string;
  steps: PipelineStep[];
  status: "draft" | "active" | "paused";
  created_at: string;
}

interface PipelineStep {
  type: "crawl" | "extract" | "enrich" | "filter" | "export";
  config: Record<string, unknown>;
}

interface SeoCheckResult {
  url: string;
  score: number;
  issues: SeoIssue[];
  metadata: { title?: string; word_count: number; load_time_ms: number };
  remediations?: Remediation[];
}

interface SeoIssue {
  category: string;
  severity: "critical" | "high" | "medium" | "low" | "info";
  message: string;
  affected_urls: string[];
}

interface CrawlKitError {
  error: string;
  message: string;
  status: number;
  retryable: boolean;
  retry_after?: number;
}

Client Implementation

A typed wrapper around fetch you can use today. Drop this into your project and import it anywhere.

class CrawlKitClient {
  private baseUrl: string;
  private apiKey: string;
  private timeout: number;

  constructor(config: CrawlKitConfig) {
    this.apiKey = config.apiKey;
    this.baseUrl = config.baseUrl ?? "https://api.crawlkit.app/api/v1";
    this.timeout = config.timeout ?? 30000;
  }

  private async request<T>(
    method: string,
    path: string,
    body?: unknown,
    params?: Record<string, string>
  ): Promise<T> {
    const url = new URL(`${this.baseUrl}${path}`);
    if (params) {
      Object.entries(params).forEach(([k, v]) => url.searchParams.set(k, v));
    }

    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), this.timeout);

    try {
      const res = await fetch(url.toString(), {
        method,
        headers: {
          "Authorization": `Bearer ${this.apiKey}`,
          "Content-Type": "application/json",
        },
        body: body ? JSON.stringify(body) : undefined,
        signal: controller.signal,
      });

      if (!res.ok) {
        const err: CrawlKitError = await res.json();
        throw new CrawlKitApiError(err);
      }

      return (await res.json()) as T;
    } finally {
      clearTimeout(timer);
    }
  }

  // ── Data Platform ──

  async createDataset(config: {
    name: string;
    description?: string;
    columns: Column[];
  }): Promise<Dataset> {
    return this.request("POST", "/datasets", config);
  }

  async listDatasets(): Promise<Dataset[]> {
    return this.request("GET", "/datasets");
  }

  async addRows(datasetId: string, rows: Record<string, unknown>[]): Promise<{
    inserted: number;
  }> {
    return this.request("POST", `/datasets/${datasetId}/rows`, {
      format: "json",
      data: rows,
    });
  }

  async createSpider(config: SpiderConfig): Promise<Spider> {
    return this.request("POST", "/spiders", config);
  }

  async enrich(config: {
    dataset_id: string;
    source_name: string;
    input_mapping: Record<string, string>;
    output_mapping: Record<string, string>;
  }): Promise<EnrichmentJob> {
    return this.request("POST", "/enrichment/enrich", config);
  }

  async createPipeline(config: {
    name: string;
    steps: PipelineStep[];
  }): Promise<Pipeline> {
    return this.request("POST", "/pipelines", config);
  }

  async runPipeline(pipelineId: string): Promise<{
    run_id: string;
    status: string;
  }> {
    return this.request("POST", `/pipelines/${pipelineId}/run`);
  }

  // ── SEO Analysis ──

  async checkUrl(url: string, options?: {
    js_rendering?: boolean;
  }): Promise<SeoCheckResult> {
    return this.request("POST", "/check-url", { url, ...options });
  }

  async auditSite(siteUrl: string, options?: {
    max_pages?: number;
    js_rendering?: boolean;
  }): Promise<{ audit_id: string; score: number; issues: SeoIssue[] }> {
    return this.request("POST", "/audit/run", { site_url: siteUrl, ...options });
  }

  // ── Crawl Jobs ──

  async startCrawl(config: {
    site_url: string;
    spider_id: string;
    budget?: { max_pages?: number };
  }): Promise<{ job_id: string; status: string }> {
    return this.request("POST", "/crawl/jobs", config);
  }

  async getCrawlStatus(jobId: string): Promise<{
    job_id: string;
    status: string;
    pages_crawled: number;
  }> {
    return this.request("GET", `/crawl/jobs/${jobId}`);
  }
}

class CrawlKitApiError extends Error {
  status: number;
  retryable: boolean;
  retry_after?: number;

  constructor(err: CrawlKitError) {
    super(err.message);
    this.name = "CrawlKitApiError";
    this.status = err.status;
    this.retryable = err.retryable;
    this.retry_after = err.retry_after;
  }
}

Examples

Create a Dataset and Add Rows

const client = new CrawlKitClient({ apiKey: "ck_live_YOUR_API_KEY" });

// Create a dataset for lead data
const dataset = await client.createDataset({
  name: "saas_companies",
  description: "SaaS companies with tech stacks and traffic",
  columns: [
    { name: "domain", data_type: "text" },
    { name: "company_name", data_type: "text" },
    { name: "monthly_traffic", data_type: "integer" },
    { name: "tech_stack", data_type: "jsonb" },
    { name: "employee_count", data_type: "integer" },
    { name: "last_updated", data_type: "timestamp" },
  ],
});

// Add rows to the dataset
const result = await client.addRows(dataset.id, [
  {
    domain: "stripe.com",
    company_name: "Stripe",
    monthly_traffic: 45000000,
    tech_stack: { frameworks: ["React", "Ruby on Rails"], cdn: "Cloudflare" },
    employee_count: 8000,
  },
  {
    domain: "linear.app",
    company_name: "Linear",
    monthly_traffic: 2500000,
    tech_stack: { frameworks: ["React", "Next.js"], cdn: "Vercel" },
    employee_count: 120,
  },
]);

console.log(`Inserted ${result.inserted} rows`);

Deploy a Spider

// Create a spider to extract job listings
const spider = await client.createSpider({
  name: "job_board_scraper",
  scope: {
    start_urls: ["https://www.ycombinator.com/jobs"],
  },
  definition: {
    handlers: [
      {
        url_pattern: "/listings/*",
        fields: [
          { name: "title", selector: "h1.job-title", type: "text" },
          { name: "company", selector: ".company-name", type: "text" },
          { name: "location", selector: ".job-location", type: "text" },
          { name: "salary", selector: ".salary-range", type: "text" },
          { name: "description", selector: ".job-description", type: "html" },
          { name: "apply_url", selector: "a.apply-btn", type: "attribute", attribute: "href" },
        ],
      },
    ],
    navigation: {
      follow_links: ".pagination a.next",
      max_depth: 5,
    },
  },
  dataset_id: dataset.id,
});

console.log(`Spider ${spider.id} is ${spider.status}`);

Run Enrichment

// Enrich dataset rows with tech stack detection
const job = await client.enrich({
  dataset_id: dataset.id,
  source_name: "tech_stack_detector",
  input_mapping: { url: "domain" },
  output_mapping: { technologies: "tech_stack" },
});

console.log(`Enrichment job ${job.job_id}: ${job.status}`);
console.log(`Processing ${job.total_rows} rows`);

Build and Run a Pipeline

// Create a multi-step pipeline: crawl → extract → enrich → export
const pipeline = await client.createPipeline({
  name: "competitor_intelligence",
  steps: [
    {
      type: "crawl",
      config: {
        spider_id: spider.id,
        max_pages: 500,
      },
    },
    {
      type: "enrich",
      config: {
        source_name: "company_data",
        input_mapping: { url: "domain" },
        output_mapping: { revenue: "estimated_revenue", employees: "employee_count" },
      },
    },
    {
      type: "filter",
      config: {
        conditions: [{ field: "employee_count", operator: "gte", value: 50 }],
      },
    },
    {
      type: "export",
      config: {
        format: "csv",
        destination: "webhook",
        webhook_url: "https://your-app.com/api/pipeline-results",
      },
    },
  ],
});

// Run the pipeline
const run = await client.runPipeline(pipeline.id);
console.log(`Pipeline run ${run.run_id}: ${run.status}`);

Check a URL for SEO Issues

// Ship code that Google actually indexes
const result = await client.checkUrl("https://crawlkit.app", {
  js_rendering: false,
});

console.log(`SEO Score: ${result.score}/100`);
console.log(`Issues: ${result.issues.length}`);

// Filter high-severity issues
const critical = result.issues.filter(
  (i) => i.severity === "critical" || i.severity === "high"
);
for (const issue of critical) {
  console.log(`[${issue.severity.toUpperCase()}] ${issue.message}`);
}

Error Handling

try {
  const result = await client.checkUrl("https://crawlkit.app");
  console.log(`Score: ${result.score}`);
} catch (err) {
  if (err instanceof CrawlKitApiError) {
    console.error(`API Error ${err.status}: ${err.message}`);

    if (err.retryable && err.retry_after) {
      console.log(`Retrying in ${err.retry_after}s...`);
      await new Promise((r) => setTimeout(r, err.retry_after! * 1000));
      // retry the request
    }

    if (err.status === 401) {
      console.error("Invalid API key. Check your key at https://crawlkit.app");
    }

    if (err.status === 429) {
      console.error("Rate limited. Upgrade at https://crawlkit.app/pricing");
    }
  } else {
    throw err; // network error or timeout
  }
}

Retry with Exponential Backoff

async function withRetry<T>(
  fn: () => Promise<T>,
  maxRetries = 3,
  baseDelay = 1000
): Promise<T> {
  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      return await fn();
    } catch (err) {
      if (
        err instanceof CrawlKitApiError &&
        err.retryable &&
        attempt < maxRetries
      ) {
        const delay = err.retry_after
          ? err.retry_after * 1000
          : baseDelay * Math.pow(2, attempt);
        await new Promise((r) => setTimeout(r, delay));
      } else {
        throw err;
      }
    }
  }
  throw new Error("Unreachable");
}

// Usage
const result = await withRetry(() => client.checkUrl("https://crawlkit.app"));