CLI Reference

# Single URL
webclaw https://example.com

# Multiple URLs
webclaw https://example.com https://news.ycombinator.com

# From a file list
webclaw --urls-file urls.txt

# Local HTML file
webclaw --file page.html

# Pipe from another command
curl -s https://example.com | webclaw --stdin

# Default markdown
webclaw https://example.com

# LLM-optimized for minimal token usage
webclaw https://example.com -f llm

# Full JSON with metadata
webclaw https://example.com -f json

# Plain text
webclaw https://example.com -f text

# Markdown with metadata header
webclaw https://example.com --metadata

# Save JSON snapshot for later diffing
webclaw https://example.com -f json > snapshot.json

# Extract only the article body
webclaw https://example.com --include "article"

# Extract specific sections
webclaw https://example.com --include ".post-content, .comments"

# Remove navigation and footer noise
webclaw https://example.com --exclude "nav, footer, .sidebar"

# Combine both
webclaw https://example.com --include "main" --exclude ".ads, .related-posts"

# Quick mode: just the main content area
webclaw https://example.com --only-main-content

# Default Chrome impersonation
webclaw https://example.com

# Firefox fingerprint
webclaw https://example.com -b firefox

# Random profile per request (good for batch)
webclaw url1 url2 url3 -b random

# Single proxy
webclaw https://example.com -p http://user:pass@proxy.example.com:8080

# Proxy pool with rotation
webclaw https://example.com --proxy-file proxies.txt

# Batch extraction with proxy rotation
webclaw --urls-file urls.txt --proxy-file proxies.txt

# Discover what's available
webclaw extractors

# GitHub PR as typed JSON
webclaw vertical github_pr https://github.com/rust-lang/rust/pull/12345

# Reddit thread with comments
webclaw vertical reddit https://www.reddit.com/r/rust/comments/abc/title/

# Amazon product (name + price + rating)
webclaw vertical amazon_product https://www.amazon.com/dp/B0C6KKQ7ND

# Pipe into jq for the fields you need
webclaw vertical github_repo https://github.com/rust-lang/rust --raw \
  | jq '{stars, forks, primary_language}'

# Basic crawl, 1 level deep
webclaw https://docs.example.com --crawl

# Deep crawl with limits
webclaw https://docs.example.com --crawl --depth 3 --max-pages 100

# Faster crawl with more concurrency
webclaw https://docs.example.com --crawl --concurrency 10 --delay 50

# Only crawl the /api/ section
webclaw https://docs.example.com --crawl --path-prefix /api/

# Seed from sitemap first, then crawl
webclaw https://docs.example.com --crawl --depth 2 --max-pages 50 --sitemap

# Discover all URLs from sitemap
webclaw https://docs.example.com --map

# Save discovered URLs to a file, then extract them
webclaw https://docs.example.com --map > urls.txt
webclaw --urls-file urls.txt -f llm

# Take a snapshot
webclaw https://example.com -f json > snapshot.json

# Later, check what changed
webclaw https://example.com --diff-with snapshot.json

webclaw https://example.com --brand

# Summarize a page
webclaw https://example.com --summarize
webclaw https://example.com --summarize 5

# Natural language extraction
webclaw https://example.com --extract-prompt "Get all pricing tiers"
webclaw https://example.com --extract-prompt "List every author name and their role"

# JSON schema extraction
webclaw https://example.com --extract-json '{"type":"object","properties":{"title":{"type":"string"},"price":{"type":"number"}}}'

# Schema from a file
webclaw https://example.com --extract-json @schema.json

# Force OpenAI instead of Ollama
webclaw https://example.com --summarize --llm-provider openai

# Use a specific model
webclaw https://example.com --summarize --llm-provider anthropic --llm-model claude-sonnet-4-20250514

# Use an Anthropic-compatible endpoint
webclaw https://example.com --summarize \
  --llm-provider anthropic \
  --llm-base-url https://anthropic-proxy.example.com/v1

# LM Studio may reject OpenAI json_object mode.
# Use text or json_schema for OpenAI-compatible local backends.
export OPENAI_BASE_URL=http://localhost:1234/v1
export OPENAI_API_KEY=lm-studio
export OPENAI_RESPONSE_FORMAT_TYPE=text

webclaw https://example.com \
  --extract-prompt "Extract the product name and price" \
  --llm-provider openai

# Auto-detected from Content-Type
webclaw https://example.com/report.pdf

# Fast mode (skip failures silently)
webclaw https://example.com/report.pdf --pdf-mode fast

# Longer timeout for slow sites
webclaw https://slow-site.example.com -t 60

# Verbose output for debugging
webclaw https://example.com -v

# Set API key via environment
export WEBCLAW_API_KEY=wc_your_key_here

# Automatic fallback (tries local first, cloud on failure)
webclaw https://protected-site.com

# Force cloud mode
webclaw https://spa-site.com --cloud

# Pass key directly
webclaw https://example.com --cloud --api-key wc_your_key_here

webclaw https://blog.example.com/post \
  -f llm \
  --include "article" \
  --exclude ".comments, .related-posts"

webclaw https://docs.example.com \
  --crawl \
  --depth 2 \
  --max-pages 50 \
  --sitemap \
  --proxy-file proxies.txt \
  -b random \
  -f llm

webclaw https://example.com/pricing \
  --extract-json '{
    "type": "object",
    "properties": {
      "plans": {
        "type": "array",
        "items": {
          "type": "object",
          "properties": {
            "name": {"type": "string"},
            "price": {"type": "string"},
            "features": {"type": "array", "items": {"type": "string"}}
          }
        }
      }
    }
  }'

# Initial snapshot
webclaw https://example.com/status -f json > baseline.json

# Check for changes (run periodically)
webclaw https://example.com/status --diff-with baseline.json

webclaw \
  https://site-a.com \
  https://site-b.com \
  https://site-c.com \
  -b firefox \
  -f llm \
  --metadata

webclaw https://app.example.com \
  --cloud \
  --crawl \
  --depth 2 \
  --max-pages 30 \
  -f llm