Python SDK

pip install webclaw

from webclaw import Webclaw

client = Webclaw("wc_your_api_key")

client = Webclaw(
    "wc_your_api_key",
    base_url="https://api.webclaw.io",  # default
    timeout=60.0,                        # seconds
)

with Webclaw("wc_your_api_key") as client:
    result = client.scrape("https://example.com")
    print(result.markdown)

from webclaw import AsyncWebclaw

async with AsyncWebclaw("wc_your_api_key") as client:
    result = await client.scrape("https://example.com", formats=["markdown"])
    print(result.markdown)

result = client.scrape(
    "https://example.com",
    formats=["markdown", "text", "llm"],
    include_selectors=["article", ".content"],
    exclude_selectors=["nav", "footer"],
    only_main_content=True,
    no_cache=True,
)

result.url        # str
result.markdown   # str | None
result.text       # str | None
result.llm        # str | None
result.metadata   # dict
result.cache      # CacheInfo | None
result.warning    # str | None

job = client.crawl(
    "https://example.com",
    max_depth=3,
    max_pages=100,
    use_sitemap=True,
)

# Block until complete (polls every 2s, 5min timeout)
status = job.wait(interval=2.0, timeout=300.0)

if status.status != "completed":
    raise RuntimeError("Crawl failed")
print(status.status)
print(status.total)      # total pages discovered
print(status.completed)  # pages successfully crawled
print(status.errors)     # pages that errored

for page in status.pages:
    print(page.url, len(page.markdown or ""))

job = await client.crawl("https://example.com", max_pages=50)
status = await job.wait()

result = client.map("https://example.com")
print(result.count)
for url in result.urls:
    print(url)

result = client.batch(
    ["https://a.com", "https://b.com", "https://c.com"],
    formats=["markdown"],
    concurrency=5,
)
for item in result.results:
    if item.error:
        print(f"FAIL {item.url}: {item.error}")
    else:
        print(f"OK   {item.url}: {len(item.markdown or '')} chars")

result = client.extract(
    "https://example.com/pricing",
    schema={
        "type": "object",
        "properties": {
            "plans": {
                "type": "array",
                "items": {
                    "type": "object",
                    "properties": {
                        "name": {"type": "string"},
                        "price": {"type": "string"},
                    },
                },
            }
        },
    },
)
print(result.data)  # {"plans": [{"name": "Pro", "price": "$49/mo"}, ...]}

result = client.extract(
    "https://example.com",
    prompt="Extract all pricing tiers with names and monthly prices",
)
print(result.data)

result = client.summarize("https://example.com", max_sentences=3)
print(result.summary)

result = client.brand("https://example.com")
print(result.data)  # {"colors": [...], "fonts": [...], "logo_url": "..."}

from webclaw.errors import (
    WebclawError,
    AuthenticationError,
    RateLimitError,
    TimeoutError,
)

try:
    result = client.scrape("https://example.com")
except AuthenticationError:
    print("Check your API key")
except RateLimitError:
    print("Slow down or upgrade your plan")
except TimeoutError:
    print("Request took too long")
except WebclawError as e:
    print(f"API error ({e.status_code}): {e}")