Turn Any URL Into Structured Data

import requests

API_KEY = "YOUR_API_KEY"
BASE = "https://api.anysite.io"
headers = {"access-token": API_KEY}

# Step 1: Get all URLs from sitemap
sitemap = requests.post(
    f"{BASE}/api/webparser/sitemap",
    headers=headers,
    json={"url": "https://example.com"}
).json()

print(f"Found {len(sitemap['urls'])} pages")

# Step 2: Extract content from each page
pages = []
for url_entry in sitemap["urls"][:100]:  # First 100 pages
    page = requests.post(
        f"{BASE}/api/webparser/parse",
        headers=headers,
        json={"url": url_entry["url"]}
    ).json()
    pages.append(page)
    print(f"  Extracted: {page['title']}")

import requests

API_KEY = "YOUR_API_KEY"
BASE = "https://api.anysite.io"
headers = {"access-token": API_KEY}

# Extract Glassdoor company reviews
reviews = requests.post(
    f"{BASE}/api/ai-parser/glassdoor",
    headers=headers,
    json={"url": "https://glassdoor.com/Reviews/TechCorp-Reviews-E12345.htm"}
).json()

# Extract G2 software reviews
g2_data = requests.post(
    f"{BASE}/api/ai-parser/g2",
    headers=headers,
    json={"url": "https://g2.com/products/techcorp/reviews"}
).json()

# Extract Amazon product data
product = requests.post(
    f"{BASE}/api/ai-parser/amazon",
    headers=headers,
    json={"url": "https://amazon.com/dp/B0XXXXXXX"}
).json()

# Parse any URL
curl -X POST "https://api.anysite.io/api/webparser/parse" \
  -H "access-token: YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com/blog/post", "extract_links": true}'

# Get sitemap
curl -X POST "https://api.anysite.io/api/webparser/sitemap" \
  -H "access-token: YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com"}'

# Parse any URL
anysite api /api/webparser/parse url="https://example.com/blog/post"

# Extract with links and images
anysite api /api/webparser/parse \
  url="https://example.com/pricing" \
  extract_links=true extract_images=true

# Get sitemap URLs
anysite api /api/webparser/sitemap url="https://example.com"

# Batch: parse multiple URLs
anysite api /api/webparser/parse --from-file urls.txt \
  --input-key url --parallel 5 --format csv

# AI parser
anysite api /api/ai-parser/glassdoor \
  url="https://glassdoor.com/Reviews/TechCorp-Reviews-E12345.htm"

name: site-crawler
sources:
  sitemap:
    endpoint: /api/webparser/sitemap
    input:
      url: "https://competitor.com"

  pages:
    endpoint: /api/webparser/parse
    depends_on: sitemap
    input:
      url: ${sitemap.url}
      extract_links: true
    parallel: 5
    on_error: skip

storage:
  format: parquet
  path: ./data/site-crawl