AI Web Scraper API for Structured Web Data

curl -G "https://app.scrapingbee.com/api/v1" \
  -H "Authorization: Bearer YOUR-API-KEY" \
  --data-urlencode "url=https://www.scrapingbee.com/blog/" \
  --data-urlencode 'ai_extract_rules={
    "title": "title of the blog post",
    "summary": "a 5 sentences summary of the blog post"
  }' \
  --data-urlencode "ai_selector=#content" \
  --data-urlencode "render_js=true"

# pip install requests
import json
import os

import requests

response = requests.get(
    "https://app.scrapingbee.com/api/v1",
    headers={
        "Authorization": f"Bearer {os.environ['SCRAPINGBEE_API_KEY']}",
    },
    params={
        "url": "https://www.scrapingbee.com/blog/",
        "ai_extract_rules": json.dumps({
            "title": "title of the blog post",
            "summary": "a 5 sentences summary of the blog post",
        }),
        "ai_selector": "#content",
        "render_js": "true",
    },
    timeout=120,
)

response.raise_for_status()
print(json.dumps(response.json(), indent=2, ensure_ascii=False))

const axios = require('axios');

axios.get('https://app.scrapingbee.com/api/v1', {
    params: {
        url: 'https://www.scrapingbee.com/blog/',
        ai_extract_rules: JSON.stringify({
            title: 'title of the blog post',
            summary: 'a 5 sentences summary of the blog post'
        }),
        ai_selector: '#content',
        render_js: 'true'
    },
    headers: {
        Authorization: `Bearer ${process.env.SCRAPINGBEE_API_KEY}`
    },
    timeout: 120000
}).then(response => {
    console.log(JSON.stringify(response.data, null, 2));
});

# pip install scrapingbee-cli
scrapingbee scrape "https://www.scrapingbee.com/blog/" \
  --ai-extract-rules '{
    "title": "title of the blog post",
    "summary": "a 5 sentences summary of the blog post"
  }' \
  --ai-selector "#content" \
  --render-js true

{
  "title": "How to web scrape with ScrapingBee",
  "summary": "We help you get better at web-scraping: detailed tutorial, case studies and writing by industry experts"
}