Web scraping with Python: requests, BeautifulSoup, Playwright or an API

# Fetch a static page with requests and pull fields out of it with BeautifulSoup.
# pip install requests beautifulsoup4
import requests
from bs4 import BeautifulSoup

URL = "https://books.toscrape.com/catalogue/category/books/travel_2/index.html"

response = requests.get(URL, headers={"User-Agent": "my-scraper/1.0 (you@example.com)"}, timeout=30)
response.raise_for_status()
# pass the raw bytes: BeautifulSoup reads the page's own charset (response.text would guess, and "£" becomes "£")
soup = BeautifulSoup(response.content, "html.parser")

books = []
for card in soup.select("article.product_pod"):
    books.append({
        "title": card.h3.a["title"],
        "price": card.select_one(".price_color").get_text(strip=True),
        "in_stock": "In stock" in card.select_one(".availability").get_text(),
    })

print(f"{len(books)} books on the page")
for book in books[:5]:
    print(book)

# Follow "next" links across a listing, politely, and save every record to a CSV file.
# pip install requests beautifulsoup4
import csv
import time
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup

url = "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html"
session = requests.Session()
session.headers["User-Agent"] = "my-scraper/1.0 (you@example.com)"
rows, pages = [], 0

while url and pages < 5:  # a hard cap, so a bug cannot crawl forever
    soup = BeautifulSoup(session.get(url, timeout=30).content, "html.parser")
    pages += 1
    for card in soup.select("article.product_pod"):
        rows.append({"title": card.h3.a["title"], "price": card.select_one(".price_color").get_text(strip=True),
                     "url": urljoin(url, card.h3.a["href"])})
    next_link = soup.select_one("li.next a")
    url = urljoin(url, next_link["href"]) if next_link else None
    time.sleep(1)  # one request a second is plenty for a small site

with open("mystery-books.csv", "w", newline="", encoding="utf-8") as f:
    writer = csv.DictWriter(f, fieldnames=["title", "price", "url"])
    writer.writeheader()
    writer.writerows(rows)

print(f"{len(rows)} books from {pages} pages saved to mystery-books.csv")
print(open("mystery-books.csv", encoding="utf-8").read().splitlines()[1])

# A page that builds its content with JavaScript: requests sees none of it, a real browser does.
# pip install playwright    python -m playwright install chromium
import requests
from playwright.sync_api import sync_playwright

URL = "https://quotes.toscrape.com/js/"

html = requests.get(URL, timeout=30).text
print(f"requests: {html.count('class=\"quote\"')} quotes in the HTML it received")

with sync_playwright() as p:
    browser = p.chromium.launch()
    page = browser.new_page()
    page.goto(URL)
    page.wait_for_selector("div.quote")  # wait for the content, not for a fixed number of seconds
    quotes = [
        {"text": q.locator(".text").inner_text(), "author": q.locator(".author").inner_text()}
        for q in page.locator("div.quote").all()
    ]
    browser.close()

print(f"playwright: {len(quotes)} quotes")
print(quotes[0])

# The same JavaScript page through a scraping API: one request, and the API decides whether a browser is needed.
# pip install httpx    export SKRYP_API_KEY=...
import os

import httpx

API = os.environ.get("SKRYP_API_URL", "https://api.skryp.dev")
response = httpx.post(
    f"{API}/v1/extract",
    headers={"Authorization": f"Bearer {os.environ['SKRYP_API_KEY']}"},
    json={
        "url": "https://quotes.toscrape.com/js/",
        "schema": {
            "type": "object",
            "properties": {"quotes": {"type": "array", "items": {"type": "object", "properties": {
                "text": {"type": "string"}, "author": {"type": "string"}}}}},
        },
    },
    timeout=120,
).json()

quotes = response["data"]["quotes"]
print(f"{len(quotes)} quotes, extracted by {response['method']}, values checked against the page: {response['checks']}")
print(quotes[0])
receipt = response["receipt"]
credits = receipt["credits"]  # the page's route price (0 when it came from cache) + 5 when a model did the extracting
print(f"route {receipt['route']}, cache {receipt['cache']}, {credits} credit{'' if credits == 1 else 's'}")

# One JavaScript-built page fetched four ways: what each returns, how long it takes, and what fails.
# pip install requests beautifulsoup4 playwright httpx    export SKRYP_API_KEY=...
import os
import time

import httpx
import requests
from bs4 import BeautifulSoup
from playwright.sync_api import sync_playwright

URL = "https://quotes.toscrape.com/js/"
API = os.environ.get("SKRYP_API_URL", "https://api.skryp.dev")
results = []


def run(name, fn):
    t0 = time.monotonic()
    found, size = fn()
    results.append((name, found, size, time.monotonic() - t0))


def plain_requests():
    html = requests.get(URL, timeout=30).text
    return html.count('class="quote"'), len(html)


def beautifulsoup():
    soup = BeautifulSoup(requests.get(URL, timeout=30).text, "html.parser")
    return len(soup.select("div.quote")), len(soup.get_text())


def playwright_browser():
    with sync_playwright() as p:
        browser = p.chromium.launch()
        page = browser.new_page()
        page.goto(URL)
        page.wait_for_selector("div.quote")
        found, size = page.locator("div.quote").count(), len(page.content())
        browser.close()
    return found, size


def scraping_api():
    page = httpx.post(f"{API}/v1/scrape", headers={"Authorization": f"Bearer {os.environ['SKRYP_API_KEY']}"},
                      json={"url": URL, "formats": ["markdown"], "max_age_s": 0}, timeout=120).json()
    return page["markdown"].count("by "), len(page["markdown"])


for name, fn in (("requests", plain_requests), ("requests + BeautifulSoup", beautifulsoup),
                 ("Playwright", playwright_browser), ("Skryp API (Markdown)", scraping_api)):
    run(name, fn)

print(f"{'method':<26}{'quotes found':>13}{'characters':>12}{'seconds':>9}")
for name, found, size, secs in results:
    print(f"{name:<26}{found:>13}{size:>12,}{secs:>9.1f}")