Web crawling vs web scraping: what's the difference?

# Crawling: start at one page and follow links to discover others. The output is a list of URLs, not data.
# pip install requests beautifulsoup4
import time
from collections import deque
from urllib.parse import urljoin, urlparse

import requests
from bs4 import BeautifulSoup

START = "https://quotes.toscrape.com/"
seen, queue, found = {START}, deque([START]), []

while queue and len(found) < 25:  # stop after 25 pages
    url = queue.popleft()
    soup = BeautifulSoup(requests.get(url, timeout=30).content, "html.parser")
    found.append(url)
    for a in soup.select("a[href]"):
        link = urljoin(url, a["href"]).split("#")[0]
        if urlparse(link).netloc == urlparse(START).netloc and link not in seen and "/login" not in link:
            seen.add(link)
            queue.append(link)
    time.sleep(0.5)

authors = [u for u in found if "/author/" in u]
print(f"crawled {len(found)} pages and saw {len(seen)} links on the site")
print(f"{len(authors)} of the crawled pages are author pages, e.g. {authors[:2]}")

# Scraping: take pages you already know and extract fields from each. The output is data.
# pip install requests beautifulsoup4
import time

import requests
from bs4 import BeautifulSoup

AUTHOR_PAGES = [
    "https://quotes.toscrape.com/author/Albert-Einstein/",
    "https://quotes.toscrape.com/author/J-K-Rowling/",
    "https://quotes.toscrape.com/author/Jane-Austen/",
]

for url in AUTHOR_PAGES:
    soup = BeautifulSoup(requests.get(url, timeout=30).content, "html.parser")
    print({
        "name": soup.select_one("h3.author-title").get_text(strip=True),
        "born": soup.select_one(".author-born-date").get_text(strip=True),
        "born_in": soup.select_one(".author-born-location").get_text(strip=True).removeprefix("in "),
    })
    time.sleep(0.5)

# Both together through an API: map a site's URLs (crawling), then extract the same fields from each (scraping).
# pip install httpx    export SKRYP_API_KEY=...
import os

import httpx

API = os.environ.get("SKRYP_API_URL", "https://api.skryp.dev")
auth = {"Authorization": f"Bearer {os.environ['SKRYP_API_KEY']}"}

site = httpx.post(f"{API}/v1/map", headers=auth, json={"url": "https://books.toscrape.com/", "limit": 200}, timeout=120).json()
urls = [link["url"] for link in site["links"]]  # each link also says where it was found: the sitemap or a page
books = [u for u in urls if "/catalogue/" in u and u.endswith("/index.html") and "/category/" not in u]
print(f"map: {len(urls)} URLs from the home page and sitemaps, {len(books)} of them book pages")

schema = {"type": "object", "properties": {"title": {"type": "string"}, "price": {"type": "string"}, "availability": {"type": "string"}}}
for url in books[:4]:
    page = httpx.post(f"{API}/v1/extract", headers=auth, json={"url": url, "schema": schema}, timeout=120).json()
    credits = page["receipt"]["credits"]  # the page, plus 5 when a model read it; learned selectors cost nothing extra
    print(page["data"], f"({page['method']}, {credits} credit{'' if credits == 1 else 's'})")