Web scraping with Selenium and Python

# Open a JavaScript-built page in Chrome with Selenium, wait for the content, and extract it.
# pip install selenium   (Selenium Manager downloads the matching ChromeDriver on first run)
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

options = webdriver.ChromeOptions()
options.add_argument("--headless=new")
driver = webdriver.Chrome(options=options)
try:
    driver.get("https://quotes.toscrape.com/js/")
    # wait until the quotes exist, up to 10 seconds; never sleep for a fixed time
    WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "div.quote")))
    quotes = [
        {"text": q.find_element(By.CSS_SELECTOR, ".text").text, "author": q.find_element(By.CSS_SELECTOR, ".author").text}
        for q in driver.find_elements(By.CSS_SELECTOR, "div.quote")
    ]
finally:
    driver.quit()  # always close the browser, even when something fails

print(f"{len(quotes)} quotes")
print(quotes[0])

# Click "Next" through a JavaScript listing, waiting for each new page to replace the old one.
# pip install selenium
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

options = webdriver.ChromeOptions()
options.add_argument("--headless=new")
driver = webdriver.Chrome(options=options)
wait = WebDriverWait(driver, 10)
authors = []
try:
    driver.get("https://quotes.toscrape.com/js/")
    for page in range(1, 4):  # first three pages
        wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "div.quote")))
        authors += [a.text for a in driver.find_elements(By.CSS_SELECTOR, "small.author")]
        print(f"page {page}: {len(authors)} quotes so far")
        try:
            first = driver.find_element(By.CSS_SELECTOR, "div.quote")
            driver.find_element(By.CSS_SELECTOR, "li.next a").click()
            wait.until(EC.staleness_of(first))  # the old page is gone: the next one is loading
        except NoSuchElementException:
            break  # no "Next" link: last page
finally:
    driver.quit()

print(f"{len(set(authors))} different authors, e.g. {sorted(set(authors))[:3]}")

# Choose options in a search form and submit it, as a person would.
# pip install selenium
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import Select, WebDriverWait

options = webdriver.ChromeOptions()
options.add_argument("--headless=new")
options.add_argument("--window-size=1280,900")  # headless windows are small; a covered button cannot be clicked
driver = webdriver.Chrome(options=options)
wait = WebDriverWait(driver, 10)
try:
    driver.get("https://quotes.toscrape.com/search.aspx")
    Select(driver.find_element(By.ID, "author")).select_by_visible_text("Albert Einstein")
    # choosing an author reloads the page with that author's tags in the second list
    wait.until(lambda d: len(Select(d.find_element(By.ID, "tag")).options) > 1)
    Select(driver.find_element(By.ID, "tag")).select_by_visible_text("inspirational")
    wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[type=submit]"))).click()
    results = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "div.quote span.content")))
    texts = [r.text for r in results]
finally:
    driver.quit()

print(f"{len(texts)} {'quote' if len(texts) == 1 else 'quotes'} by Albert Einstein tagged inspirational")
print(texts[0][:100])

# Before reaching for Selenium, look in the page source: this "JavaScript" page carries its data as JSON.
# pip install requests
import json
import re

import requests

html = requests.get("https://quotes.toscrape.com/js/", timeout=30).text
raw = re.search(r"var data = (\[.*?\]);", html, re.S).group(1)
quotes = json.loads(raw)

print(f"{len(quotes)} quotes straight from the page source, no browser")
print({"text": quotes[0]["text"], "author": quotes[0]["author"]["name"]})