Playwright vs Puppeteer vs Selenium for web scraping

// Four JavaScript-heavy tasks, each run 3 times with each library: did it get the data, and how long did it take?
// npm install playwright puppeteer selenium-webdriver
// timeout: 900
import { chromium } from "playwright";
import puppeteer from "puppeteer";
import { Builder, By, until } from "selenium-webdriver";
import chrome from "selenium-webdriver/chrome.js";

const TASKS = [
  { name: "Rendered by JavaScript", url: "https://quotes.toscrape.com/js/", want: 10 },
  { name: "Rendered after 10 s", url: "https://quotes.toscrape.com/js-delayed/", want: 10 },
  { name: "Infinite scroll (30+ items)", url: "https://quotes.toscrape.com/scroll", want: 30, scroll: true },
  { name: "Table loaded by a click", url: "https://www.scrapethissite.com/pages/ajax-javascript/", want: 6, click: "a#\\32 015", rows: "tr.film" },
];

// Each library does each task its own idiomatic way and returns how many items it got.
const LIBS = {
  Playwright: {
    async open() { const b = await chromium.launch(); return { b, page: await b.newPage() }; },
    async run({ page }, t) {
      await page.goto(t.url);
      if (t.click) { await page.click(t.click); await page.locator(t.rows).first().waitFor({ timeout: 20_000 }); return page.locator(t.rows).count(); }
      await page.locator("div.quote").first().waitFor({ timeout: 20_000 });
      while (t.scroll && (await page.locator("div.quote").count()) < t.want) { await page.mouse.wheel(0, 4000); await page.waitForTimeout(300); }
      return page.locator("div.quote").count();
    },
    close: ({ b }) => b.close(),
  },
  Puppeteer: {
    async open() { const b = await puppeteer.launch(); return { b, page: await b.newPage() }; },
    async run({ page }, t) {
      await page.goto(t.url);
      if (t.click) { await page.click(t.click); await page.waitForSelector(t.rows, { timeout: 20_000 }); return (await page.$$(t.rows)).length; }
      await page.waitForSelector("div.quote", { timeout: 20_000 });
      while (t.scroll && (await page.$$("div.quote")).length < t.want) { await page.mouse.wheel({ deltaY: 4000 }); await new Promise((r) => setTimeout(r, 300)); }
      return (await page.$$("div.quote")).length;
    },
    close: ({ b }) => b.close(),
  },
  Selenium: {
    async open() { return { d: await new Builder().forBrowser("chrome").setChromeOptions(new chrome.Options().addArguments("--headless=new")).build() }; },
    async run({ d }, t) {
      await d.get(t.url);
      if (t.click) { await d.findElement(By.css(t.click)).click(); await d.wait(until.elementLocated(By.css(t.rows)), 20_000); return (await d.findElements(By.css(t.rows))).length; }
      await d.wait(until.elementLocated(By.css("div.quote")), 20_000);
      while (t.scroll && (await d.findElements(By.css("div.quote"))).length < t.want) { await d.executeScript("window.scrollBy(0, 4000)"); await d.sleep(300); }
      return (await d.findElements(By.css("div.quote"))).length;
    },
    close: ({ d }) => d.quit(),
  },
};

const median = (xs) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)];
const rows = [];
for (const [lib, L] of Object.entries(LIBS)) {
  const launches = [];
  for (const t of TASKS) {
    const times = [];
    let ok = 0;
    for (let i = 0; i < 3; i++) {
      let t0 = performance.now();
      const s = await L.open();
      launches.push(performance.now() - t0);
      t0 = performance.now();
      try {
        if ((await L.run(s, t)) >= t.want) ok++;
      } catch {}
      times.push(performance.now() - t0);
      await L.close(s);
    }
    rows.push([lib, t.name, `${ok}/3`, (median(times) / 1000).toFixed(1)]);
  }
  rows.push([lib, "Browser start (median)", "", (median(launches) / 1000).toFixed(2)]);
}

console.log("library     task                          got the data  median seconds");
for (const [lib, task, ok, s] of rows) console.log(`${lib.padEnd(12)}${task.padEnd(30)}${ok.padStart(12)}${s.padStart(16)}`);

// Playwright: a page whose quotes appear 10 seconds after it loads. Locators wait for elements on their own.
// npm install playwright    npx playwright install chromium
import { chromium } from "playwright";

const browser = await chromium.launch();
const page = await browser.newPage();
await page.goto("https://quotes.toscrape.com/js-delayed/");
const quotes = page.locator("div.quote");
await quotes.first().waitFor({ timeout: 20_000 });
const rows = await quotes.evaluateAll((els) => els.map((e) => ({ text: e.querySelector(".text").textContent, author: e.querySelector(".author").textContent })));
await browser.close();

console.log(`${rows.length} quotes, first by ${rows[0].author}`);

// Puppeteer: the same delayed page. Wait for the selector explicitly, then read the elements in the page.
// npm install puppeteer   (downloads a matching Chrome)
import puppeteer from "puppeteer";

const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto("https://quotes.toscrape.com/js-delayed/");
await page.waitForSelector("div.quote", { timeout: 20_000 });
const rows = await page.$$eval("div.quote", (els) => els.map((e) => ({ text: e.querySelector(".text").textContent, author: e.querySelector(".author").textContent })));
await browser.close();

console.log(`${rows.length} quotes, first by ${rows[0].author}`);

// Selenium WebDriver for Node.js: the same delayed page. An explicit wait holds until the quotes exist.
// npm install selenium-webdriver   (Selenium Manager downloads the matching ChromeDriver)
import { Builder, By, until } from "selenium-webdriver";
import chrome from "selenium-webdriver/chrome.js";

const driver = await new Builder().forBrowser("chrome").setChromeOptions(new chrome.Options().addArguments("--headless=new")).build();
try {
  await driver.get("https://quotes.toscrape.com/js-delayed/");
  await driver.wait(until.elementLocated(By.css("div.quote")), 20_000);
  const rows = [];
  for (const q of await driver.findElements(By.css("div.quote"))) {
    rows.push({ text: await q.findElement(By.css(".text")).getText(), author: await q.findElement(By.css(".author")).getText() });
  }
  console.log(`${rows.length} quotes, first by ${rows[0].author}`);
} finally {
  await driver.quit();
}