Web scraping with JavaScript and Node.js

// Fetch a static page with Node's built-in fetch and pick fields out of it with Cheerio.
// npm install cheerio   (Node.js 18 or later has fetch built in)
import * as cheerio from "cheerio";

const url = "https://books.toscrape.com/catalogue/category/books/travel_2/index.html";
const res = await fetch(url, { headers: { "User-Agent": "my-scraper/1.0 (you@example.com)" } });
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
const $ = cheerio.load(await res.text());

const books = $("article.product_pod")
  .map((_, el) => ({
    title: $(el).find("h3 a").attr("title"),
    price: $(el).find(".price_color").text().trim(),
    inStock: $(el).find(".availability").text().includes("In stock"),
  }))
  .get();

console.log(`${books.length} books on the page`);
console.log(books.slice(0, 3));

// Follow "next" links across a listing, one request a second, and save every record to a JSON file.
// npm install cheerio
import { writeFile } from "node:fs/promises";
import * as cheerio from "cheerio";

let url = "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html";
const rows = [];
let pages = 0;

while (url && pages < 5) {
  const $ = cheerio.load(await (await fetch(url)).text());
  pages++;
  $("article.product_pod").each((_, el) => {
    rows.push({ title: $(el).find("h3 a").attr("title"), price: $(el).find(".price_color").text().trim(), url: new URL($(el).find("h3 a").attr("href"), url).href });
  });
  const next = $("li.next a").attr("href");
  url = next ? new URL(next, url).href : null;
  await new Promise((r) => setTimeout(r, 1000));
}

await writeFile("mystery-books.json", JSON.stringify(rows, null, 2));
console.log(`${rows.length} books from ${pages} pages saved to mystery-books.json`);
console.log(rows[0]);

// A page that builds its content with JavaScript: fetch gets none of it, Puppeteer runs it in Chrome.
// npm install puppeteer
import puppeteer from "puppeteer";

const url = "https://quotes.toscrape.com/js/";
const html = await (await fetch(url)).text();
console.log(`fetch: ${(html.match(/class="quote"/g) ?? []).length} quotes in the HTML it received`);

const browser = await puppeteer.launch();
try {
  const page = await browser.newPage();
  await page.goto(url);
  await page.waitForSelector("div.quote");
  const quotes = await page.$$eval("div.quote", (els) =>
    els.map((e) => ({ text: e.querySelector(".text").textContent, author: e.querySelector(".author").textContent })),
  );
  console.log(`puppeteer: ${quotes.length} quotes`);
  console.log(quotes[0]);
} finally {
  await browser.close();
}

// An infinite-scroll list: scroll until enough items have loaded or nothing new arrives.
// npm install puppeteer
import puppeteer from "puppeteer";

const browser = await puppeteer.launch();
try {
  const page = await browser.newPage();
  await page.goto("https://quotes.toscrape.com/scroll");
  await page.waitForSelector("div.quote");
  let count = 0;
  for (let i = 0; i < 20; i++) {
    const now = await page.$$eval("div.quote", (els) => els.length);
    if (now >= 50 || (i > 0 && now === count)) break; // enough, or the list stopped growing
    count = now;
    await page.mouse.wheel({ deltaY: 4000 });
    await new Promise((r) => setTimeout(r, 800)); // give the next batch time to arrive
  }
  const authors = await page.$$eval("small.author", (els) => els.map((e) => e.textContent));
  console.log(`${authors.length} quotes loaded by scrolling, ${new Set(authors).size} authors`);
} finally {
  await browser.close();
}

// The same JavaScript page through a scraping API with fetch: one request, typed JSON back.
// export SKRYP_API_KEY=...
const api = process.env.SKRYP_API_URL ?? "https://api.skryp.dev";
const res = await fetch(`${api}/v1/extract`, {
  method: "POST",
  headers: { Authorization: `Bearer ${process.env.SKRYP_API_KEY}`, "Content-Type": "application/json" },
  body: JSON.stringify({
    url: "https://quotes.toscrape.com/js/",
    schema: { type: "object", properties: { quotes: { type: "array", items: { type: "object", properties: { text: { type: "string" }, author: { type: "string" } } } } } },
  }),
});
const out = await res.json();
console.log(`${out.data.quotes.length} quotes, values checked against the page: ${JSON.stringify(out.checks)}`);
console.log(out.data.quotes[0]);
console.log(`route ${out.receipt.route}, cache ${out.receipt.cache}, ${out.receipt.credits} credit${out.receipt.credits === 1 ? "" : "s"}`);