Web Scraper — JavaScript Project

1. The Problem

Build a web scraper that fetches a page with fetch(), extracts structured data (titles, links, prices, or any repeating pattern) using regex and string methods, and saves the results as JSON. Includes a rate limiter to be a polite scraper.

Why this project? fetch in Node.js, HTML parsing without a DOM (regex on raw text), rate limiting with Promise delays, and async concurrency patterns — all used in real data collection pipelines.

2. How to Think About This

Three layers: fetcher (gets raw HTML, handles HTTP errors, retries on failure), extractor (pure function — takes HTML string, returns structured data), runner (loops over a list of URLs with a delay between each to avoid overwhelming the server). The extractor is testable without any network.

3. The Build

scraper.mjs

import { writeFileSync } from "fs";

// ── Fetcher — with retry and rate limiting ─────────────────────────────────
const delay = ms => new Promise(r => setTimeout(r, ms));

export async function fetchHTML(url, options = {}) {
  const { retries = 3, retryDelay = 1000, timeout = 10000 } = options;
  for (let attempt = 1; attempt <= retries; attempt++) {
    try {
      const controller = new AbortController();
      const timer = setTimeout(() => controller.abort(), timeout);
      const res = await fetch(url, {
        signal: controller.signal,
        headers: { "User-Agent": "TheCodex-Bot/1.0 (educational scraper)" },
      });
      clearTimeout(timer);
      if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`);
      return await res.text();
    } catch (e) {
      if (attempt === retries) throw e;
      console.warn(`  Attempt ${attempt} failed: ${e.message}. Retrying...`);
      await delay(retryDelay * attempt);
    }
  }
}

// ── Extractor — pure functions on HTML strings ─────────────────────────────
export function extractLinks(html, baseUrl = "") {
  const links = [];
  const pattern = /<a\s[^>]*href="([^"]+)"[^>]*>([^<]*)/gi;
  let match;
  while ((match = pattern.exec(html)) !== null) {
    const href = match[1].trim();
    const text = match[2].trim().replace(/\s+/g, " ");
    if (!href || href.startsWith("#") || href.startsWith("javascript:")) continue;
    const url = href.startsWith("http") ? href : new URL(href, baseUrl).href;
    if (text) links.push({ url, text });
  }
  return links;
}

export function extractMeta(html) {
  const get = (pattern) => html.match(pattern)?.[1]?.trim() ?? "";
  return {
    title:       get(/<title[^>]*>([^<]+)/i),
    description: get(/<meta[^>]*name="description"[^>]*content="([^"]+)"/i)
                 || get(/<meta[^>]*content="([^"]+)"[^>]*name="description"/i),
    h1:          get(/<h1[^>]*>([^<]+)/i),
    canonical:   get(/<link[^>]*rel="canonical"[^>]*href="([^"]+)"/i),
  };
}

export function extractPattern(html, pattern) {
  // Generic: extract all matches of a regex with named groups
  const results = [];
  let match;
  const re = typeof pattern === "string" ? new RegExp(pattern, "gi") : pattern;
  while ((match = re.exec(html)) !== null) {
    results.push(match.groups ?? match.slice(1));
  }
  return results;
}

// ── Runner ─────────────────────────────────────────────────────────────────
export async function scrapePages(urls, extractFn, rateLimit = 1000) {
  const results = [];
  for (const url of urls) {
    try {
      console.log(`  Fetching: ${url}`);
      const html = await fetchHTML(url);
      const data = extractFn(html, url);
      results.push({ url, data, fetchedAt: new Date().toISOString() });
    } catch (e) {
      console.error(`  Failed: ${url} — ${e.message}`);
      results.push({ url, error: e.message, fetchedAt: new Date().toISOString() });
    }
    if (rateLimit > 0) await delay(rateLimit);
  }
  return results;
}

// ── CLI ─────────────────────────────────────────────────────────────────────
const [,, ...args] = process.argv;
const urlArg = args.find(a => a.startsWith("http"));
const outArg = args[args.indexOf("--out") + 1];

if (!urlArg) {
  console.log("Usage: node scraper.mjs <url> [--out results.json]");
  console.log("Example: node scraper.mjs https://example.com --out out.json");
  process.exit(0);
}

try {
  console.log(`
Fetching: ${urlArg}`);
  const html = await fetchHTML(urlArg);
  const meta  = extractMeta(html);
  const links = extractLinks(html, urlArg);

  console.log(`
Title:       ${meta.title}`);
  console.log(`Description: ${meta.description?.slice(0, 80) || "(none)"}`);
  console.log(`H1:          ${meta.h1}`);
  console.log(`Links found: ${links.length}`);
  links.slice(0, 5).forEach(l => console.log(`  ${l.text.padEnd(30)} ${l.url}`));

  if (outArg) {
    const result = { url: urlArg, fetchedAt: new Date().toISOString(), meta, links };
    writeFileSync(outArg, JSON.stringify(result, null, 2));
    console.log(`
Saved to ${outArg}`);
  }
} catch(e) { console.error("Error:", e.message); process.exit(1); }

4. Test & Prove

import { extractLinks, extractMeta, extractPattern } from "./scraper.mjs";
let p=0,f=0;
function test(d,fn){try{fn();console.log("  ✓",d);p++;}catch(e){console.log("  ✗",d,"—",e.message);f++;}}
const html = `<html><head><title>Test Page</title>
<meta name="description" content="A test page"></head>
<body><h1>Hello World</h1>
<a href="https://example.com">Example</a>
<a href="/about">About</a>
<a href="#skip">Skip</a></body></html>`;
test("title extracted",       () => { if(extractMeta(html).title!=="Test Page") throw new Error(); });
test("description extracted", () => { if(extractMeta(html).description!=="A test page") throw new Error(); });
test("h1 extracted",          () => { if(extractMeta(html).h1!=="Hello World") throw new Error(); });
test("external link found",   () => { const l=extractLinks(html); if(!l.find(x=>x.url==="https://example.com")) throw new Error(JSON.stringify(l)); });
test("anchor skipped (#)",    () => { const l=extractLinks(html); if(l.find(x=>x.url.includes("#skip"))) throw new Error("should skip"); });
test("relative resolved",     () => { const l=extractLinks(html,"https://base.com"); if(!l.find(x=>x.url==="https://base.com/about")) throw new Error(JSON.stringify(l)); });
test("pattern extraction",    () => { const r=extractPattern(html,/href="(?<url>https?:[^"]+)"/gi); if(!r.length) throw new Error(); });
console.log(`
${p} passed, ${f} failed`);

5. The Interface

node scraper.mjs https://example.com
node scraper.mjs https://example.com --out results.json

Fetching: https://example.com
Title:       Example Domain
H1:          Example Domain
Links found: 1
  More information...    https://www.iana.org/domains/...

6. Run It

node scraper.mjs https://example.com --out out.json
node scraper.test.mjs

🎯 Try this next

Add recursive crawling — follow internal links up to a depth limit
Add robots.txt parsing — check Disallow rules before fetching
Add CSS selector-style extraction using a simple attribute parser
Save results to SQLite via the better-sqlite3 package

What you practised

fetch() with AbortController for timeout, retry with exponential backoff, regex extraction on raw HTML strings, new URL() for resolving relative URLs, rate limiting with Promise delays, and the three-layer scraper architecture (fetch / extract / run).

Reference: Async & Promises · Regular Expressions · Error Handling