Google Search Results Scraper

// 1. Parse the raw HTML string into a queryable DOM
const parser = new DOMParser();
const doc = parser.parseFromString($$data.html(), "text/html");

// 2. Initialize results array
const extractedData = [];

// 3. Helper function to clean text (remove extra spaces/newlines)
const cleanText = (text) => (text ? text.replace(/\s+/g, " ").trim() : "");

// 4. Select all search result containers
const results = doc.querySelectorAll(".MjjYud");

results.forEach((result, index) => {
  // --- Extraction Logic ---

  // Title & URL: Found in the main anchor tag
  const anchor = result.querySelector("a[jsname='UWckNb']") || result.querySelector(".yuRUbf a");
  const titleEl = result.querySelector("h3");

  // Skip if we can't find a title or link (filters out "People also ask" or empty fillers)
  if (!anchor || !titleEl) return;

  const title = cleanText(titleEl.textContent);
  const url = anchor.href;

  // Displayed URL: Usually in a <cite> tag
  const displayedUrlEl = result.querySelector("cite");
  const displayedUrl = displayedUrlEl ? cleanText(displayedUrlEl.textContent) : "";

  // Description: Found in the snippet div (often class .VwiC3b or data-sncf="1")
  const descEl = result.querySelector(".VwiC3b, [data-sncf='1']");
  const description = descEl ? cleanText(descEl.textContent) : "";

  // Emphasized Keywords: <em> or <b> tags inside the description
  const keywords = [];
  if (descEl) {
    descEl.querySelectorAll("em, b").forEach((em) => {
      keywords.push(cleanText(em.textContent));
    });
  }
  const emphasizedKeywords = keywords.join(", ");

  // Site Links: Sub-links often found in specific containers like .kb0PBd
  const siteLinkEls = result.querySelectorAll(".kb0PBd a, .St3GK a");
  const siteLinks = Array.from(siteLinkEls)
    .map((link) => cleanText(link.textContent))
    .filter((text) => text && text !== "Read more") // Remove "Read more" noise
    .join(" | ");

  // Date: Look for specific date spans (common classes: .LEwnzc, .r0bn4c, .YrbPuc)
  const dateEl = result.querySelector(".LEwnzc, .r0bn4c, .YrbPuc, .OSrXXb span");
  let date = "";
  if (dateEl) {
    // If the date is inside a complex span, try to isolate the date text
    date = cleanText(dateEl.textContent);
  } else if (description) {
    // Fallback: Check if description starts with a date pattern (e.g., "Sep 8, 2025 — ...")
    const dateMatch = description.match(/^([A-Z][a-z]{2}\s\d{1,2},?\s\d{4})/);
    if (dateMatch) date = dateMatch[0];
  }

  // Product Info: Ratings or store info (rare in this specific HTML but good to have)
  const productInfoEl = result.querySelector(".fG8Fp, .os-seller-name-primary");
  const productInfo = productInfoEl ? cleanText(productInfoEl.textContent) : "";

  // Type: Detect if it's a Video, Ad, or Organic result
  let type = "Organic";
  if (result.querySelector(".WZiv5b") || result.textContent.includes("Videos")) {
    type = "Video";
  }
  if (result.querySelector(".vdQmEd")) {
    type = "Ad";
  }

  // Position: 1-based index
  const position = index + 1;

  // --- Push to Array ---
  extractedData.push({
    Title: title,
    url: url,
    Description: description,
    displayedUrl: displayedUrl,
    "Emphasized keywords": emphasizedKeywords,
    siteLinks: siteLinks,
    Date: date,
    productInfo: productInfo,
    Type: type,
    position: position,
  });
});

// 5. Convert to CSV Format
const headers = [
  "Title",
  "url",
  "Description",
  "displayedUrl",
  "Emphasized keywords",
  "siteLinks",
  "Date",
  "productInfo",
  "Type",
  "position",
];

// Create CSV Header row
const csvRows = [headers.join(",")];

// Create CSV Data rows
extractedData.forEach((row) => {
  const values = headers.map((header) => {
    const value = row[header];
    // Escape double quotes by doubling them (" -> "") and wrap the field in quotes
    const escapedValue = String(value).replace(/"/g, '""');
    return `"${escapedValue}"`;
  });
  csvRows.push(values.join(","));
});

// 6. Return the final CSV string
return csvRows.join("\n");