bidaangmamamayan/lib/crawl.js

// BidaAngMamamayan crawler — the tribute edition's supply line.
//
// Forked from forthena's lib/crawl.js, retargeted for ONE subject: Sen. BAM
// Aquino. The differences that matter:
//
//   * Every job is a BAM query (lib/sources.js) — nothing general ever runs
//     on a normal tick. The PH native wires run ONLY when a tick produced
//     zero named-BAM stories (quiet-day fallback).
//   * A RELEVANCE GATE, not just a freshness window. Bing's keyword search
//     matches on article content, so "Bam Aquino power electricity" returns
//     VAT-op-eds and committee gossip that never name him. Only stories whose
//     title OR description names him (or a senate context with an Aquino
//     surname) are admitted from the query jobs.
//   * maxAge is 10 days, not 48h. A senator's mention volume is low
//     (measured: ~12 unique stories/week in Sept 2026); a named-BAM story
//     from last Thursday is still front-page material this weekend.
//   * No per-reader profiles, no learning, no feedback. One public pool,
//     one public edition.

import Parser from "rss-parser";
import { getCollections } from "./db.js";
import { sanitize, extractImage } from "./llm.js";
import { BAM_QUERIES, PH_NATIVE, queryFeed, unwrapFeedLink, stripPublisherSuffix, publisherFrom, publisherFromUrl, publisherVerified } from "./sources.js";
import { normalizeCategory } from "./categories.js";
import { prefetchArticles } from "./article.js";

const UA = {
  "User-Agent":
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
  Accept: "application/rss+xml,application/atom+xml,application/xml,text/xml,*/*",
};

const parser = new Parser({
  timeout: 20000,
  headers: UA,
  customFields: {
    item: [
      ["media:content", "mediaContent", { keepArray: true }],
      ["media:thumbnail", "mediaThumbnail", { keepArray: true }],
      ["content:encoded", "contentEncoded"],
      ["News:Source", "newsSource"],
      ["News:Image", "newsImage"],
    ],
  },
});

// ── the relevance gate ──────────────────────────────────────────────────
// Strong: the story says his name (in any of the ways the wire does).
const BAM_NAMED = /\bBam\s*(?:A\.)?\s*Aquino\b|\bSen(?:ator|ador|\.?)\s+Bam\b|\bSen\s*Bam\b|\bBam\b(?=.{0,80}\bAquino\b)/i;
// Context: a legislative-chamber story that may be about him without naming
// him in the snippet. Admitted ONLY from the fallback native wires, where the
// story also has to carry an "Aquino" surname. (The 2026-09-15 PNA headline
// for the VisMin power probe reads "Aquino echoes House call..." — "house"
// is a required context word; a bare surname is not, but a bare "senate bill"
// with no Aquino in sight is dropped too.)
const AQUINO_CTX = /\b(senate|senator|sen\.|congress|house|congressional|legislature|floor|vote[d]?|bill|inquiry|resolution|probe|lawmakers?|committee)\b/i;

export function bamRelevance(title, description, { requireNamed = true } = {}) {
  const t = `${title || ""} ${description || ""}`;
  if (BAM_NAMED.test(t)) return { ok: true, named: true };
  if (!requireNamed && /\bAquino\b/i.test(t) && AQUINO_CTX.test(t)) return { ok: true, named: false };
  return { ok: false, named: false };
}

// Bing's own thumbnail host — a re-encoded cache copy. Fine as a fallback.
const BING_IMG = /bing\.com\/th\?id=ONUT/i;

function pickImage(item) {
  const real = extractImage(item, null);
  if (real && !BING_IMG.test(real)) return { image: real, fromBing: false };
  const b = String(item.newsImage || "").trim();
  if (/^https?:\/\//i.test(b)) return { image: b, fromBing: true };
  return { image: real || null, fromBing: false };
}

async function mapWithLimit(items, size, fn) {
  const out = [];
  let i = 0;
  const workers = Array.from({ length: Math.min(size, items.length) }, async () => {
    while (i < items.length) {
      const idx = i++;
      try {
        out.push(await fn(items[idx]));
      } catch (e) {
        out.push({ error: String(e && e.message ? e.message : e).slice(0, 120) });
      }
    }
  });
  await Promise.all(workers);
  return out;
}

// Syndication mirrors: drop them at ingest so the reader never lands in
// MSN/Yahoo chrome.
const MIRROR_HOSTS = ["msn.com", "yahoo.com", "microsoft.com", "bing.com", "msftcdn.net"];
function isMirrorHost(url) {
  let host;
  try {
    host = new URL(url).hostname.replace(/^www\./i, "").toLowerCase();
  } catch {
    return false;
  }
  return MIRROR_HOSTS.some((h) => host === h || host.endsWith("." + h));
}

// Title fingerprint for cross-query dedupe (the same story arrives from
// "Bam Aquino" and "Bam Aquino power electricity").
function titleKey(t) {
  return String(t || "")
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, " ")
    .trim()
    .split(" ")
    .filter((w) => w.length > 3)
    .slice(0, 6)
    .join(" ");
}

function buildJobs({ useFallback = false } = {}) {
  const jobs = [];
  for (const q of BAM_QUERIES) jobs.push({ url: queryFeed(q.query), kind: "bam", name: q.name, category: q.category });
  if (useFallback) {
    for (const src of PH_NATIVE) for (const url of src.feeds) jobs.push({ url, kind: "native", name: src.name, category: src.category });
  }
  return jobs;
}

// og:image (or first large <img>) from the real article page. Best-effort.
async function resolveOgImage(link) {
  if (!/^https?:\/\//i.test(link)) return null;
  try {
    const res = await fetch(link, {
      headers: { "User-Agent": UA["User-Agent"], Accept: "text/html,*/*" },
      redirect: "follow",
      signal: AbortSignal.timeout(8000),
    });
    if (!res.ok) return null;
    const head = (await res.text()).slice(0, 120000);
    const m =
      /<meta[^>]+property=["']og:image["'][^>]+content=["']([^"']+)["']/i.exec(head) ||
      /<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:image["']/i.exec(head) ||
      /<meta[^>]+name=["']twitter:image["'][^>]+content=["']([^"']+)["']/i.exec(head);
    let u = m && m[1] ? m[1].trim() : null;
    if (!u) return null;
    if (u.startsWith("//")) u = "https:" + u;
    if (!/^https?:\/\//i.test(u)) return null;
    if (/1x1|spacer|pixel|blank|logo|sprite\.svg/i.test(u)) return null;
    return u.slice(0, 500);
  } catch {
    return null;
  }
}

// Headline-date veto (kept from forthena, learned the hard way): an aggregator
// can re-deliver a week-old story with a fresh pubDate. Scan the text for
// explicit dates and refuse anything older than the window.
const MONTHS = { january: 0, feb: 1, february: 1, mar: 2, march: 2, apr: 3, april: 3, may: 4, jun: 5, june: 5, jul: 6, july: 6, aug: 7, august: 7, sep: 8, sept: 8, september: 8, oct: 9, october: 9, nov: 10, november: 10, dec: 11, december: 11 };
function headlineDateAge(text, nowMs) {
  const t = String(text || "").toLowerCase();
  let newest = -1;
  const re = /\b(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\s+(\d{1,2})(?:st|nd|rd|th)?(?:,\s*(\d{4}))?|\b(\d{1,2})\s+(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)(?:,\s*(\d{4}))?/gi;
  let m;
  while ((m = re.exec(t))) {
    const monName = (m[1] || m[5] || "").replace(/\.$/, "");
    const mon = MONTHS[monName];
    const day = parseInt(m[2] || m[4] || "1", 10);
    let year = parseInt(m[3] || m[6] || "", 10);
    if (mon === undefined || Number.isNaN(mon)) continue;
    if (!Number.isFinite(year)) {
      const nowY = new Date(nowMs).getFullYear();
      year = new Date(nowY, mon, day) > nowMs ? nowY - 1 : nowY;
    }
    const d = new Date(year, mon, day).getTime();
    if (Number.isFinite(d) && d <= nowMs + 86400e3) newest = Math.max(newest, d);
  }
  if (newest < 0) return null;
  return (nowMs - newest) / 86400e3;
}

export async function crawlOnce({ perFeed = 20, maxStories = 300, concurrency = 6, ogBudget = 24, maxAgeH = 240 } = {}) {
  const { stories } = await getCollections();
  const now = new Date();
  const maxAgeMs = maxAgeH * 3600e3;
  const newestAllowed = now.getTime() - maxAgeMs;

  // First pass: the BAM queries only.
  let jobs = buildJobs({ useFallback: false });
  let results = await mapWithLimit(jobs, concurrency, async (job) => {
    const feed = await parser.parseURL(job.url);
    return { job, feed };
  });

  // Normalise + gate (Pass 1).
  let candidates = [];
  const errors = [];
  let skippedStale = 0, skippedUndated = 0, skippedSynd = 0, skippedUnverified = 0, gatedOut = 0;
  for (const r of results) {
    if (!r || r.error || !r.feed) {
      if (r && r.error) errors.push(`${r.job?.name || "job"}: ${r.error}`);
      continue;
    }
    const { job, feed } = r;
    const items = (feed.items || []).slice(0, perFeed).map((item) => {
      const rawTitle = sanitize(item.title || "");
      const wrapped = (item.link || "").trim();
      const link = job.kind === "native" ? wrapped : unwrapFeedLink(wrapped);
      if (!link) return null;
      const desc = sanitize(item.contentSnippet || item.contentEncoded || item.content || item.summary || "").slice(0, 900);
      const hadDate = !!item.isoDate;
      const pub = hadDate ? new Date(item.isoDate) : now;
      const img = job.kind === "native"
        ? { image: extractImage(item, feed), fromBing: false }
        : pickImage(item);
      const source = job.kind === "native"
        ? sanitize(job.name || feed.title).slice(0, 80)
        : (sanitize(item.newsSource || "").slice(0, 80)
          || publisherFromUrl(link)
          || publisherFrom(rawTitle, job.name).slice(0, 80));
      return { job, rawTitle, link, feedUrl: job.kind === "native" ? null : wrapped, hadDate, title: job.kind === "native" ? rawTitle : stripPublisherSuffix(rawTitle), desc, source, summary: desc, image: img.image, bingImage: img.fromBing, published: pub, author: sanitize(item.creator || "").slice(0, 120) };
    }).filter(Boolean);
    // relevance gate: query jobs must NAME him; native fallback must carry
    // an Aquino surname in a senate context.
    candidates.push(...items.filter((it) => {
      const gate = bamRelevance(it.title, it.desc, { requireNamed: it.job.kind !== "native" });
      if (!gate.ok) { gatedOut++; return false; }
      it.named = gate.named;
      return true;
    }).filter((it) => {
      // VERIFIED PUBLISHERS ONLY (Daniel, 2026-09-15): the article URL's host
      // must be on the allowlist. This runs before mirror/stale so a tabloid
      // story never reaches the desk even if it names him.
      if (!publisherVerified(it.link)) { skippedUnverified++; return false; }
      if (isNaN(it.published.getTime())) { skippedUndated++; return false; }
      if (isMirrorHost(it.link)) { skippedSynd++; return false; }
      if (it.published.getTime() < newestAllowed) { skippedStale++; return false; }
      return true;
    }));
  }

  // Quiet day: zero named-BAM stories this tick -> run the associated wires.
  let usedFallback = false;
  const namedCount = candidates.filter((c) => c.named).length;
  if (namedCount === 0) {
    usedFallback = true;
    const fbJobs = buildJobs({ useFallback: true }).filter((j) => j.kind === "native");
    const fbResults = await mapWithLimit(fbJobs, concurrency, async (job) => {
      const feed = await parser.parseURL(job.url);
      return { job, feed };
    });
    for (const r of fbResults) {
      if (!r || r.error || !r.feed) {
        if (r && r.error) errors.push(`${r.job?.name || "job"}: ${r.error}`);
        continue;
      }
      const { job, feed } = r;
      const items = (feed.items || []).slice(0, perFeed).map((item) => {
        const rawTitle = sanitize(item.title || "");
        const wrapped = (item.link || "").trim();
        const link = wrapped;
        if (!link) return null;
        const desc = sanitize(item.contentSnippet || item.contentEncoded || item.content || item.summary || "").slice(0, 900);
        const hadDate = !!item.isoDate;
        const pub = hadDate ? new Date(item.isoDate) : now;
        const img = { image: extractImage(item, feed), fromBing: false };
        return { job, rawTitle, link, feedUrl: null, hadDate, title: rawTitle, desc, source: sanitize(job.name || feed.title).slice(0, 80), summary: desc, image: img.image, bingImage: false, published: pub, author: sanitize(item.creator || "").slice(0, 120) };
      }).filter(Boolean);
      candidates.push(...items.filter((it) => {
        const gate = bamRelevance(it.title, it.desc, { requireNamed: false });
        if (!gate.ok) { gatedOut++; return false; }
        it.named = gate.named;
        if (!publisherVerified(it.link)) { skippedUnverified++; return false; }
        if (isNaN(it.published.getTime())) { skippedUndated++; return false; }
        if (isMirrorHost(it.link)) { skippedSynd++; return false; }
        if (it.published.getTime() < newestAllowed) { skippedStale++; return false; }
        return true;
      }));
    }
  }

  // Global image-reuse guard (kept from forthena): one URL on >=5 distinct
  // stories = boilerplate.
  const globalFreq = new Map();
  for (const it of candidates) {
    if (!it.image) continue;
    const s = globalFreq.get(it.image) || new Set();
    s.add(titleKey(it.title) || it.link);
    globalFreq.set(it.image, s);
  }
  const boiler = new Set([...globalFreq.entries()].filter(([, set]) => set.size >= 5).map(([u]) => u));
  if (boiler.size) for (const it of candidates) if (boiler.has(it.image)) it.image = null;

  // Cross-query dedupe: URL exact, then title fingerprint. Named-BAM stories
  // beat context stories; the earlier query wins ties (BAM Wire is first).
  const urlKeys = new Set();
  const titleKeys = new Map();
  const picked = [];
  const ordered = [...candidates].sort((a, b) => (b.named ? 1 : 0) - (a.named ? 1 : 0));
  for (const it of ordered) {
    if (!it.rawTitle || !it.link || !/^https?:\/\//i.test(it.link)) continue;
    if (urlKeys.has(it.link)) continue;
    // Headline-date veto (a named-BAM story is worth 10 days, so the window
    // here is the whole crawl window, not half of it).
    const txtAge = headlineDateAge(it.title + " " + it.summary, now.getTime());
    if (txtAge !== null && txtAge > maxAgeH / 24 + 2) { skippedStale++; continue; }
    const tk = titleKey(it.title);
    if (tk) {
      const prev = titleKeys.get(tk);
      if (prev) {
        // only replace when the new one is stronger (named over context)
        if (!(it.named && !prev.named)) continue;
      }
      titleKeys.set(tk, { named: it.named });
    }
    urlKeys.add(it.link);
    picked.push(it);
    if (picked.length >= maxStories) break;
  }

  // og:image rescue for photo-less stories, budget-capped.
  const needOg = picked.filter((it) => !it.image).slice(0, ogBudget);
  await mapWithLimit(needOg, 5, async (it) => {
    const og = await resolveOgImage(it.link);
    if (og) it.image = og;
  });

  // Insert (upsert by url). `named` and `fallback` are stored: the front page
  // leads with named-BAM stories and the masthead can count them.
  let added = 0, withImage = 0, reingested = 0;
  for (const it of picked) {
    const res = await stories.updateOne(
      { url: it.link },
      {
        $setOnInsert: {
          url: it.link,
          feedUrl: it.feedUrl || null,
          title: it.title.slice(0, 300),
          summary: it.summary,
          image: it.image,
          imageSource: it.image ? (it.bingImage ? "bing" : "feed") : null,
          author: it.author || it.source,
          source: it.source,
          category: normalizeCategory(it.job ? it.job.category : "ph"),
          desk: it.job ? it.job.name : "Associated",
          feedKind: it.job ? it.job.kind : "native",
          named: !!it.named,
          fallback: it.job ? false : true,
          published: it.published,
          discoveredAt: now,
          ingestedAt: now,
        },
      },
      { upsert: true }
    );
    if (res.upsertedCount) { added++; if (it.image) withImage++; }
    else {
      // refresh the image on an existing row when we newly rescued one
      reingested++;
      if (it.image && !it.bingImage) {
        const cur = await stories.findOne({ url: it.link }, { projection: { image: 1 } });
        if (!cur || !cur.image) await stories.updateOne({ url: it.link }, { $set: { image: it.image, imageSource: "og" } });
      }
    }
  }

  // Warm the reader (kept from forthena): fetch the article bodies NOW so
  // opening a story is a cache hit.
  let articles = { fetched: 0, ok: 0 };
  try {
    articles = await prefetchArticles(
      picked.map((it) => it.link),
      { budget: 12, concurrency: 3, deadlineMs: 40000 }
    );
  } catch (e) {
    console.warn("[bama] article prefetch failed:", String((e && e.message) || e).slice(0, 120));
  }

  return { added, reingested, withImage, gatedOut, skippedStale, skippedUndated, skippedSynd, skippedUnverified, named: namedCount, feeds: jobs.length, usedFallback, articles, errors: errors.slice(0, 12), collectedAt: now };
}