bidaangmamamayan/lib/article.js

// Article fetcher — the reader pane's supply line.
//
// Goal (Daniel, 2026-08-31): reading a story must not require leaving BAM-tribute. We fetch the
// publisher page server-side, run it through Readability to get the article body (the same
// extraction engine Firefox's reader mode ships), and cache it on the story doc.
//
// Discipline:
//   * Server-only (imports fs-free DOM code). Never pulled into a client component.
//   * Returns PLAIN PARAGRAPHS, never raw HTML — the reader renders text nodes, so a publisher
//     page can't inject markup, scripts, or trackers into our paper.
//   * Budgeted at crawl time + on demand, with an in-flight lock: one URL is fetched once,
//     no matter how many tickers/cards point at it.
//   * Failure is a state, not an exception: status "blocked"/"error" is stored and the UI
//     falls back to "read at <source>" honestly. No fake article text, ever.

import { Readability } from "@mozilla/readability";
import { parseHTML } from "linkedom";
import { getCollections } from "./db.js";

const UA =
  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";

const MAX_BYTES = 2_500_000; // stop reading a page past 2.5 MB
const MAX_PARAS = 80;
const MAX_CHARS = 24_000; // stored body cap — a long-read stays readable, Mongo stays sane;
                          // the overflow gets an honest "continue at the source" link
const MIN_WORDS = 90;     // below this Readability chewed off a shell/teaser, not an article
const FETCH_TIMEOUT_MS = 12_000;
const RETRY_AFTER_MS = 1000 * 60 * 60 * 12; // re-try a blocked/error page twice a day at most

const inflight = new Map(); // url -> promise

// Hosts that refuse us, with a cooling-off window. Discovered the hard way on
// 2026-08-31: newsinfo.inquirer.net answers EVERY article with 403 (every UA and header
// combination we tried), and because its feed sits early in the crawl order it ate 10 of
// the 14 prefetch slots in one tick — the reader came out warm for only 4 stories, all
// SunStar. Remembering the wall means we pay for it once per window instead of per story,
// and it frees the budget for publishers who actually answer.
const blockedHosts = new Map(); // host -> { until, reason }
const BLOCK_TTL_MS = 1000 * 60 * 60 * 6;

function hostOf(url) {
  try { return new URL(url).hostname.toLowerCase(); } catch { return null; }
}

function blockedUntil(host) {
  const e = host && blockedHosts.get(host);
  if (!e) return 0;
  if (Date.now() > e.until) { blockedHosts.delete(host); return 0; }
  return e.until;
}

function noteBlock(host, reason) {
  if (!host) return;
  blockedHosts.set(host, { until: Date.now() + BLOCK_TTL_MS, reason });
}

// Google News "articles/CBMi..." links are JS redirect wrappers; the publisher URL is
// packed in the base64url article id. Same decoder lib/crawl.js uses for og:image.
export function unwrapGoogleNews(link) {
  const m = /news\.google\.com\/(?:rss\/)?articles\/([A-Za-z0-9_-]+)/.exec(String(link || ""));
  if (!m) return link;
  try {
    const raw = Buffer.from(m[1], "base64url").toString("latin1");
    const urls = raw.match(/https?:\/\/[^\x00-\x1f\x7f"]+/g) || [];
    const real = urls.map((u) => u.replace(/[\x00-\x1f]+.*$/, "")).filter((u) => !/google|gstatic/i.test(u));
    if (real.length) return real.sort((a, b) => b.length - a.length)[0];
  } catch {
    /* keep the wrapper */
  }
  return link;
}

function absolutise(src, base) {
  try {
    return new URL(src, base).href;
  } catch {
    return null;
  }
}

// Readability hands back a fragment of markup. We re-parse it and pull text out of the
// block elements — deliberately losing all formatting so nothing hostile survives.
function parasFrom(article, baseHref) {
  // article.textContent is the authority Readability itself computed (and the
  // same value `words` is counted from). linkedom's document.body can parse to an EMPTY
  // textContent on a re-parsed fragment — verified on scmp.com 2026-08-31: a 633-word
  // story where body.textContent was "" while 6 <p> blocks existed — so the plain-text
  // fallback below must never re-read the DOM for its source.
  const plainText = String(article?.textContent || "").replace(/\r/g, "");
  const frag = String(article?.content || "");
  if (!frag) return [];
  const { document } = parseHTML(`<body>${frag}</body>`);
  const out = [];
  let chars = 0;
  const blocks = [...document.querySelectorAll("h2, h3, h4, p, blockquote, li, pre, figcaption")];
  for (const node of blocks) {
    const tag = node.nodeName ? String(node.nodeName).toLowerCase() : "p";
    const text = String(node.textContent || "").replace(/\s+/g, " ").trim();
    if (!text || text.length < 3) continue;
    chars += text.length;
    if (chars > MAX_CHARS || out.length > MAX_PARAS) {
      out.push({ tag: "p", text: "", truncated: true });
      break;
    }
    out.push({ tag, text });
  }
  const images = [];
  for (const img of document.querySelectorAll("img")) {
    const u = absolutise(img.getAttribute("src") || img.getAttribute("data-src") || "", baseHref);
    if (u && /^https?:/i.test(u) && !/1x1|spacer|pixel|blank|logo|sprite|tracking/i.test(u)) images.push(u);
    if (images.length >= 12) break;
  }

  // Safety net: if the block walk produced nothing usable, the copy is sitting in
  // non-block elements (a bare <div>, or <p> wrappers that were all empty). SCMP returned
  // 1107 words but 0 surviving paragraphs on 2026-08-31 — a blank pane: status ok, words
  // populated, nothing to render. This net runs whenever NO usable paragraph survived, so
  // words>0 can never again ship an empty page.
  if (!out.some((p) => p.text)) {
    // Split the Readability text into paragraphs: prefer blank-line breaks, then single
    // newlines, and as a last resort (some publishers return one flat line) break long
    // runs on sentence boundaries so the pane never shows a wall of unbroken text.
    const splitInto = (src, re) => src.split(re).map((x) => x.replace(/\s+/g, " ").trim()).filter((t) => t.length > 2);
    let chunks = splitInto(plainText, /\n{2,}/);
    if (chunks.length <= 1) chunks = splitInto(plainText, /\n/);
    if (chunks.length <= 1 && plainText.replace(/\s+/g, " ").trim().length > 300) {
      chunks = splitInto(plainText.replace(/\s+/g, " "), /(?<=[.!?])\s+(?=["A-Z\u201c])/);
    }
    out.length = 0;
    chars = 0;
    for (const t of chunks) {
      chars += t.length;
      if (chars > MAX_CHARS || out.length > MAX_PARAS) { out.push({ tag: "p", text: "", truncated: true }); break; }
      out.push({ tag: "p", text: t });
    }
  }
  return { paras: out, images };
}

// Some outlets ship the article as JSON in the page and render it client-side, so
// Readability sees a shell (ABS-CBN is the case that forced this: 60 words extracted
// out of a real 500-word story). Pull the body out of the structured data instead.
// Order: JSON-LD articleBody -> Next.js pageProps body_html -> any big body-ish field.
function structuredBody(doc, html) {
  // 1. schema.org NewsArticle / articleBody
  for (const s of doc.querySelectorAll('script[type="application/ld+json"]')) {
    const raw = String(s.textContent || "").trim();
    if (!raw) continue;
    try {
      const j = JSON.parse(raw);
      const seen = [];
      const walk = (n, d) => {
        if (!n || d > 6 || seen.length > 3) return;
        if (Array.isArray(n)) { for (const x of n) walk(x, d + 1); return; }
        if (typeof n === "object") {
          for (const [k, v] of Object.entries(n)) {
            if (typeof v === "string" && /^(articlebody|articleBody|text)$/i.test(k) && v.length > 240) seen.push(v);
            else walk(v, d + 1);
          }
        }
      };
      walk(j, 0);
      if (seen.length) return { html: seen.sort((a, b) => b.length - a.length)[0], from: "jsonld" };
    } catch { /* malformed ld+json is normal on news sites */ }
  }

  // 2. Next.js / Nuxt payload with a body_html (EV/Superdesk "content" object)
  const nd = /<script[^>]*id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/.exec(html);
  if (nd) {
    try {
      const j = JSON.parse(nd[1]);
      const hits = [];
      const walk = (n, d) => {
        if (!n || d > 8 || hits.length > 4) return;
        if (Array.isArray(n)) { for (const x of n) walk(x, d + 1); return; }
        if (typeof n === "object") {
          for (const [k, v] of Object.entries(n)) {
            if (typeof v === "string" && /^(body_html|bodyHtml|articleBody|content_html)$/i.test(k) && v.length > 240) hits.push(v);
            else walk(v, d + 1);
          }
        }
      };
      walk(j?.props?.pageProps ?? j, 0);
      if (hits.length) return { html: hits.sort((a, b) => b.length - a.length)[0], from: "nextdata" };
    } catch { /* ignore */ }
  }
  return null;
}

// Readability leaks photo credits at the top ("DOST-PAGASA FB PAGE", "Abhijeet Mishra /
// SamMobile", "… | INQUIRER File Photo") and newsletter/cookie boilerplate at the tail.
// Trim both — never reorder or rewrite what is left.
const CREDIT = /(?:file photo|courtesy|photo by|image:|getty|facebook page|^by \p{Lu}|\p{Lu}\/(?:inquirer|rappler|cnn|gmanetwork|abs-cbn|philstar|bworld|pna)\b)/iu;
const ALLCAPS = /^[\p{Lu}\d\s.,'’&()\/-]+$/u;
const JUNK_TAIL = /^(?:cookie|subscribe|sign ?in|log ?in|advertisement|sponsored content|all rights reserved|read more\b|follow us|join the (?:conversation|discussion)|related stories|comments?\b|tags?\b|this article first appeared|want news alerts|email this article|print this article|also read|read next|most read|trending|popular\b|newsletter\b|share this\b)/i;

// Structured bodies arrive as HTML; parasFrom's fallback wants a text form of it.
function stripTagsOf(html) {
  return String(html || "").replace(/<\/(p|div|section|article|li|h[1-6]|blockquote)>/gi, "\n\n").replace(/<[^>]*>/g, " ");
}

function cleanParas(paras) {
  const list = paras.filter((p) => p && p.text);
  // Leading credit lines are always short — a genuine lede is not. That length guard is
  // what keeps "KATHMANDU, Nepal – Three months before…" (mixed case, long) intact.
  let start = 0;
  while (start < list.length - 1 && list[start].text.length < 90) {
    const t = list[start].text.trim();
    const isCredit = CREDIT.test(t) || ALLCAPS.test(t) || /^(?:updated|published|manila|quezon city|cebucity|cebuphoto)\b/i.test(t);
    if (!isCredit) break;
    start++;
  }
  let end = list.length - 1;
  while (end > start && (JUNK_TAIL.test(list[end].text.trim()) || (list[end].text.length < 24 && list[end].tag === "p"))) end--;
  return list.slice(start, end + 1);
}

async function fetchArticle(url) {
  const res = await fetch(url, {
    headers: {
      "User-Agent": UA,
      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
      "Accept-Language": "en-PH,en;q=0.9",
      Referer: new URL(url).origin + "/",
    },
    redirect: "follow",
    signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
  });

  // 403/451 is a bot wall, not a bug — the reader shows the honest "read at source"
  // state instead of retrying forever (newsinfo.inquirer.net hard-403s every UA we tried).
  if (!res.ok) return { status: res.status === 403 || res.status === 451 || res.status === 429 ? "blocked" : "error", reason: `upstream ${res.status}` };

  // Read with a byte ceiling so a runaway page cannot eat the box's memory.
  const chunks = [];
  let total = 0;
  if (res.body) {
    for await (const c of res.body) {
      total += c.length || 0;
      chunks.push(c);
      if (total > MAX_BYTES) break;
    }
  } else {
    const buf = await res.arrayBuffer();
    chunks.push(new Uint8Array(buf));
    total = buf.byteLength;
  }
  const html = Buffer.concat(chunks.map((c) => Buffer.from(c))).toString("utf8");
  if (/<(script[^>]*>\s*window\.__sveltekit|js-circleci|recaptcha)/i.test(html) && total < 4000) {
    return { status: "blocked", reason: "bot shell" };
  }

  const { window } = parseHTML(html);
  const doc = window.document;
  const canonical =
    absolutise(doc.querySelector('link[rel="canonical"]')?.getAttribute("href") || "", url) || url;
  const baseHref = canonical || url;

  // Structured body must be read BEFORE the script strip — it lives inside <script> tags.
  // ABS-CBN and other SPA newsrooms render the copy client-side, so Readability alone
  // returns a 60-word shell while the real article sits in __NEXT_DATA__.body_html.
  const sb = structuredBody(doc, html);

  // Strip the usual non-content noise before Readability sees it (script/style/ads/nav).
  for (const sel of ["script", "style", "noscript", "iframe", "form", "svg", "[id*=advert]", "[class*=advert]", "[class*=newsletter]", "[id*=paywall]"]) {
    for (const n of doc.querySelectorAll(sel)) n.remove?.();
  }

  let article = null;
  try {
    article = new Readability(doc.cloneNode(true), { charThreshold: MIN_WORDS }).parse();
  } catch (e) {
    if (!sb) return { status: "error", reason: `readability: ${String(e && e.message || e).slice(0, 80)}` };
  }

  const body = article?.textContent ? String(article.textContent).replace(/\s+/g, " ").trim() : "";
  let words = body ? body.split(" ").length : 0;

  // Prefer whichever source actually has the story.
  let useStructured = false;
  if (sb) {
    const sbWords = String(sb.html).replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim().split(" ").filter(Boolean).length;
    if (sbWords > words) { useStructured = true; words = sbWords; }
  }

  if (!useStructured && (!body || words < MIN_WORDS)) {
    const metaDesc = String(doc.querySelector('meta[property="og:description"], meta[name="description"]')?.getAttribute("content") || "").trim();
    return {
      status: "blocked",
      reason: words ? `thin extract (${words}w)` : "no article body",
      metaDesc: metaDesc.slice(0, 600),
      siteName: article?.siteName || null,
    };
  }

  const parsed = useStructured
    ? parasFrom({ content: String(sb.html).replace(/<\/?(?:html|body|head)[^>]*>/gi, " "),
                  textContent: stripTagsOf(sb.html) }, baseHref)
    : parasFrom(article, baseHref);
  let paras = Array.isArray(parsed) ? parsed : parsed.paras || [];
  // Readability sometimes yields a single giant <div>; fall back to split paragraphs.
  if (paras.length === 1 && paras[0].text.length > 600) {
    paras = paras[0].text.split(/\n{2,}/).map((t) => ({ tag: "p", text: t })).filter((p) => p.text.length > 2);
  }
  paras = paras.filter((p) => p.text).slice(0, MAX_PARAS);

  // Words but no surviving paragraphs is a blank pane, not an article. Reporting it honestly
  // as "blocked" means the reader shows the "read at source" path AND the 12h retry window
  // re-attempts it — whereas status:"ok" with an empty body is cached FOREVER. That is how
  // every SCMP Tech story got stored as words:407 / paras:0 on 2026-08-31.
  if (!paras.length) {
    const metaDesc = String(doc.querySelector('meta[property="og:description"], meta[name="description"]')?.getAttribute("content") || "").trim();
    return {
      status: "blocked",
      reason: `no paragraphs (${words}w extracted)`,
      metaDesc: metaDesc.slice(0, 600),
      siteName: article?.siteName || null,
    };
  }

  return {
    status: "ok",
    title: (article?.title && String(article.title).trim().slice(0, 300)) || null,
    byline: (article?.byline && String(article.byline).trim().slice(0, 160)) || null,
    siteName: (article?.siteName && String(article.siteName).trim().slice(0, 80)) || null,
    excerpt: (article?.excerpt && String(article.excerpt).trim().slice(0, 600)) || null,
    published: article?.publishedTime || null,
    lang: article?.lang || null,
    canonical: baseHref,
    words,
    truncated: total > MAX_BYTES || paras.some((p) => p.truncated),
    images: (Array.isArray(parsed) ? [] : parsed.images) || [],
    paras,
  };
}

// Public entry: the stored body if we have one, otherwise fetch + cache. Never throws.
export async function getArticle({ url, force = false } = {}) {
  if (!url || !/^https?:\/\//i.test(url)) return { status: "error", reason: "bad url" };
  const real = unwrapGoogleNews(url);

  // Known wall: return the honest blocked state immediately rather than spending another
  // 12s upstream on a page that has already refused us this hour.
  const h = hostOf(real);
  if (blockedUntil(h)) {
    const e = blockedHosts.get(h);
    return { status: "blocked", reason: e.reason || "publisher blocks us", paras: [], words: 0, images: [], hostBlocked: true };
  }

  const { stories } = await getCollections();
  const st = await stories.findOne({ url }, { projection: { article: 1, title: 1, source: 1, summary: 1, url: 1, category: 1, image: 1, published: 1 } });
  if (!st) return { status: "error", reason: "story not found" };

  const have = st.article;
  // An "ok" body with no paragraphs IS the blank pane, not a cache hit. Stories stored before
  // the 2026-08-31 parasFrom fix (every SCMP Tech row) look perfectly fresh and would pin a
  // white page for the life of the row, so emptiness is treated as stale and re-fetched.
  const usable = have && have.status === "ok" && Array.isArray(have.paras) && have.paras.some((p) => p && p.text);
  const fresh =
    have &&
    (usable || (have.fetchedAt && Date.now() - new Date(have.fetchedAt).getTime() < RETRY_AFTER_MS));
  if (have && have.status === "ok" && usable && fresh && !force) {
    return { ...have, story: shapeStory(st), realUrl: have.canonical || real };
  }

  if (inflight.has(real)) {
    const r = await inflight.get(real);
    return { ...r, story: shapeStory(st), realUrl: real };
  }

  const job = (async () => {
    try {
      const a = await fetchArticle(real);
      const doc = {
        status: a.status,
        fetchedAt: new Date(),
        reason: a.reason || null,
        ...(a.status === "ok"
          ? {
              title: a.title, byline: a.byline, siteName: a.siteName, excerpt: a.excerpt,
              published: a.published, lang: a.lang, canonical: a.canonical, words: a.words,
              truncated: !!a.truncated, images: a.images, paras: a.paras,
            }
          : { metaDesc: a.metaDesc || null, siteName: a.siteName || null }),
      };
      if (a.status === "blocked" && /upstream 4\d\d/i.test(a.reason || "")) noteBlock(hostOf(real), a.reason);
      else if (a.status === "ok" && h) blockedHosts.delete(h);
      await stories.updateOne({ url }, { $set: { article: doc } });
      // A page we could read deserves its picture too — backfill the card thumb.
      if (a.status === "ok" && !st.image && a.images?.[0]) {
        await stories.updateOne({ url }, { $set: { image: a.images[0] } });
      }
      return doc;
    } catch (e) {
      const msg = String((e && e.message) || e).slice(0, 120);
      const doc = { status: /timeout|abort/i.test(msg) ? "error" : "error", reason: msg, fetchedAt: new Date() };
      await stories.updateOne({ url }, { $set: { article: doc } }).catch(() => {});
      return doc;
    } finally {
      inflight.delete(real);
    }
  })();
  inflight.set(real, job);

  const result = await job;
  return { ...result, story: shapeStory(st), realUrl: real };
}

// Prefetch at crawl time: the reader must feel instant, and we must not be fetching a
// publisher page while the reader is waiting. Bounded budget, low concurrency, and only
// for the stories the paper actually prints.
export async function prefetchArticles(urls, { budget = 12, concurrency = 3, deadlineMs = 35000 } = {}) {
  // Diversity cap: without it, one host at the top of the crawl order consumes the whole
  // budget (that is exactly how Inquirer took 10 of 14 slots on 2026-08-31). Two stories
  // per outlet, and outlets we already know refuse us are skipped entirely.
  const MAX_PER_HOST = 2;
  const perHost = new Map();
  const list = [];
  for (const u of new Set((urls || []).filter(Boolean))) {
    const hh = hostOf(u);
    if (blockedUntil(hh)) continue;
    const n = perHost.get(hh) || 0;
    if (n >= MAX_PER_HOST) continue;
    perHost.set(hh, n + 1);
    list.push(u);
    if (list.length >= budget) break;
  }
  if (!list.length) return { fetched: 0, ok: 0 };
  // A hard wall clock, not just a per-item count: the digest trigger aborts the whole
  // /api/crawl request at 160s, and a slow newsroom (BusinessWorld has been measured at
  // 10s for one page) could otherwise push the AI pass off the end of the tick and freeze
  // the paper at last hour's front page. Warm bodies are a luxury; the edition is not.
  const deadline = Date.now() + deadlineMs;
  const { stories } = await getCollections();
  let fetched = 0;
  let ok = 0;
  let i = 0;
  const workers = Array.from({ length: Math.min(concurrency, list.length) }, async () => {
    while (i < list.length) {
      if (Date.now() > deadline) break;
      const url = list[i++];
      try {
        const existing = await stories.findOne({ url }, { projection: { "article.status": 1 } });
        if (existing?.article?.status === "ok") continue;
        const a = await getArticle({ url });
        fetched++;
        if (a.status === "ok") ok++;
      } catch {
        /* best effort — a failed prefetch only means the reader fetches on demand */
      }
    }
  });
  await Promise.all(workers);
  return { fetched, ok };
}

// What the reader pane needs to draw its own masthead even when extraction failed.
function shapeStory(st) {
  return {
    url: st.url,
    title: st.title,
    source: st.source,
    category: st.category,
    summary: st.summary,
    image: st.image || null,
    published: st.published,
  };
}