news-digest: real article summaries + per-desk collapse

Two upgrades to make the digest actually readable:

1) Article-grounded 2-3 sentence summaries (everywhere)

   The old prompt got just the title + miniflux's content excerpt,
   which for HN/Lobsters/wire feeds is barely more than the title
   itself — so summaries paraphrased the title and added nothing.

   Now every URL gets fetched and main-content-extracted via
   trafilatura on a parallel pre-pass (10 workers, ~15s for ~50
   URLs). Extracted text caches to /output/.article-cache.json with
   a 7-day TTL so repeat runs in the same window don't re-pull.

   Headlines also get summarized now — one batched LLM call per
   category (world / local). Rendered as a paragraph below the
   title with source + time on the right rail.

   Prompt rewrites tell the model to pull names/numbers/places
   from the body and explicitly forbid restating the title.
   Result: real specifics ("71% saw no pay increase globally",
   "third time in less than two weeks", "Islamabad and Moscow
   intermediaries") instead of title paraphrase.

2) Per-desk collapse buttons

   Chevron next to .desk-count toggles a .is-collapsed class.
   Collapsed state is per-device (localStorage by section id) since
   collapse is a viewing preference, not content state.

stacks/news-digest/Dockerfile

@@ -17,12 +17,21 @@ ENV PYTHONUNBUFFERED=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1
 
 # tzdata so $TZ works for cron + datetime; tini so signals propagate cleanly.
-RUN apk add --no-cache tzdata tini bash
+RUN apk add --no-cache tzdata tini bash curl
 
 # fastapi + uvicorn[standard] for the web container; requests + jinja2
 # for the worker. Both shipped in both containers — neither set is
 # heavy enough to justify splitting the image.
-RUN pip install --no-cache-dir requests jinja2 'fastapi>=0.115' 'uvicorn[standard]>=0.30'
+# trafilatura: main-content extractor for the article-summary upgrade
+# (worker only — it pulls lxml + a handful of HTML utils, ~80 MB total).
+# libxml2-dev/libxslt-dev are for lxml's musl wheels. apk caches are
+# cleaned in the same RUN to keep the layer small.
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev libxml2-dev libxslt-dev \
+    && apk add --no-cache libxml2 libxslt \
+    && pip install --no-cache-dir \
+        requests jinja2 trafilatura \
+        'fastapi>=0.115' 'uvicorn[standard]>=0.30' \
+    && apk del .build-deps
 
 WORKDIR /app
 COPY digest.py /app/digest.py

stacks/news-digest/digest.py

@@ -26,6 +26,7 @@ import os
 import re
 import sys
 import time
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
@@ -115,13 +116,13 @@ class Source:
 
 @dataclass
 class Headline:
-    """One row in the dense world/local headlines list. Skips
-    LLM summarization — title is the whole point."""
+    """One row in the dense world/local headlines list."""
     id: str
     title: str
     url: str
     source: str          # display name of the originating feed
     posted_at: datetime
+    tldr: str = ""       # 2-3 sentence LLM summary of the linked article
 
 # ── http session shared across calls ─────────────────────────────────
 
@@ -131,6 +132,91 @@ S.headers["User-Agent"] = REDDIT_USER_AGENT
 def log(msg: str) -> None:
     print(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}", flush=True)
 
+# ── article-text cache ───────────────────────────────────────────────
+# Most feeds ship just titles + thin excerpts. Real summaries need the
+# article body, so we fetch + extract with trafilatura. Cache to disk
+# so re-runs on the same window don't re-pull.
+ARTICLE_CACHE_PATH = OUTPUT_DIR / ".article-cache.json"
+ARTICLE_CACHE_TTL_HOURS = 7 * 24       # keep extracted text ~1 week
+ARTICLE_FETCH_TIMEOUT = 12             # seconds per URL
+ARTICLE_TEXT_CAP = 4000                # chars; LLM doesn't need more
+ARTICLE_FETCH_WORKERS = 10             # parallel fetches per warm pass
+REDDIT_DOMAIN_RE = re.compile(r"^https?://(?:[^/]*\.)?reddit\.com/", re.I)
+
+
+def article_cache_load() -> dict:
+    if not ARTICLE_CACHE_PATH.exists():
+        return {}
+    try:
+        return json.loads(ARTICLE_CACHE_PATH.read_text())
+    except Exception:
+        return {}
+
+
+def article_cache_save(cache: dict) -> None:
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    tmp = ARTICLE_CACHE_PATH.with_suffix(".json.tmp")
+    tmp.write_text(json.dumps(cache))
+    tmp.rename(ARTICLE_CACHE_PATH)
+
+
+def fetch_article_text(url: str, cache: dict) -> str:
+    """Return main-content text for `url`, cached. Empty string on any
+    failure — caller is expected to fall back to the feed body / title.
+
+    Skips reddit.com URLs (callers already have selftext as `body`)
+    and anything that 404s, paywalls, or extracts to less than a
+    paragraph."""
+    if not url or REDDIT_DOMAIN_RE.match(url):
+        return ""
+    key = hashlib.sha1(url.encode("utf-8")).hexdigest()
+    now = int(time.time())
+    cached = cache.get(key)
+    if cached and (now - int(cached.get("ts", 0))) < ARTICLE_CACHE_TTL_HOURS * 3600:
+        return cached.get("text", "")
+    try:
+        import trafilatura
+        downloaded = trafilatura.fetch_url(url)
+        if not downloaded:
+            cache[key] = {"ts": now, "text": ""}
+            return ""
+        text = trafilatura.extract(
+            downloaded,
+            include_comments=False,
+            include_tables=False,
+            no_fallback=False,
+        ) or ""
+        text = text.strip()[:ARTICLE_TEXT_CAP]
+        cache[key] = {"ts": now, "text": text}
+        return text
+    except Exception as e:
+        log(f"  ! article fetch failed for {url[:80]}: {e!r}")
+        cache[key] = {"ts": now, "text": ""}
+        return ""
+
+
+def warm_article_cache(urls: Iterable[str], cache: dict) -> None:
+    """Parallel-prefetch article text for `urls` into `cache`."""
+    pending = []
+    seen_urls: set[str] = set()
+    cutoff = int(time.time()) - ARTICLE_CACHE_TTL_HOURS * 3600
+    for url in urls:
+        if not url or url in seen_urls or REDDIT_DOMAIN_RE.match(url):
+            continue
+        seen_urls.add(url)
+        key = hashlib.sha1(url.encode("utf-8")).hexdigest()
+        cached = cache.get(key)
+        if cached and int(cached.get("ts", 0)) > cutoff:
+            continue
+        pending.append(url)
+    if not pending:
+        return
+    log(f"  warming article cache: {len(pending)} URLs ({ARTICLE_FETCH_WORKERS} parallel)")
+    t0 = time.time()
+    with ThreadPoolExecutor(max_workers=ARTICLE_FETCH_WORKERS) as ex:
+        list(ex.map(lambda u: fetch_article_text(u, cache), pending))
+    log(f"  done in {time.time() - t0:.1f}s")
+
 # ── miniflux: discover subreddits + pull tech-aggregator items ───────
 
 def miniflux_get(path: str, **params) -> Any:
@@ -312,16 +398,18 @@ def fetch_reddit_top(sub: str) -> Source:
 SUMMARIZE_SYSTEM = (
     "You are a curator producing a tight intelligence briefing for an "
     "engineer who reads many feeds. You are concise, neutral, and never "
-    "editorialize. You skip pure shitposts and screenshots-without-context."
+    "editorialize. You write summaries grounded in the article body — "
+    "never paraphrase the title back at the reader. You skip pure "
+    "shitposts and screenshots-without-context."
 )
 
 SUMMARIZE_USER_TEMPLATE = """Given the {n} posts from {source} below, return a JSON ARRAY where each element has:
 
-- "id":  the post id from the input
-- "tldr": ONE sentence, 25 words max, capturing the substantive point. Lead with a verb. No "this post discusses". No "a user shares".
-- "tag": ONE word from {{news, tutorial, release, discussion, question, showcase, drama, meme, other}}
+- "id":   the post id from the input
+- "tldr": 2-3 sentences (40-80 words) summarizing the SUBSTANCE — what happened, what was announced, what conclusion the author drew. Pull facts, names, numbers from the body. Do NOT restate the title; the reader already sees it. Do NOT begin with "this post" / "the article" / "a user". If the body is too thin to add anything beyond the title, return tldr="".
+- "tag":  ONE word from {{news, tutorial, release, discussion, question, showcase, drama, meme, other}}
 
-If a post is a pure shitpost / screenshot-without-context / duplicate of an item already in this batch, set "tldr" to "" and "tag" to "skip".
+If a post is a pure shitpost / screenshot-without-context / duplicate of another item in this batch, set "tldr" to "" and "tag" to "skip".
 
 Output ONLY the JSON array. No prose, no markdown fence.
 
@@ -329,11 +417,60 @@ POSTS:
 {posts_json}
 """
 
-def summarize_source(src: Source) -> None:
+HEADLINE_SUMMARIZE_USER_TEMPLATE = """Given the {n} {label} headlines below, return a JSON ARRAY where each element has:
+
+- "id":   the headline id from the input
+- "tldr": 2-3 sentences (40-80 words) summarizing the article body — who, what, when, where, why. Pull names, numbers, places from the body. Do NOT restate the headline; the reader already sees it. Do NOT editorialize. If the body is too thin (e.g. just the headline rehashed), return tldr="".
+
+Output ONLY the JSON array. No prose, no markdown fence.
+
+HEADLINES:
+{posts_json}
+"""
+
+
+def _llm_chat(messages: list[dict], label: str) -> dict[str, dict]:
+    """Send a chat request and parse the JSON-array reply into a
+    {id: row} map. Returns {} on any failure (caller falls back to
+    raw titles)."""
+    try:
+        r = S.post(
+            f"{LLAMA_SWAP_URL.rstrip('/')}/v1/chat/completions",
+            json={
+                "model": LLAMA_SWAP_MODEL,
+                "messages": messages,
+                "temperature": 0.2,
+                "max_tokens": 4000,
+            },
+            timeout=LLAMA_SWAP_TIMEOUT,
+        )
+        r.raise_for_status()
+        msg = r.json()["choices"][0]["message"]
+        # Extended-thinking models (Qwen3.x) put output in
+        # reasoning_content while content is still streaming. Fall back
+        # so we get something to parse.
+        content = (msg.get("content") or msg.get("reasoning_content") or "").strip()
+        # Some models wrap JSON in ```...``` even when told not to.
+        content = re.sub(r"^```(?:json)?\s*|\s*```$", "", content, flags=re.M).strip()
+        return {x.get("id"): x for x in json.loads(content)}
+    except Exception as e:
+        log(f"  ! llm failed for {label}: {e!r}")
+        return {}
+
+
+def summarize_source(src: Source, cache: dict) -> None:
     if not src.items:
         return
     posts_json = json.dumps([
-        {"id": it.id, "title": it.title, "body": it.body[:600], "url": it.url}
+        {
+            "id": it.id,
+            "title": it.title,
+            # Real article text (cached) wins over feed-shipped excerpt.
+            # Falls back to feed body for self-posts (Reddit selftext)
+            # and any URL where extraction failed.
+            "body": (fetch_article_text(it.url, cache) or it.body or "")[:2500],
+            "url": it.url,
+        }
         for it in src.items
     ], ensure_ascii=False)
     user = SUMMARIZE_USER_TEMPLATE.format(
@@ -342,31 +479,14 @@ def summarize_source(src: Source) -> None:
         posts_json=posts_json,
     )
     log(f"  llm: summarizing {len(src.items)} items from {src.name}")
-    try:
-        r = S.post(
-            f"{LLAMA_SWAP_URL.rstrip('/')}/v1/chat/completions",
-            json={
-                "model": LLAMA_SWAP_MODEL,
-                "messages": [
-                    {"role": "system", "content": SUMMARIZE_SYSTEM},
-                    {"role": "user", "content": user},
-                ],
-                "temperature": 0.2,
-                "max_tokens": 2000,
-            },
-            timeout=LLAMA_SWAP_TIMEOUT,
-        )
-        r.raise_for_status()
-        msg = r.json()["choices"][0]["message"]
-        # Models in extended-thinking mode (e.g. Qwen3.x defaults) put
-        # output in reasoning_content and leave content empty until they
-        # exit thinking — fall back so we get *something* to parse.
-        content = (msg.get("content") or msg.get("reasoning_content") or "").strip()
-        # Some models wrap JSON in ```...``` even when told not to.
-        content = re.sub(r"^```(?:json)?\s*|\s*```$", "", content, flags=re.M).strip()
-        mapped = {x.get("id"): x for x in json.loads(content)}
-    except Exception as e:
-        log(f"  ! llm failed for {src.name}: {e!r} — keeping raw titles")
+    mapped = _llm_chat(
+        [
+            {"role": "system", "content": SUMMARIZE_SYSTEM},
+            {"role": "user", "content": user},
+        ],
+        src.name,
+    )
+    if not mapped:
         return
 
     for it in src.items:
@@ -378,6 +498,41 @@ def summarize_source(src: Source) -> None:
     src.items = [it for it in src.items if it.tag != "skip" and (it.tldr or it.score is None)]
     log(f"  -> {len(src.items)} kept after llm filter")
 
+
+def summarize_headlines(headlines: list[Headline], label: str, cache: dict) -> None:
+    """Batch-summarize a headline list in-place. One LLM call for the
+    whole batch. Quietly leaves tldr empty on failure so the dense
+    list still renders (just without summaries)."""
+    if not headlines:
+        return
+    posts_json = json.dumps([
+        {
+            "id": h.id,
+            "title": h.title,
+            "source": h.source,
+            "body": fetch_article_text(h.url, cache)[:2000],
+        }
+        for h in headlines
+    ], ensure_ascii=False)
+    user = HEADLINE_SUMMARIZE_USER_TEMPLATE.format(
+        n=len(headlines),
+        label=label,
+        posts_json=posts_json,
+    )
+    log(f"  llm: summarizing {len(headlines)} {label} headlines")
+    mapped = _llm_chat(
+        [
+            {"role": "system", "content": SUMMARIZE_SYSTEM},
+            {"role": "user", "content": user},
+        ],
+        f"{label} headlines",
+    )
+    if not mapped:
+        return
+    for h in headlines:
+        m = mapped.get(h.id, {})
+        h.tldr = (m.get("tldr") or "").strip()
+
 # ── render ───────────────────────────────────────────────────────────
 
 def render(reddit_sources: list[Source], tech_sources: list[Source],
@@ -525,10 +680,25 @@ def main() -> int:
     tech_sources = fetch_miniflux_tech_items()
     log(f"  found {len(tech_sources)} non-reddit feeds with recent items")
 
-    log("phase 4: summarizing each source via llama-swap")
-    # Headlines bypass the LLM — title is the whole deliverable.
+    log("phase 3d: warming article-text cache (parallel)")
+    article_cache = article_cache_load()
+    all_urls: list[str] = []
+    for src in tech_sources + reddit_sources:
+        for it in src.items:
+            all_urls.append(it.url)
+    for h in world_headlines + local_headlines:
+        all_urls.append(h.url)
+    warm_article_cache(all_urls, article_cache)
+
+    log("phase 4a: summarizing reddit + tech sources via llama-swap")
     for src in reddit_sources + tech_sources:
-        summarize_source(src)
+        summarize_source(src, article_cache)
+
+    log("phase 4b: summarizing world + local headlines via llama-swap")
+    summarize_headlines(world_headlines, "world", article_cache)
+    summarize_headlines(local_headlines, "local", article_cache)
+
+    article_cache_save(article_cache)
 
     log("phase 5: rendering")
     html = render(reddit_sources, tech_sources, world_headlines, local_headlines, now_local)

stacks/news-digest/templates/app.js

@@ -201,6 +201,38 @@
     });
   }
 
+  /* ── desk collapse (per-device, localStorage) ───────────────────── */
+  // Stored as a JSON object {deskId: true} of collapsed desks. Per
+  // device by design — collapse is a viewing preference, not content
+  // state, so no server roundtrip.
+  const COLLAPSE_KEY = "digest:collapsed-desks";
+  function readCollapsed() {
+    try { return JSON.parse(localStorage.getItem(COLLAPSE_KEY) || "{}"); }
+    catch (_) { return {}; }
+  }
+  function writeCollapsed(obj) {
+    try { localStorage.setItem(COLLAPSE_KEY, JSON.stringify(obj)); } catch (_) {}
+  }
+  function applyCollapseState() {
+    const state = readCollapsed();
+    document.querySelectorAll(".desk[id]").forEach((desk) => {
+      desk.classList.toggle("is-collapsed", !!state[desk.id]);
+    });
+  }
+  applyCollapseState();
+  document.addEventListener("click", (ev) => {
+    const btn = ev.target.closest(".desk-collapse");
+    if (!btn) return;
+    const desk = btn.closest(".desk[id]");
+    if (!desk) return;
+    ev.preventDefault();
+    desk.classList.toggle("is-collapsed");
+    const state = readCollapsed();
+    if (desk.classList.contains("is-collapsed")) state[desk.id] = true;
+    else delete state[desk.id];
+    writeCollapsed(state);
+  });
+
   /* ── initial paint ──────────────────────────────────────────────── */
 
   apiGetHidden().then((hidden) => {

stacks/news-digest/templates/digest.html.j2

@@ -72,6 +72,7 @@
           <h2 class="desk-title">World</h2>
           <span class="desk-sub">top of last {{ generated_at.hour < 14 and "8" or "8" }} hours · headlines</span>
           <span class="desk-count">{{ world_total }} items</span>
+          <button class="desk-collapse" type="button" aria-label="Collapse section" title="Collapse / expand">▾</button>
         </header>
         <ol class="headlines-list">
           {% for h in world_headlines %}
@@ -82,6 +83,9 @@
               </h3>
               <span class="headline-source">{{ h.source }}</span>
               <span class="headline-time">{{ h.posted_at|humanago }}</span>
+              {% if h.tldr %}
+                <p class="headline-tldr">{{ h.tldr }}</p>
+              {% endif %}
             </li>
           {% endfor %}
         </ol>
@@ -96,6 +100,7 @@
           <h2 class="desk-title">Local</h2>
           <span class="desk-sub">orange county · greater LA · last 8 hours</span>
           <span class="desk-count">{{ local_total }} items</span>
+          <button class="desk-collapse" type="button" aria-label="Collapse section" title="Collapse / expand">▾</button>
         </header>
         <ol class="headlines-list">
           {% for h in local_headlines %}
@@ -106,6 +111,9 @@
               </h3>
               <span class="headline-source">{{ h.source }}</span>
               <span class="headline-time">{{ h.posted_at|humanago }}</span>
+              {% if h.tldr %}
+                <p class="headline-tldr">{{ h.tldr }}</p>
+              {% endif %}
             </li>
           {% endfor %}
         </ol>
@@ -120,6 +128,7 @@
           <h2 class="desk-title">Reddit</h2>
           <span class="desk-sub">top of last {{ generated_at.hour < 14 and "12" or "12" }} hours · score-filtered</span>
           <span class="desk-count">{{ reddit_total }} items</span>
+          <button class="desk-collapse" type="button" aria-label="Collapse section" title="Collapse / expand">▾</button>
         </header>
 
         {% for src in reddit_sources %}
@@ -189,6 +198,7 @@
           <h2 class="desk-title">Tech Feeds</h2>
           <span class="desk-sub">non-reddit · last {{ generated_at.hour < 14 and "12" or "12" }} hours</span>
           <span class="desk-count">{{ tech_total }} items</span>
+          <button class="desk-collapse" type="button" aria-label="Collapse section" title="Collapse / expand">▾</button>
         </header>
 
         {% for src in tech_sources %}

stacks/news-digest/templates/style.css

@@ -305,6 +305,47 @@ a:hover { color: var(--accent); }
   text-transform: uppercase;
 }
 
+/* Per-desk collapse toggle. Sits on the right edge of the desk header,
+   clicking collapses the body to a single-line summary. State is
+   persisted in localStorage by section id (#world, #local, etc), so
+   each device remembers per-user preference. */
+.desk-collapse {
+  background: transparent;
+  border: 1px solid transparent;
+  border-radius: 4px;
+  padding: 2px 6px;
+  margin-left: 12px;
+  font-family: var(--font-mono);
+  font-size: 14px;
+  line-height: 1;
+  color: var(--fg-dim);
+  cursor: pointer;
+  transition: color 0.15s ease, border-color 0.15s ease, background 0.15s ease, transform 0.18s ease;
+}
+.desk-collapse:hover,
+.desk-collapse:focus-visible {
+  color: var(--accent);
+  border-color: var(--rule-bright);
+  background: var(--bg-elev);
+  outline: none;
+}
+.desk.is-collapsed .desk-collapse {
+  transform: rotate(-90deg);
+}
+/* When collapsed, hide everything inside the desk EXCEPT the header.
+   The .desk-head is direct child; .source / .headlines-list etc are
+   the rest of the body. */
+.desk.is-collapsed > :not(.desk-head) {
+  display: none;
+}
+/* Tighten the header bottom-border in collapsed state — no items below
+   means the heavy underline looks weird floating alone. */
+.desk.is-collapsed .desk-head {
+  margin-bottom: 0;
+  padding-bottom: 8px;
+  border-bottom: 1px dashed var(--rule);
+}
+
 /* -------------------------------------------------------------------------
    per-source block
    ------------------------------------------------------------------------- */
@@ -509,11 +550,25 @@ a:hover { color: var(--accent); }
   position: relative;       /* anchors the absolute-positioned × button */
   display: grid;
   grid-template-columns: 1fr auto auto;
+  grid-template-rows: auto auto;
   align-items: baseline;
-  gap: 16px;
-  padding: 12px 0;
+  column-gap: 16px;
+  row-gap: 4px;
+  padding: 14px 0;
   border-bottom: 1px solid var(--rule);
 }
+.headline-title  { grid-column: 1 / 2; grid-row: 1; }
+.headline-source { grid-column: 2 / 3; grid-row: 1; }
+.headline-time   { grid-column: 3 / 4; grid-row: 1; }
+.headline-tldr {
+  grid-column: 1 / 4;
+  grid-row: 2;
+  margin: 4px 0 2px;
+  font-family: var(--font-sans);
+  font-size: 14px;
+  line-height: 1.5;
+  color: var(--fg-dim);
+}
 .headline:last-child {
   border-bottom: 0;
 }
@@ -855,16 +910,20 @@ a:hover { color: var(--accent); }
   .item-hide { opacity: 1; width: 28px; height: 28px; font-size: 20px; }
   .hidden-tray { padding: 12px 14px; }
 
-  /* Headlines: stack source + time below title on narrow screens so
-     the title gets the full width. */
+  /* Headlines: stack title / source / time / tldr vertically on narrow
+     screens so the title gets the full width. */
   .headline {
     grid-template-columns: 1fr;
     gap: 4px;
-    padding: 10px 0;
+    padding: 12px 0;
   }
+  .headline-title,
   .headline-source,
-  .headline-time {
+  .headline-time,
+  .headline-tldr {
     grid-column: 1 / 2;
+    grid-row: auto;
   }
   .headline-time { text-align: left; }
+  .headline-tldr { font-size: 13.5px; }
 }