news-digest: add world + local headlines sections

Two new dense headline rails above the existing reddit/tech cards.
Designed for high-volume "what happened" coverage where the title
is the deliverable — no LLM summarization, ~15 items per section,
6-column-collapsing grid (title / source / time).

Digest pipeline:
  * fetch_miniflux_headlines(category) — flat list per category, dedup
    by lowercased title (different feeds syndicate the same wire stories)
  * 8h look-back window (vs 12h for tech/reddit) since headlines move
    faster
  * cap of 15 per section (DIGEST_MINIFLUX_HEADLINES_MAX)

Frontend:
  * .headline element parallels .item for the hide-button machinery
    (both have data-id, both honored by app.js)
  * dense 3-col layout collapses to 1-col on narrow screens
  * jumpnav now numbers world=01, local=02, reddit=03, tech=04

Setup:
  * seed-headlines.py — one-shot script (lives in the image at
    /app/seed-headlines.py). Creates the World + Local categories in
    miniflux, subscribes a curated feed list, and renames each feed
    to a short display title (BBC vs "BBC News", "LA Times" vs "California").
    Idempotent — reruns only add new feeds.
  * Default world: BBC, NPR, Al Jazeera. Default local: LA Times Local,
    LA Times CA, Voice of OC. (OC Register blocks miniflux; left out.)
  * entrypoint.sh now syncs templates/{style.css,app.js,favicon.svg}
    to /output on container start so frontend asset updates land
    without a manual copy after rebuild.
This commit is contained in:
vh
2026-04-28 11:14:15 -07:00
parent d2ed7671d7
commit 9bdb41ea6a
9 changed files with 391 additions and 13 deletions
+8
View File
@@ -52,6 +52,14 @@ DIGEST_MAX_PER_SUB=8
DIGEST_MINIFLUX_HOURS=12
DIGEST_MINIFLUX_MAX=8
# Miniflux World/Local headlines (high-volume, dense list, no LLM
# summarization). Categories must exist in miniflux — see
# stacks/news-digest/seed-headlines.py for the one-shot setup.
DIGEST_MINIFLUX_WORLD_CATEGORY=World
DIGEST_MINIFLUX_LOCAL_CATEGORY=Local
DIGEST_MINIFLUX_HEADLINES_HOURS=8
DIGEST_MINIFLUX_HEADLINES_MAX=15
# ── output dir on host (bind-mounted) ────────────────────────────────
# Separate from /opt/docker/conf/<stack>/ to keep generated content
# distinct from config. Owned by container UID; writes are atomic.
+1
View File
@@ -26,6 +26,7 @@ RUN pip install --no-cache-dir requests jinja2 'fastapi>=0.115' 'uvicorn[standar
WORKDIR /app
COPY digest.py /app/digest.py
COPY seed-headlines.py /app/seed-headlines.py
COPY web.py /app/web.py
COPY templates /app/templates
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+4
View File
@@ -40,6 +40,10 @@ services:
- DIGEST_MINIFLUX_HOURS=${DIGEST_MINIFLUX_HOURS:-12}
- DIGEST_MINIFLUX_MAX=${DIGEST_MINIFLUX_MAX:-8}
- DIGEST_MINIFLUX_TECH_CATEGORY=${DIGEST_MINIFLUX_TECH_CATEGORY:-Tech aggregators}
- DIGEST_MINIFLUX_WORLD_CATEGORY=${DIGEST_MINIFLUX_WORLD_CATEGORY:-World}
- DIGEST_MINIFLUX_LOCAL_CATEGORY=${DIGEST_MINIFLUX_LOCAL_CATEGORY:-Local}
- DIGEST_MINIFLUX_HEADLINES_HOURS=${DIGEST_MINIFLUX_HEADLINES_HOURS:-8}
- DIGEST_MINIFLUX_HEADLINES_MAX=${DIGEST_MINIFLUX_HEADLINES_MAX:-15}
volumes:
- ${NEWS_DIGEST_OUTPUT_DIR}:/output
networks:
+85 -2
View File
@@ -74,6 +74,16 @@ MINIFLUX_TECH_CATEGORY = os.environ.get(
MINIFLUX_HOURS = int(os.environ.get("DIGEST_MINIFLUX_HOURS", "12"))
MINIFLUX_MAX_PER_SOURCE = int(os.environ.get("DIGEST_MINIFLUX_MAX", "8"))
# Headlines (world + local) — high-volume sections, no LLM summarization.
MINIFLUX_WORLD_CATEGORY = os.environ.get(
"DIGEST_MINIFLUX_WORLD_CATEGORY", "World"
)
MINIFLUX_LOCAL_CATEGORY = os.environ.get(
"DIGEST_MINIFLUX_LOCAL_CATEGORY", "Local"
)
MINIFLUX_HEADLINES_HOURS = int(os.environ.get("DIGEST_MINIFLUX_HEADLINES_HOURS", "8"))
MINIFLUX_HEADLINES_MAX = int(os.environ.get("DIGEST_MINIFLUX_HEADLINES_MAX", "15"))
TZ_NAME = os.environ.get("TZ", "America/Los_Angeles")
# ── data shapes ──────────────────────────────────────────────────────
@@ -103,6 +113,16 @@ class Source:
href: str # link to the source's homepage / sub
items: list[Item] = field(default_factory=list)
@dataclass
class Headline:
"""One row in the dense world/local headlines list. Skips
LLM summarization — title is the whole point."""
id: str
title: str
url: str
source: str # display name of the originating feed
posted_at: datetime
# ── http session shared across calls ─────────────────────────────────
S = requests.Session()
@@ -190,6 +210,55 @@ def fetch_miniflux_tech_items() -> list[Source]:
))
return [s for s in by_feed.values() if s.items]
def fetch_miniflux_headlines(category_name: str) -> list[Headline]:
"""Pull recent items from a miniflux category as flat headlines.
Used for high-volume sections (world / local) where headlines move
fast and the volume justifies a dense list rather than the per-source
cards used for tech / reddit. No LLM summarization — the title is
the deliverable. Cross-feed dedup by lowercased title (different
feeds syndicate the same wire stories)."""
cats = miniflux_get("/v1/categories")
cat = next(
(c for c in cats if c["title"].lower() == category_name.lower()),
None,
)
if not cat:
log(f"miniflux: category {category_name!r} not found, skipping")
return []
cutoff = int(
(datetime.now(timezone.utc) - timedelta(hours=MINIFLUX_HEADLINES_HOURS)).timestamp()
)
entries = miniflux_get(
"/v1/entries",
category_id=cat["id"],
published_after=cutoff,
order="published_at",
direction="desc",
limit=200,
)
headlines: list[Headline] = []
seen: set[str] = set()
for e in entries.get("entries", []):
title = (e.get("title") or "(untitled)").strip()
key = title.lower()
if key in seen:
continue
seen.add(key)
feed = e.get("feed") or {}
headlines.append(Headline(
id=_stable_id("headline", str(e["id"])),
title=title,
url=e.get("url", ""),
source=feed.get("title", "?"),
posted_at=_parse_dt(e.get("published_at")),
))
if len(headlines) >= MINIFLUX_HEADLINES_MAX:
break
return headlines
def _parse_dt(s: Optional[str]) -> datetime:
if not s:
return datetime.now(timezone.utc)
@@ -312,6 +381,7 @@ def summarize_source(src: Source) -> None:
# ── render ───────────────────────────────────────────────────────────
def render(reddit_sources: list[Source], tech_sources: list[Source],
world_headlines: list[Headline], local_headlines: list[Headline],
generated_at: datetime) -> str:
env = Environment(
loader=FileSystemLoader(str(TEMPLATE_DIR)),
@@ -328,8 +398,12 @@ def render(reddit_sources: list[Source], tech_sources: list[Source],
return template.render(
reddit_sources=reddit_kept,
tech_sources=tech_kept,
world_headlines=world_headlines,
local_headlines=local_headlines,
reddit_total=sum(len(s.items) for s in reddit_kept),
tech_total=sum(len(s.items) for s in tech_kept),
world_total=len(world_headlines),
local_total=len(local_headlines),
generated_at=generated_at,
edition=edition,
edition_short="AM" if edition == "morning" else "PM",
@@ -439,16 +513,25 @@ def main() -> int:
reddit_sources.append(fetch_reddit_top(sub))
time.sleep(1.5) # gentle to anonymous Reddit
log("phase 3: fetching tech-aggregator items from miniflux")
log("phase 3a: fetching world headlines from miniflux")
world_headlines = fetch_miniflux_headlines(MINIFLUX_WORLD_CATEGORY)
log(f" found {len(world_headlines)} world headlines")
log("phase 3b: fetching local headlines from miniflux")
local_headlines = fetch_miniflux_headlines(MINIFLUX_LOCAL_CATEGORY)
log(f" found {len(local_headlines)} local headlines")
log("phase 3c: fetching tech-aggregator items from miniflux")
tech_sources = fetch_miniflux_tech_items()
log(f" found {len(tech_sources)} non-reddit feeds with recent items")
log("phase 4: summarizing each source via llama-swap")
# Headlines bypass the LLM — title is the whole deliverable.
for src in reddit_sources + tech_sources:
summarize_source(src)
log("phase 5: rendering")
html = render(reddit_sources, tech_sources, now_local)
html = render(reddit_sources, tech_sources, world_headlines, local_headlines, now_local)
write_output(html, now_local)
log("done")
return 0
+10
View File
@@ -13,6 +13,16 @@ set -e
mkdir -p /output
# Sync static frontend assets from /app/templates → /output. The web
# container serves /output as static; the assets are baked into the
# image but the bind-mounted /output otherwise wouldn't pick up
# CSS/JS updates on rebuild without a manual copy.
for asset in style.css app.js favicon.svg; do
if [ -f "/app/templates/$asset" ]; then
cp -f "/app/templates/$asset" "/output/$asset"
fi
done
if [ ! -f /output/index.html ]; then
echo "[entrypoint] no /output/index.html yet — running first digest"
/usr/local/bin/run-digest.sh || \
+139
View File
@@ -0,0 +1,139 @@
#!/usr/bin/env python3
"""One-shot setup for the digest's World + Local headlines sections.
Creates the two miniflux categories (idempotent — no-op if present)
and subscribes a curated default feed list into each. Skips any feed
that's already subscribed in miniflux, so reruns are safe.
Run from inside the news-digest-worker container so the MINIFLUX_*
env vars are already set:
docker exec news-digest-worker python3 /app/seed-headlines.py
After this completes, the next digest run picks up the categories
automatically — no app restart needed.
"""
from __future__ import annotations
import os
import sys
import requests
MINIFLUX_URL = os.environ.get("MINIFLUX_URL", "http://miniflux:8080").rstrip("/")
MINIFLUX_USER = os.environ.get("MINIFLUX_USER", "lkraven")
MINIFLUX_PASS = os.environ.get("MINIFLUX_PASSWORD", "")
WORLD_CATEGORY = os.environ.get("DIGEST_MINIFLUX_WORLD_CATEGORY", "World")
LOCAL_CATEGORY = os.environ.get("DIGEST_MINIFLUX_LOCAL_CATEGORY", "Local")
# Default feed lists. Picked for: well-maintained RSS, low duplication
# across the set, mix of styles (wire-service vs editorial). Edit here
# and rerun the script to add/remove — already-subscribed feeds are
# skipped, so adding a new one only adds.
# (feed_url, display_title) — display_title overrides whatever miniflux
# pulls from the feed's <title> element. Keeps the dense headlines rail
# tidy ("BBC" beats "BBC News"; "Al Jazeera" beats the 60-char default).
WORLD_FEEDS = [
("http://feeds.bbci.co.uk/news/world/rss.xml", "BBC"),
("https://feeds.npr.org/1001/rss.xml", "NPR"),
("https://www.aljazeera.com/xml/rss/all.xml", "Al Jazeera"),
]
LOCAL_FEEDS = [
("https://www.latimes.com/local/rss2.0.xml", "LA Times"),
("https://www.latimes.com/california/rss2.0.xml", "LA Times CA"),
("https://voiceofoc.org/feed/", "Voice of OC"),
# OC Register blocks miniflux's fetcher (403). KTLA / Daily Pilot
# are options if more OC-specific coverage is needed later.
]
S = requests.Session()
S.auth = (MINIFLUX_USER, MINIFLUX_PASS)
def get(path: str) -> object:
r = S.get(f"{MINIFLUX_URL}{path}", timeout=20)
r.raise_for_status()
return r.json()
def post(path: str, body: dict) -> requests.Response:
return S.post(f"{MINIFLUX_URL}{path}", json=body, timeout=30)
def ensure_category(title: str) -> int:
"""Return the ID of the category named `title`, creating it if missing."""
cats = get("/v1/categories")
for c in cats:
if c["title"].lower() == title.lower():
print(f" category {title!r} already exists (id={c['id']})")
return c["id"]
r = post("/v1/categories", {"title": title})
r.raise_for_status()
cat_id = r.json()["id"]
print(f" created category {title!r} (id={cat_id})")
return cat_id
def find_feed_id(feed_url: str) -> int | None:
"""Return miniflux feed ID for `feed_url` if subscribed; else None.
Compares with both the original URL and miniflux's canonical
rewrite (some feeds get http→https'd or trailing-slashed at fetch
time)."""
feeds = get("/v1/feeds")
target = feed_url.rstrip("/")
for f in feeds:
url = (f.get("feed_url") or "").rstrip("/")
if url == target or url == target.replace("http://", "https://", 1):
return f["id"]
return None
def set_feed_title(feed_id: int, title: str) -> None:
r = S.put(f"{MINIFLUX_URL}/v1/feeds/{feed_id}", json={"title": title}, timeout=20)
if not r.ok:
print(f" ! rename failed ({r.status_code}): {r.text[:120]}")
def subscribe(feed_url: str, display_title: str, category_id: int) -> None:
existing = find_feed_id(feed_url)
if existing is not None:
print(f" skip (already subscribed): {feed_url}")
set_feed_title(existing, display_title)
return
r = post("/v1/feeds", {"feed_url": feed_url, "category_id": category_id})
if r.status_code in (200, 201):
new_id = r.json().get("feed_id")
print(f" + subscribed: {feed_url}")
if new_id:
set_feed_title(new_id, display_title)
else:
# Most common failure mode is a feed-discovery hiccup at
# miniflux's end — log loudly and continue so one bad URL
# doesn't block the rest.
print(f" ! FAILED ({r.status_code}): {feed_url} {r.text[:200]}")
def main() -> int:
if not MINIFLUX_PASS:
print("MINIFLUX_PASSWORD not set — bailing", file=sys.stderr)
return 2
print(f"miniflux: {MINIFLUX_URL} user={MINIFLUX_USER}")
print(f"\n→ {WORLD_CATEGORY}")
world_id = ensure_category(WORLD_CATEGORY)
for url, title in WORLD_FEEDS:
subscribe(url, title, world_id)
print(f"\n→ {LOCAL_CATEGORY}")
local_id = ensure_category(LOCAL_CATEGORY)
for url, title in LOCAL_FEEDS:
subscribe(url, title, local_id)
print("\ndone — next digest run will pick up the new sections.")
return 0
if __name__ == "__main__":
sys.exit(main())
+6 -5
View File
@@ -30,9 +30,10 @@
const TRAY_COUNT = document.getElementById("hidden-tray-count");
const TRAY_TOGGLE = TRAY && TRAY.querySelector(".hidden-tray-toggle");
/** Map<id, HTMLElement> — every .item on this page, keyed by data-id */
/** Map<id, HTMLElement> — every hideable element on this page, keyed by
* data-id. Both .item (cards) and .headline (compact rows) qualify. */
const itemsById = new Map();
document.querySelectorAll(".item[data-id]").forEach((el) => {
document.querySelectorAll(".item[data-id], .headline[data-id]").forEach((el) => {
itemsById.set(el.dataset.id, el);
});
@@ -76,7 +77,7 @@
src.classList.toggle("is-empty", visible === 0);
});
document.querySelectorAll(".desk").forEach((desk) => {
const visibleItems = desk.querySelectorAll(".item:not(.is-hidden)").length;
const visibleItems = desk.querySelectorAll(".item:not(.is-hidden), .headline:not(.is-hidden)").length;
const badge = desk.querySelector(".desk-count");
if (badge) badge.textContent = `${visibleItems} items`;
desk.classList.toggle("is-empty", visibleItems === 0);
@@ -130,7 +131,7 @@
}
function titleOf(el) {
const t = el.querySelector(".item-title");
const t = el.querySelector(".item-title, .headline-title");
return (t && t.textContent.trim()) || "(untitled)";
}
@@ -187,7 +188,7 @@
document.addEventListener("click", (ev) => {
const btn = ev.target.closest(".item-hide");
if (!btn) return;
const item = btn.closest(".item[data-id]");
const item = btn.closest("[data-id]");
if (!item) return;
ev.preventDefault();
hideItem(item.dataset.id);
+60 -6
View File
@@ -27,7 +27,7 @@
</div>
<div class="meta-row">
<span class="meta-label">DESK</span>
<span class="meta-value">SUBREDDITS · TECH FEEDS</span>
<span class="meta-value">WORLD · LOCAL · SUBREDDITS · TECH FEEDS</span>
</div>
<div class="meta-row">
<span class="meta-label">CURATED BY</span>
@@ -43,22 +43,76 @@
{# ────── nav strip ────── #}
<nav class="jumpnav" aria-label="sections">
<ol>
{% if world_headlines %}
<li><a href="#world"><span class="jump-num">01</span> WORLD</a></li>
{% endif %}
{% if local_headlines %}
<li><a href="#local"><span class="jump-num">02</span> LOCAL</a></li>
{% endif %}
{% if reddit_sources %}
<li><a href="#reddit"><span class="jump-num">01</span> REDDIT</a></li>
<li><a href="#reddit"><span class="jump-num">03</span> REDDIT</a></li>
{% endif %}
{% if tech_sources %}
<li><a href="#tech"><span class="jump-num">02</span> TECH FEEDS</a></li>
<li><a href="#tech"><span class="jump-num">04</span> TECH FEEDS</a></li>
{% endif %}
</ol>
</nav>
<main class="brief">
{# ────── WORLD HEADLINES ────── #}
{% if world_headlines %}
<section id="world" class="desk desk-headlines">
<header class="desk-head">
<span class="desk-num">01</span>
<h2 class="desk-title">World</h2>
<span class="desk-sub">top of last {{ generated_at.hour < 14 and "8" or "8" }} hours · headlines</span>
<span class="desk-count">{{ world_total }} items</span>
</header>
<ol class="headlines-list">
{% for h in world_headlines %}
<li class="headline" data-id="{{ h.id }}">
<button class="item-hide" type="button" title="Hide this headline" aria-label="Hide this headline">×</button>
<h3 class="headline-title">
<a href="{{ h.url }}" target="_blank" rel="noopener">{{ h.title }}</a>
</h3>
<span class="headline-source">{{ h.source }}</span>
<span class="headline-time">{{ h.posted_at|humanago }}</span>
</li>
{% endfor %}
</ol>
</section>
{% endif %}
{# ────── LOCAL HEADLINES ────── #}
{% if local_headlines %}
<section id="local" class="desk desk-headlines">
<header class="desk-head">
<span class="desk-num">02</span>
<h2 class="desk-title">Local</h2>
<span class="desk-sub">orange county · greater LA · last 8 hours</span>
<span class="desk-count">{{ local_total }} items</span>
</header>
<ol class="headlines-list">
{% for h in local_headlines %}
<li class="headline" data-id="{{ h.id }}">
<button class="item-hide" type="button" title="Hide this headline" aria-label="Hide this headline">×</button>
<h3 class="headline-title">
<a href="{{ h.url }}" target="_blank" rel="noopener">{{ h.title }}</a>
</h3>
<span class="headline-source">{{ h.source }}</span>
<span class="headline-time">{{ h.posted_at|humanago }}</span>
</li>
{% endfor %}
</ol>
</section>
{% endif %}
{# ────── REDDIT SECTION ────── #}
{% if reddit_sources %}
<section id="reddit" class="desk">
<header class="desk-head">
<span class="desk-num">01</span>
<span class="desk-num">03</span>
<h2 class="desk-title">Reddit</h2>
<span class="desk-sub">top of last {{ generated_at.hour < 14 and "12" or "12" }} hours · score-filtered</span>
<span class="desk-count">{{ reddit_total }} items</span>
@@ -127,7 +181,7 @@
{% if tech_sources %}
<section id="tech" class="desk">
<header class="desk-head">
<span class="desk-num">02</span>
<span class="desk-num">04</span>
<h2 class="desk-title">Tech Feeds</h2>
<span class="desk-sub">non-reddit · last {{ generated_at.hour < 14 and "12" or "12" }} hours</span>
<span class="desk-count">{{ tech_total }} items</span>
@@ -184,7 +238,7 @@
</section>
{% endif %}
{% if not reddit_sources and not tech_sources %}
{% if not reddit_sources and not tech_sources and not world_headlines and not local_headlines %}
<section class="desk empty">
<p class="empty-msg">
No items cleared the filters in the last window.<br>
+78
View File
@@ -490,6 +490,71 @@ a:hover { color: var(--accent); }
.item[data-tag="meme"] { opacity: .58; }
.item[data-tag="other"] { opacity: .85; }
/* -------------------------------------------------------------------------
headlines — dense list (world / local).
Different rhythm from .item cards: one row per story, source on the
right, title-led. Volume-driven sections where the deliverable is
"what happened" not "what someone said about what happened".
------------------------------------------------------------------------- */
.headlines-list {
list-style: none;
margin: 0;
padding: 0;
display: grid;
gap: 0;
}
.headline {
position: relative; /* anchors the absolute-positioned × button */
display: grid;
grid-template-columns: 1fr auto auto;
align-items: baseline;
gap: 16px;
padding: 12px 0;
border-bottom: 1px solid var(--rule);
}
.headline:last-child {
border-bottom: 0;
}
.headline-title {
margin: 0;
font-family: var(--font-display);
font-size: 17px;
line-height: 1.35;
font-weight: 500;
letter-spacing: -0.005em;
color: var(--fg);
font-variation-settings: "opsz" 24;
/* Title takes the leftmost column; allow it to wrap on narrow screens. */
min-width: 0;
}
.headline-title a { color: inherit; }
.headline-title a:hover { color: var(--accent); }
.headline-source {
font-family: var(--font-mono);
font-size: 10px;
font-weight: 600;
letter-spacing: .12em;
color: var(--fg-faint);
text-transform: uppercase;
white-space: nowrap;
}
.headline-time {
font-family: var(--font-mono);
font-size: 10.5px;
color: var(--fg-faint);
font-variant-numeric: tabular-nums;
min-width: 32px;
text-align: right;
}
/* Hidden state — same mechanism as .item, applies to .headline too. */
.headline.is-hidden { display: none; }
/* -------------------------------------------------------------------------
archive page — list of all editions
------------------------------------------------------------------------- */
@@ -789,4 +854,17 @@ a:hover { color: var(--accent); }
bigger tap target too. */
.item-hide { opacity: 1; width: 28px; height: 28px; font-size: 20px; }
.hidden-tray { padding: 12px 14px; }
/* Headlines: stack source + time below title on narrow screens so
the title gets the full width. */
.headline {
grid-template-columns: 1fr;
gap: 4px;
padding: 10px 0;
}
.headline-source,
.headline-time {
grid-column: 1 / 2;
}
.headline-time { text-align: left; }
}