From df526708718497d2c15df82fb24bfb4b94b11c85 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:03:12 -0700 Subject: [PATCH] parsing --- package.json | 1 + pnpm-lock.yaml | 3 +++ src/widgets/feed/utils.js | 26 ++++++++++++++++++++- src/widgets/feed/utils.test.js | 42 ++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 94f3fb729..1922d9b03 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "react-i18next": "^17.0.12", "react-icons": "^5.6.0", "recharts": "^3.1.2", + "sax": "^1.6.1", "swr": "^2.5.1", "systeminformation": "^5.33.1", "tough-cookie": "^6.0.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 58e45faaa..467952842 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -83,6 +83,9 @@ importers: recharts: specifier: ^3.1.2 version: 3.10.1(@types/react@19.2.18)(react-dom@19.2.8(react@19.2.8))(react-is@17.0.2)(react@19.2.8)(redux@5.0.1) + sax: + specifier: ^1.6.1 + version: 1.6.1 swr: specifier: ^2.5.1 version: 2.5.1(react@19.2.8) diff --git a/src/widgets/feed/utils.js b/src/widgets/feed/utils.js index 756168900..65f2a25ef 100644 --- a/src/widgets/feed/utils.js +++ b/src/widgets/feed/utils.js @@ -1,3 +1,4 @@ +import sax from "sax"; import { xml2js } from "xml-js"; const asArray = (value) => (value === undefined || value === null ? [] : [].concat(value)); @@ -39,6 +40,27 @@ function findImage(item) { return image ? attrs(image).url || attrs(image).href : null; } +const isPixel = ({ width, height }) => ["0", "1"].includes(width) || ["0", "1"].includes(height); + +// first usable in item html using sax tokenizer +function findHtmlImage(htmls, baseUrl) { + for (const html of htmls) { + if (!html) continue; + let image = null; + const parser = sax.parser(false, { lowercase: true }); + parser.onopentag = ({ name, attributes }) => { + if (!image && name === "img" && !isPixel(attributes)) image = httpUrl(attributes.src, baseUrl); + }; + parser.onerror = () => { + parser.error = null; + parser.resume(); + }; + parser.write(html).close(); + if (image) return image; + } + return null; +} + function parseRssItem(item) { const guid = asArray(item.guid)[0]; const guidLink = attrs(guid).isPermaLink !== "false" ? getText(guid) : null; @@ -47,6 +69,7 @@ function parseRssItem(item) { link: getText(item.link) || guidLink, date: getText(item.pubDate) || getText(item["dc:date"]), image: findImage(item), + html: [getText(item["content:encoded"]), getText(item.description)], }; } @@ -58,6 +81,7 @@ function parseAtomEntry(entry) { link: attrs(link).href, date: getText(entry.published) || getText(entry.updated), image: findImage(entry), + html: [getText(entry.content), getText(entry.summary)], }; } @@ -74,7 +98,7 @@ export function parseFeed(xml, baseUrl) { title: item.title.trim(), link: httpUrl(item.link, baseUrl), date: parseDate(item.date), - image: httpUrl(item.image, baseUrl), + image: httpUrl(item.image, baseUrl) ?? findHtmlImage(item.html, baseUrl), })) .filter((item) => item.title); } diff --git a/src/widgets/feed/utils.test.js b/src/widgets/feed/utils.test.js index e6115d082..b83685f17 100644 --- a/src/widgets/feed/utils.test.js +++ b/src/widgets/feed/utils.test.js @@ -88,6 +88,48 @@ describe("widgets/feed/utils", () => { ]); }); + it("falls back to the first usable image in item html", () => { + const xml = ` + + Content image + Hi + ]]> + + + Description image + <img src="https://example.com/desc.png"> unclosed <b> + + + Media wins + + ]]> + + + No usable image + ]]> + + `; + + expect(parseFeed(xml, "https://example.com/feed").map((item) => item.image)).toEqual([ + "https://example.com/big.jpg?w=925&h=925", + "https://example.com/desc.png", + "https://example.com/thumb.jpg", + null, + ]); + }); + + it("finds images in atom html content and summary", () => { + const xml = ` + Content<img src="https://example.com/c.jpg"> + Summary<img src="https://example.com/s.jpg"> + `; + + expect(parseFeed(xml).map((item) => item.image)).toEqual([ + "https://example.com/c.jpg", + "https://example.com/s.jpg", + ]); + }); + it("handles single-item and empty feeds", () => { const single = "Only"; expect(parseFeed(single)).toEqual([{ title: "Only", link: null, date: null, image: null }]);