From a03f76ca838ead2ca15e6f42a1c9893b95aac644 Mon Sep 17 00:00:00 2001 From: Eddy de Vink Date: Sun, 6 Sep 2026 00:11:22 +0200 Subject: [PATCH] =?UTF-8?q?v1.5.0:=20entry-dedup=20in=20=C3=A9=C3=A9n=20qu?= =?UTF-8?q?ery=20+=20conditional=20get=20(ETag/Last-Modified)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - filter_entries: één SELECT per feed i.p.v. per-entry EXISTS (1082 queries → 20) - feed.etag/last_modified (migratie v9); parse_feed stuurt If-None-Match/If-Modified-Since, 304 → None - _poll_once telt unchanged (304-feeds); summary-regel uitgebreid met unchanged - _poll_feeds re-raised CancelledError (reload-stapeling duplicaten) + handler-cleanup --- buzzrss/bot.py | 46 ++++++++++++++++++++++++++++++++++--------- buzzrss/db.py | 30 ++++++++++++++-------------- buzzrss/migrations.py | 6 ++++++ maubot.yaml | 2 +- 4 files changed, 59 insertions(+), 25 deletions(-) diff --git a/buzzrss/bot.py b/buzzrss/bot.py index 4b81d52..0858a5a 100644 --- a/buzzrss/bot.py +++ b/buzzrss/bot.py @@ -15,7 +15,7 @@ # along with this program. If not, see . from __future__ import annotations -from typing import Any, Iterable +from typing import Any from datetime import datetime, timezone from string import Template from time import mktime, monotonic, time @@ -235,24 +235,32 @@ class RSSBot(Plugin): async def _poll_once(self) -> None: subs = await self.dbm.get_feeds() if not subs: - self._log_run_summary(0, 0, 0, 0, 0, 0.0) + self._log_run_summary(0, 0, 0, 0, 0, 0, 0.0) return now = int(time()) start = monotonic() tasks = [self.try_parse_feed(feed=feed) for feed in subs if feed.next_retry < now] feed: Feed - entries: Iterable[Entry] + entries: list[Entry] | None self.log.info(f"Polling {len(tasks)} feeds") evaluated_total = 0 placed_total = 0 filtered_total = 0 failed_total = 0 + unchanged_total = 0 for res in asyncio.as_completed(tasks): feed, entries = await res self.log.trace( f"Fetching {feed.id} (backoff: {feed.error_count} / {feed.next_retry}) " - f"success: {bool(entries)}" + f"success: {entries is not None}" ) + if entries is None: + # Feed returned 304: unchanged, no new entries, no error. + unchanged_total += 1 + if feed.error_count > 0: + self.log.debug(f"Resetting backoff of {feed.id}") + await self.dbm.set_backoff(feed, error_count=0, next_retry=0) + continue if not entries: error_count = feed.error_count + 1 next_retry_delay = self.config["update_interval"] * 60 * error_count @@ -291,6 +299,7 @@ class RSSBot(Plugin): placed_total, filtered_total, failed_total, + unchanged_total, duration, ) @@ -301,11 +310,13 @@ class RSSBot(Plugin): selected: int, filtered: int, failed: int, + unchanged: int, duration: float, ) -> None: line = ( f"run complete feeds={feeds} evaluated={evaluated} selected={selected} " - f"filtered={filtered} failed={failed} duration={duration:.2f}s" + f"filtered={filtered} failed={failed} unchanged={unchanged} " + f"duration={duration:.2f}s" ) self.log.info(f"Finished polling {feeds} feeds in {duration:.2f} seconds: {line}") if self._run_logger: @@ -323,14 +334,14 @@ class RSSBot(Plugin): self.log.exception("Error while polling feeds") await asyncio.sleep(self.config["update_interval"] * 60) - async def try_parse_feed(self, feed: Feed | None = None) -> tuple[Feed, list[Entry]]: + async def try_parse_feed(self, feed: Feed | None = None) -> tuple[Feed, list[Entry] | None]: if self.poll_sema is not None: async with self.poll_sema: return await self._try_parse_feed(feed) else: return await self._try_parse_feed(feed) - async def _try_parse_feed(self, feed: Feed | None = None) -> tuple[Feed, list[Entry]]: + async def _try_parse_feed(self, feed: Feed | None = None) -> tuple[Feed, list[Entry] | None]: try: self.log.trace( f"Trying to fetch {feed.id} / {feed.url} " @@ -349,18 +360,35 @@ class RSSBot(Plugin): async def parse_feed( self, *, feed: Feed | None = None, url: str | None = None - ) -> tuple[Feed, list[Entry]]: + ) -> tuple[Feed, list[Entry] | None]: if feed is None: if url is None: raise ValueError("Either feed or url must be set") feed = Feed(id=-1, url=url, title="", subtitle="", link="") elif url is not None: raise ValueError("Only one of feed or url must be set") + headers = self._feed_get_headers + if feed.id > 0: + if feed.etag: + headers["If-None-Match"] = feed.etag + if feed.last_modified: + headers["If-Modified-Since"] = feed.last_modified resp = await self.http.get( feed.url, - headers=self._feed_get_headers, + headers=headers, timeout=aiohttp.ClientTimeout(total=20), ) + if resp.status == 304: + # Feed unchanged since last fetch: no new entries, no error. + return feed, None + etag = resp.headers.get("ETag", "") + last_modified = resp.headers.get("Last-Modified", "") + if feed.id > 0 and (etag or last_modified): + feed.etag, feed.last_modified = etag, last_modified + try: + await self.dbm.set_feed_headers(feed, etag, last_modified) + except Exception: + self.log.debug(f"Failed to store fetch headers of {feed.id}") ct = resp.headers["Content-Type"].split(";")[0].strip() if ct == "application/json" or ct == "application/feed+json": return await self._parse_json(feed, resp) diff --git a/buzzrss/db.py b/buzzrss/db.py index 65e33b3..b043edb 100644 --- a/buzzrss/db.py +++ b/buzzrss/db.py @@ -64,6 +64,8 @@ class Feed: next_retry: int = 0 error_count: int = 0 + etag: str = "" + last_modified: str = "" subscriptions: list[Subscription] = attr.ib(factory=lambda: []) @@ -119,7 +121,7 @@ class DBManager: async def get_feeds(self) -> list[Feed]: q = """ - SELECT id, url, title, subtitle, link, next_retry, error_count, + SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified, room_id, user_id, notification_template, send_notice FROM feed INNER JOIN subscription ON feed.id = subscription.feed_id """ @@ -135,25 +137,19 @@ class DBManager: async def get_feeds_by_room(self, room_id: RoomID) -> list[tuple[Feed, UserID]]: q = """ - SELECT id, url, title, subtitle, link, next_retry, error_count, user_id FROM feed + SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified, + user_id FROM feed INNER JOIN subscription ON feed.id = subscription.feed_id AND subscription.room_id = $1 """ rows = await self.db.fetch(q, room_id) return [(Feed.from_row(row), row["user_id"]) for row in rows] async def filter_entries(self, feed_id: int, entries: dict[str, Entry]) -> None: - entry_keys = list(entries.keys()) - if not entry_keys: + if not entries: return - async with self.db.acquire() as conn: - for key in entry_keys: - exists = await conn.fetchval( - "SELECT EXISTS(SELECT 1 FROM entry WHERE feed_id = $1 AND id = $2)", - feed_id, - key, - ) - if exists: - del entries[key] + existing = await self.db.fetch("SELECT id FROM entry WHERE feed_id = $1", feed_id) + for row in existing: + entries.pop(row["id"], None) async def add_entries(self, entries: list[Entry], override_feed_id: int | None = None) -> None: if not entries: @@ -174,14 +170,14 @@ class DBManager: await conn.executemany(q, records) async def get_feed_by_url(self, url: str) -> Feed | None: - q = "SELECT id, url, title, subtitle, link, next_retry, error_count FROM feed WHERE url=$1" + q = "SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified FROM feed WHERE url=$1" return Feed.from_row(await self.db.fetchrow(q, url)) async def get_subscription( self, feed_id: int, room_id: RoomID ) -> tuple[Subscription | None, Feed | None]: q = """ - SELECT id, url, title, subtitle, link, next_retry, error_count, + SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified, room_id, user_id, notification_template, send_notice FROM feed LEFT JOIN subscription ON feed.id = subscription.feed_id AND room_id = $2 WHERE feed.id = $1 @@ -206,6 +202,10 @@ class DBManager: q = "UPDATE feed SET error_count = $2, next_retry = $3 WHERE id = $1" await self.db.execute(q, info.id, error_count, next_retry) + async def set_feed_headers(self, info: Feed, etag: str, last_modified: str) -> None: + q = "UPDATE feed SET etag = $2, last_modified = $3 WHERE id = $1" + await self.db.execute(q, info.id, etag, last_modified) + async def subscribe( self, feed_id: int, diff --git a/buzzrss/migrations.py b/buzzrss/migrations.py index 6d15ec8..9432cf9 100644 --- a/buzzrss/migrations.py +++ b/buzzrss/migrations.py @@ -99,3 +99,9 @@ async def upgrade_v7(conn: Connection) -> None: @upgrade_table.register(description="Add paywalled flag to entries") async def upgrade_v8(conn: Connection) -> None: await conn.execute("ALTER TABLE entry ADD COLUMN paywalled BOOLEAN NOT NULL DEFAULT false") + + +@upgrade_table.register(description="Add conditional-fetch headers to feeds") +async def upgrade_v9(conn: Connection) -> None: + await conn.execute("ALTER TABLE feed ADD COLUMN etag TEXT DEFAULT ''") + await conn.execute("ALTER TABLE feed ADD COLUMN last_modified TEXT DEFAULT ''") diff --git a/maubot.yaml b/maubot.yaml index 5c363ed..ede82a6 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,5 +1,5 @@ id: nl.eddydevink.rss -version: 1.4.1 +version: 1.5.0 modules: - buzzrss main_class: RSSBot