diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7a60b85 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/base-config.yaml b/base-config.yaml index 9f4b52b..a45943d 100644 --- a/base-config.yaml +++ b/base-config.yaml @@ -23,6 +23,10 @@ whatsapp_session: EdGpt whatsapp_chat_id: 120363429643087130@g.us # Exclude paywalled articles (Telegraaf/NHD markeren dit met true) exclude_paywalled: true +# Titel-dedup over feeds: skip een artikel waarvan de genormaliseerde titel al is gepost +# binnen X dagen, ook als de feed/URL anders is (zelfde uitgever, meerdere sites). +# 0 = uit. +title_dedup_days: 30 # OV-matchfilter: VAST in code (gewogen families), niet via config te wijzigen. # Doorlaten als 2×(titelscore) + 1×(samenvattingscore) >= 3. Zie de families in # buzzrss/bot.py (MATCH_FILTER_FAMILIES). `!rss set-filter` en `!rss filter` diff --git a/buzzrss/__pycache__/bot.cpython-314.pyc b/buzzrss/__pycache__/bot.cpython-314.pyc deleted file mode 100644 index 19c4c36..0000000 Binary files a/buzzrss/__pycache__/bot.cpython-314.pyc and /dev/null differ diff --git a/buzzrss/bot.py b/buzzrss/bot.py index dd38b0b..248ebac 100644 --- a/buzzrss/bot.py +++ b/buzzrss/bot.py @@ -16,7 +16,7 @@ from __future__ import annotations from typing import Any -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from string import Template from time import mktime, monotonic, time import asyncio @@ -43,7 +43,7 @@ from mautrix.types import ( from mautrix.util.async_db import UpgradeTable from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper -from .db import DBManager, Entry, Feed, Subscription +from .db import DBManager, Entry, Feed, Subscription, normalize_title from .migrations import upgrade_table rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STATE) @@ -96,6 +96,7 @@ class Config(BaseProxyConfig): helper.copy("command_prefix") helper.copy("notification_template") helper.copy("exclude_paywalled") + helper.copy("title_dedup_days") helper.copy("run_log_file") helper.copy("whatsapp_api") helper.copy("whatsapp_session") @@ -332,10 +333,15 @@ class RSSBot(Plugin): async def _poll_once(self) -> None: subs = await self.dbm.get_feeds() if not subs: - self._log_run_summary(0, 0, 0, 0, 0, 0, 0.0) + self._log_run_summary(0, 0, 0, 0, 0, 0, 0, 0.0) return now = int(time()) start = monotonic() + title_dedup_days = int(self.config.get("title_dedup_days", 30)) + posted_titles: set[str] = set() + if title_dedup_days > 0: + cutoff = datetime.now(timezone.utc) - timedelta(days=title_dedup_days) + posted_titles = await self.dbm.get_recent_posted_titles(cutoff) tasks = [self.try_parse_feed(feed=feed) for feed in subs if feed.next_retry < now] feed: Feed entries: list[Entry] | None @@ -343,6 +349,7 @@ class RSSBot(Plugin): evaluated_total = 0 placed_total = 0 filtered_total = 0 + deduped_total = 0 failed_total = 0 unchanged_total = 0 for res in asyncio.as_completed(tasks): @@ -383,18 +390,36 @@ class RSSBot(Plugin): new_entry_list.sort(key=lambda entry: (entry.date, entry.id)) await self.dbm.add_entries(new_entry_list) for entry in new_entry_list: + norm = normalize_title(entry.title) + if title_dedup_days > 0 and norm and norm in posted_titles: + deduped_total += 1 + self.log.info( + f"{entry.id} of feed {feed.id} title already posted ({norm[:60]}), skipping" + ) + continue placed, filtered, failed = await self._broadcast( feed, entry, feed.subscriptions ) placed_total += placed filtered_total += filtered failed_total += failed + if placed and norm: + # In-memory set: vangt ook duplicaten binnen dezelfde poll-run + # (twee sites van dezelfde uitgever in één run). + posted_titles.add(norm) + try: + await self.dbm.mark_entry_posted(feed.id, entry.id) + except Exception: + self.log.warning( + f"Failed to mark {entry.id} of feed {feed.id} as posted" + ) duration = monotonic() - start self._log_run_summary( len(tasks), evaluated_total, placed_total, filtered_total, + deduped_total, failed_total, unchanged_total, duration, @@ -406,13 +431,14 @@ class RSSBot(Plugin): evaluated: int, selected: int, filtered: int, + deduped: int, failed: int, unchanged: int, duration: float, ) -> None: line = ( f"run complete feeds={feeds} evaluated={evaluated} selected={selected} " - f"filtered={filtered} failed={failed} unchanged={unchanged} " + f"filtered={filtered} deduped={deduped} failed={failed} unchanged={unchanged} " f"duration={duration:.2f}s" ) self.log.info(f"Finished polling {feeds} feeds in {duration:.2f} seconds: {line}") diff --git a/buzzrss/db.py b/buzzrss/db.py index 1c10433..3e81e4d 100644 --- a/buzzrss/db.py +++ b/buzzrss/db.py @@ -15,6 +15,8 @@ # along with this program. If not, see . from __future__ import annotations +import html +import re from datetime import datetime, timezone from string import Template @@ -85,6 +87,19 @@ date_fmt = "%Y-%m-%d %H:%M:%S" date_fmt_microseconds = "%Y-%m-%d %H:%M:%S.%f" +def normalize_title(title: str) -> str: + """Genormaliseerde titel-sleutel voor cross-feed duplicate-detectie. + + Stript HTML, lowercaset, vouwt whitespace samen en verwijdert rand-interpunctie, + zodat hetzelfde artikel dat door één uitgever onder een andere URL/feed wordt + gepubliceerd (vrijwel) identieke titels matcht. Bewust exacte matching: + terugkerende koppen ("treinstoring", "staking") met écht andere titels + mogen niet als duplicaat worden weggefilterd.""" + text = html.unescape(re.sub(r"<[^>]+>", "", title or "")) + text = re.sub(r"\s+", " ", text) + return text.strip(" \t\n.,;:!?…-–—").lower() + + @dataclass class Entry: feed_id: int @@ -151,6 +166,21 @@ class DBManager: for row in existing: entries.pop(row["id"], None) + async def get_recent_posted_titles(self, since: datetime) -> set[str]: + """Genormaliseerde titels van entries die sinds `since` daadwerkelijk zijn + gebroadcast (posted=true). Dit is de bron voor cross-feed titel-dedup.""" + rows = await self.db.fetch( + "SELECT title FROM entry WHERE posted AND date >= $1", since + ) + return {normalize_title(row["title"]) for row in rows if row["title"]} + + async def mark_entry_posted(self, feed_id: int, entry_id: str) -> None: + await self.db.execute( + "UPDATE entry SET posted = true WHERE feed_id = $1 AND id = $2", + feed_id, + entry_id, + ) + async def add_entries(self, entries: list[Entry], override_feed_id: int | None = None) -> None: if not entries: return diff --git a/buzzrss/migrations.py b/buzzrss/migrations.py index a31a8da..c7e0077 100644 --- a/buzzrss/migrations.py +++ b/buzzrss/migrations.py @@ -117,3 +117,13 @@ async def upgrade_v10(conn: Connection, scheme: Scheme) -> None: f"NOT NULL DEFAULT {default}" f")" ) + + +@upgrade_table.register(description="Add posted flag to entries for cross-feed title dedup") +async def upgrade_v11(conn: Connection, scheme: Scheme) -> None: + # posted = deze entry is daadwerkelijk gebroadcast (Matrix en/of WhatsApp). + # Nieuwe entries die alleen 'gezien' zijn (bv. feed-historis bij subscribe) en + # filter-afgewezen entries blijven false en blokkeren geen latere titel-match. + await conn.execute("ALTER TABLE entry ADD COLUMN posted BOOLEAN NOT NULL DEFAULT false") + # Venster-query voor de titel-dedup: recent gepostede titels ophalen. + await conn.execute("CREATE INDEX entry_date_idx ON entry (date)") diff --git a/maubot.yaml b/maubot.yaml index 9ebea5d..f5fa9ce 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,5 +1,5 @@ id: nl.eddydevink.rss -version: 1.7.0 +version: 1.8.0 modules: - buzzrss main_class: RSSBot