v1.8.0: titel-dedup over feeds (zelfde artikel, andere url)

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*.pyc

base-config.yaml

@@ -23,6 +23,10 @@ whatsapp_session: EdGpt
 whatsapp_chat_id: 120363429643087130@g.us
 # Exclude paywalled articles (Telegraaf/NHD markeren dit met <premium>true</premium>)
 exclude_paywalled: true
+# Titel-dedup over feeds: skip een artikel waarvan de genormaliseerde titel al is gepost
+# binnen X dagen, ook als de feed/URL anders is (zelfde uitgever, meerdere sites).
+# 0 = uit.
+title_dedup_days: 30
 # OV-matchfilter: VAST in code (gewogen families), niet via config te wijzigen.
 # Doorlaten als 2×(titelscore) + 1×(samenvattingscore) >= 3. Zie de families in
 # buzzrss/bot.py (MATCH_FILTER_FAMILIES). `!rss set-filter` en `!rss filter`

buzzrss/bot.py

@@ -16,7 +16,7 @@
 from __future__ import annotations
 
 from typing import Any
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from string import Template
 from time import mktime, monotonic, time
 import asyncio
@@ -43,7 +43,7 @@ from mautrix.types import (
 from mautrix.util.async_db import UpgradeTable
 from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper
 
-from .db import DBManager, Entry, Feed, Subscription
+from .db import DBManager, Entry, Feed, Subscription, normalize_title
 from .migrations import upgrade_table
 
 rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STATE)
@@ -96,6 +96,7 @@ class Config(BaseProxyConfig):
         helper.copy("command_prefix")
         helper.copy("notification_template")
         helper.copy("exclude_paywalled")
+        helper.copy("title_dedup_days")
         helper.copy("run_log_file")
         helper.copy("whatsapp_api")
         helper.copy("whatsapp_session")
@@ -332,10 +333,15 @@ class RSSBot(Plugin):
     async def _poll_once(self) -> None:
         subs = await self.dbm.get_feeds()
         if not subs:
-            self._log_run_summary(0, 0, 0, 0, 0, 0, 0.0)
+            self._log_run_summary(0, 0, 0, 0, 0, 0, 0, 0.0)
             return
         now = int(time())
         start = monotonic()
+        title_dedup_days = int(self.config.get("title_dedup_days", 30))
+        posted_titles: set[str] = set()
+        if title_dedup_days > 0:
+            cutoff = datetime.now(timezone.utc) - timedelta(days=title_dedup_days)
+            posted_titles = await self.dbm.get_recent_posted_titles(cutoff)
         tasks = [self.try_parse_feed(feed=feed) for feed in subs if feed.next_retry < now]
         feed: Feed
         entries: list[Entry] | None
@@ -343,6 +349,7 @@ class RSSBot(Plugin):
         evaluated_total = 0
         placed_total = 0
         filtered_total = 0
+        deduped_total = 0
         failed_total = 0
         unchanged_total = 0
         for res in asyncio.as_completed(tasks):
@@ -383,18 +390,36 @@ class RSSBot(Plugin):
             new_entry_list.sort(key=lambda entry: (entry.date, entry.id))
             await self.dbm.add_entries(new_entry_list)
             for entry in new_entry_list:
+                norm = normalize_title(entry.title)
+                if title_dedup_days > 0 and norm and norm in posted_titles:
+                    deduped_total += 1
+                    self.log.info(
+                        f"{entry.id} of feed {feed.id} title already posted ({norm[:60]}), skipping"
+                    )
+                    continue
                 placed, filtered, failed = await self._broadcast(
                     feed, entry, feed.subscriptions
                 )
                 placed_total += placed
                 filtered_total += filtered
                 failed_total += failed
+                if placed and norm:
+                    # In-memory set: vangt ook duplicaten binnen dezelfde poll-run
+                    # (twee sites van dezelfde uitgever in één run).
+                    posted_titles.add(norm)
+                    try:
+                        await self.dbm.mark_entry_posted(feed.id, entry.id)
+                    except Exception:
+                        self.log.warning(
+                            f"Failed to mark {entry.id} of feed {feed.id} as posted"
+                        )
         duration = monotonic() - start
         self._log_run_summary(
             len(tasks),
             evaluated_total,
             placed_total,
             filtered_total,
+            deduped_total,
             failed_total,
             unchanged_total,
             duration,
@@ -406,13 +431,14 @@ class RSSBot(Plugin):
         evaluated: int,
         selected: int,
         filtered: int,
+        deduped: int,
         failed: int,
         unchanged: int,
         duration: float,
     ) -> None:
         line = (
             f"run complete feeds={feeds} evaluated={evaluated} selected={selected} "
-            f"filtered={filtered} failed={failed} unchanged={unchanged} "
+            f"filtered={filtered} deduped={deduped} failed={failed} unchanged={unchanged} "
             f"duration={duration:.2f}s"
         )
         self.log.info(f"Finished polling {feeds} feeds in {duration:.2f} seconds: {line}")

buzzrss/db.py

@@ -15,6 +15,8 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
+import html
+import re
 from datetime import datetime, timezone
 from string import Template
 
@@ -85,6 +87,19 @@ date_fmt = "%Y-%m-%d %H:%M:%S"
 date_fmt_microseconds = "%Y-%m-%d %H:%M:%S.%f"
 
 
+def normalize_title(title: str) -> str:
+    """Genormaliseerde titel-sleutel voor cross-feed duplicate-detectie.
+
+    Stript HTML, lowercaset, vouwt whitespace samen en verwijdert rand-interpunctie,
+    zodat hetzelfde artikel dat door één uitgever onder een andere URL/feed wordt
+    gepubliceerd (vrijwel) identieke titels matcht. Bewust exacte matching:
+    terugkerende koppen ("treinstoring", "staking") met écht andere titels
+    mogen niet als duplicaat worden weggefilterd."""
+    text = html.unescape(re.sub(r"<[^>]+>", "", title or ""))
+    text = re.sub(r"\s+", " ", text)
+    return text.strip(" \t\n.,;:!?…-–—").lower()
+
+
 @dataclass
 class Entry:
     feed_id: int
@@ -151,6 +166,21 @@ class DBManager:
         for row in existing:
             entries.pop(row["id"], None)
 
+    async def get_recent_posted_titles(self, since: datetime) -> set[str]:
+        """Genormaliseerde titels van entries die sinds `since` daadwerkelijk zijn
+        gebroadcast (posted=true). Dit is de bron voor cross-feed titel-dedup."""
+        rows = await self.db.fetch(
+            "SELECT title FROM entry WHERE posted AND date >= $1", since
+        )
+        return {normalize_title(row["title"]) for row in rows if row["title"]}
+
+    async def mark_entry_posted(self, feed_id: int, entry_id: str) -> None:
+        await self.db.execute(
+            "UPDATE entry SET posted = true WHERE feed_id = $1 AND id = $2",
+            feed_id,
+            entry_id,
+        )
+
     async def add_entries(self, entries: list[Entry], override_feed_id: int | None = None) -> None:
         if not entries:
             return

buzzrss/migrations.py

@@ -117,3 +117,13 @@ async def upgrade_v10(conn: Connection, scheme: Scheme) -> None:
         f"NOT NULL DEFAULT {default}"
         f")"
     )
+
+
+@upgrade_table.register(description="Add posted flag to entries for cross-feed title dedup")
+async def upgrade_v11(conn: Connection, scheme: Scheme) -> None:
+    # posted = deze entry is daadwerkelijk gebroadcast (Matrix en/of WhatsApp).
+    # Nieuwe entries die alleen 'gezien' zijn (bv. feed-historis bij subscribe) en
+    # filter-afgewezen entries blijven false en blokkeren geen latere titel-match.
+    await conn.execute("ALTER TABLE entry ADD COLUMN posted BOOLEAN NOT NULL DEFAULT false")
+    # Venster-query voor de titel-dedup: recent gepostede titels ophalen.
+    await conn.execute("CREATE INDEX entry_date_idx ON entry (date)")

maubot.yaml

@@ -1,5 +1,5 @@
 id: nl.eddydevink.rss
-version: 1.7.0
+version: 1.8.0
 modules:
 - buzzrss
 main_class: RSSBot