diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..7a60b85
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+__pycache__/
+*.pyc
diff --git a/base-config.yaml b/base-config.yaml
index 9f4b52b..a45943d 100644
--- a/base-config.yaml
+++ b/base-config.yaml
@@ -23,6 +23,10 @@ whatsapp_session: EdGpt
whatsapp_chat_id: 120363429643087130@g.us
# Exclude paywalled articles (Telegraaf/NHD markeren dit met true)
exclude_paywalled: true
+# Titel-dedup over feeds: skip een artikel waarvan de genormaliseerde titel al is gepost
+# binnen X dagen, ook als de feed/URL anders is (zelfde uitgever, meerdere sites).
+# 0 = uit.
+title_dedup_days: 30
# OV-matchfilter: VAST in code (gewogen families), niet via config te wijzigen.
# Doorlaten als 2×(titelscore) + 1×(samenvattingscore) >= 3. Zie de families in
# buzzrss/bot.py (MATCH_FILTER_FAMILIES). `!rss set-filter` en `!rss filter`
diff --git a/buzzrss/__pycache__/bot.cpython-314.pyc b/buzzrss/__pycache__/bot.cpython-314.pyc
deleted file mode 100644
index 19c4c36..0000000
Binary files a/buzzrss/__pycache__/bot.cpython-314.pyc and /dev/null differ
diff --git a/buzzrss/bot.py b/buzzrss/bot.py
index dd38b0b..248ebac 100644
--- a/buzzrss/bot.py
+++ b/buzzrss/bot.py
@@ -16,7 +16,7 @@
from __future__ import annotations
from typing import Any
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
from string import Template
from time import mktime, monotonic, time
import asyncio
@@ -43,7 +43,7 @@ from mautrix.types import (
from mautrix.util.async_db import UpgradeTable
from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper
-from .db import DBManager, Entry, Feed, Subscription
+from .db import DBManager, Entry, Feed, Subscription, normalize_title
from .migrations import upgrade_table
rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STATE)
@@ -96,6 +96,7 @@ class Config(BaseProxyConfig):
helper.copy("command_prefix")
helper.copy("notification_template")
helper.copy("exclude_paywalled")
+ helper.copy("title_dedup_days")
helper.copy("run_log_file")
helper.copy("whatsapp_api")
helper.copy("whatsapp_session")
@@ -332,10 +333,15 @@ class RSSBot(Plugin):
async def _poll_once(self) -> None:
subs = await self.dbm.get_feeds()
if not subs:
- self._log_run_summary(0, 0, 0, 0, 0, 0, 0.0)
+ self._log_run_summary(0, 0, 0, 0, 0, 0, 0, 0.0)
return
now = int(time())
start = monotonic()
+ title_dedup_days = int(self.config.get("title_dedup_days", 30))
+ posted_titles: set[str] = set()
+ if title_dedup_days > 0:
+ cutoff = datetime.now(timezone.utc) - timedelta(days=title_dedup_days)
+ posted_titles = await self.dbm.get_recent_posted_titles(cutoff)
tasks = [self.try_parse_feed(feed=feed) for feed in subs if feed.next_retry < now]
feed: Feed
entries: list[Entry] | None
@@ -343,6 +349,7 @@ class RSSBot(Plugin):
evaluated_total = 0
placed_total = 0
filtered_total = 0
+ deduped_total = 0
failed_total = 0
unchanged_total = 0
for res in asyncio.as_completed(tasks):
@@ -383,18 +390,36 @@ class RSSBot(Plugin):
new_entry_list.sort(key=lambda entry: (entry.date, entry.id))
await self.dbm.add_entries(new_entry_list)
for entry in new_entry_list:
+ norm = normalize_title(entry.title)
+ if title_dedup_days > 0 and norm and norm in posted_titles:
+ deduped_total += 1
+ self.log.info(
+ f"{entry.id} of feed {feed.id} title already posted ({norm[:60]}), skipping"
+ )
+ continue
placed, filtered, failed = await self._broadcast(
feed, entry, feed.subscriptions
)
placed_total += placed
filtered_total += filtered
failed_total += failed
+ if placed and norm:
+ # In-memory set: vangt ook duplicaten binnen dezelfde poll-run
+ # (twee sites van dezelfde uitgever in één run).
+ posted_titles.add(norm)
+ try:
+ await self.dbm.mark_entry_posted(feed.id, entry.id)
+ except Exception:
+ self.log.warning(
+ f"Failed to mark {entry.id} of feed {feed.id} as posted"
+ )
duration = monotonic() - start
self._log_run_summary(
len(tasks),
evaluated_total,
placed_total,
filtered_total,
+ deduped_total,
failed_total,
unchanged_total,
duration,
@@ -406,13 +431,14 @@ class RSSBot(Plugin):
evaluated: int,
selected: int,
filtered: int,
+ deduped: int,
failed: int,
unchanged: int,
duration: float,
) -> None:
line = (
f"run complete feeds={feeds} evaluated={evaluated} selected={selected} "
- f"filtered={filtered} failed={failed} unchanged={unchanged} "
+ f"filtered={filtered} deduped={deduped} failed={failed} unchanged={unchanged} "
f"duration={duration:.2f}s"
)
self.log.info(f"Finished polling {feeds} feeds in {duration:.2f} seconds: {line}")
diff --git a/buzzrss/db.py b/buzzrss/db.py
index 1c10433..3e81e4d 100644
--- a/buzzrss/db.py
+++ b/buzzrss/db.py
@@ -15,6 +15,8 @@
# along with this program. If not, see .
from __future__ import annotations
+import html
+import re
from datetime import datetime, timezone
from string import Template
@@ -85,6 +87,19 @@ date_fmt = "%Y-%m-%d %H:%M:%S"
date_fmt_microseconds = "%Y-%m-%d %H:%M:%S.%f"
+def normalize_title(title: str) -> str:
+ """Genormaliseerde titel-sleutel voor cross-feed duplicate-detectie.
+
+ Stript HTML, lowercaset, vouwt whitespace samen en verwijdert rand-interpunctie,
+ zodat hetzelfde artikel dat door één uitgever onder een andere URL/feed wordt
+ gepubliceerd (vrijwel) identieke titels matcht. Bewust exacte matching:
+ terugkerende koppen ("treinstoring", "staking") met écht andere titels
+ mogen niet als duplicaat worden weggefilterd."""
+ text = html.unescape(re.sub(r"<[^>]+>", "", title or ""))
+ text = re.sub(r"\s+", " ", text)
+ return text.strip(" \t\n.,;:!?…-–—").lower()
+
+
@dataclass
class Entry:
feed_id: int
@@ -151,6 +166,21 @@ class DBManager:
for row in existing:
entries.pop(row["id"], None)
+ async def get_recent_posted_titles(self, since: datetime) -> set[str]:
+ """Genormaliseerde titels van entries die sinds `since` daadwerkelijk zijn
+ gebroadcast (posted=true). Dit is de bron voor cross-feed titel-dedup."""
+ rows = await self.db.fetch(
+ "SELECT title FROM entry WHERE posted AND date >= $1", since
+ )
+ return {normalize_title(row["title"]) for row in rows if row["title"]}
+
+ async def mark_entry_posted(self, feed_id: int, entry_id: str) -> None:
+ await self.db.execute(
+ "UPDATE entry SET posted = true WHERE feed_id = $1 AND id = $2",
+ feed_id,
+ entry_id,
+ )
+
async def add_entries(self, entries: list[Entry], override_feed_id: int | None = None) -> None:
if not entries:
return
diff --git a/buzzrss/migrations.py b/buzzrss/migrations.py
index a31a8da..c7e0077 100644
--- a/buzzrss/migrations.py
+++ b/buzzrss/migrations.py
@@ -117,3 +117,13 @@ async def upgrade_v10(conn: Connection, scheme: Scheme) -> None:
f"NOT NULL DEFAULT {default}"
f")"
)
+
+
+@upgrade_table.register(description="Add posted flag to entries for cross-feed title dedup")
+async def upgrade_v11(conn: Connection, scheme: Scheme) -> None:
+ # posted = deze entry is daadwerkelijk gebroadcast (Matrix en/of WhatsApp).
+ # Nieuwe entries die alleen 'gezien' zijn (bv. feed-historis bij subscribe) en
+ # filter-afgewezen entries blijven false en blokkeren geen latere titel-match.
+ await conn.execute("ALTER TABLE entry ADD COLUMN posted BOOLEAN NOT NULL DEFAULT false")
+ # Venster-query voor de titel-dedup: recent gepostede titels ophalen.
+ await conn.execute("CREATE INDEX entry_date_idx ON entry (date)")
diff --git a/maubot.yaml b/maubot.yaml
index 9ebea5d..f5fa9ce 100644
--- a/maubot.yaml
+++ b/maubot.yaml
@@ -1,5 +1,5 @@
id: nl.eddydevink.rss
-version: 1.7.0
+version: 1.8.0
modules:
- buzzrss
main_class: RSSBot