v1.7.0: gewogen OV-families-filter vervangt match_filter-regex

Filter is nu vast in code (MATCH_FILTER_FAMILIES + MATCH_FILTER_THRESHOLD):
doorlaten als 2x(titelfamiliescore) + 1x(samenvattingscore) >= 3.
Sterk(3): vervoerders, spoorinfra, station, voertuig-samenstellingen, trein.
Medium(2): tram, metro, dienst, vervoersbewijs/staking/pont.
Zwak(1): losse bus/kaartje/rails - moet combineren met ander OV-woord.
Weert 'VW-bus'/'schoolbus', woningbrand-leads met 'bussen rijden om' en
figuratief 'op de rails', zonder bushalte/overweg/NS/trein-berichten te raken.
Gewogen per familie ipv per cluster dicht ook het 'bus x3 in lead'-gat.
!rss set-filter en !rss filter zijn read-only geworden (tonen het vaste filter);
config-keys match_filter/allow_filter verwijderd uit do_update.
V1.7.0 gedeployed op nextcloud (instance rss, enabled+started, run-log actief).

base-config.yaml

@@ -16,12 +16,17 @@ notification_template: "New post in $feed_title: [$title]($link)"
 # Absoluut pad in de container aanraden: /data is de persistente maubot-mount (host: /matrix/maubot/data).
 # Leeg = uitgeschakeld.
 run_log_file: /data/buzzrss-runs.log
+# Doorsturen naar WhatsApp (wwebjs-api op nextcloud :3001). Leeg whatsapp_api/whatsapp_chat_id = uit. Alleen de link wordt gepost (geen titel.
+
+whatsapp_api: http://192.168.5.252:3001
+whatsapp_session: EdGpt
+whatsapp_chat_id: 120363429643087130@g.us
 # Exclude paywalled articles (Telegraaf/NHD markeren dit met <premium>true</premium>)
 exclude_paywalled: true
-# Global match filter (whitelist): applies to ALL feeds in this instance.
+# OV-matchfilter: VAST in code (gewogen families), niet via config te wijzigen.
-# A post is ONLY sent when the regex matches the title OR the (HTML-stripped) summary.
+# Doorlaten als 2×(titelscore) + 1×(samenvattingscore) >= 3. Zie de families in
-# Set it live with `!rss set-filter <regex>`. Empty = send everything.
+# buzzrss/bot.py (MATCH_FILTER_FAMILIES). `!rss set-filter` en `!rss filter`
-match_filter: "(?i)\\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\\b|\\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking|sen)?\\b|\\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\\b|\\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\\b|\\b(?:openbaar vervoer|dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\\b|\\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\\b|\\b(?:ov-staking|conducteursstaking)\\b"
+# tonen alleen de huidige (vaste) filter.
 # Users who can bypass room permission checks
 admins:
 - "@user:example.com"

buzzrss/bot.py

@@ -49,16 +49,42 @@ from .migrations import upgrade_table
 rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STATE)
 
 
-DEFAULT_MATCH_FILTER = (
+# Gewogen OV-trefwoord-families voor het match-filter. Elke tuple = (regex, gewicht).
-    r"(?i)"
+# Per familie wordt 1× geteld per tekst (niet per cluster/herhaling). Drempel (3):
-    r"\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\b"
+#   doorlaten als 2×(titelscore) + 1×(samenvattingscore) >= MATCH_FILTER_THRESHOLD
-    r"|\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking|sen)?\b"
+# Gewicht 3 (sterk, onmiskenbaar OV): één woord in titel óf samenvatting volstaat.
-    r"|\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\b"
+# Gewicht 2 (medium): één woord in de titel volstaat; in de samenvatting moet er
-    r"|\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\b"
+#   een tweede OV-woord bijkomen.
-    r"|\b(?:openbaar vervoer|dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\b"
+# Gewicht 1 (zwak/dubbelzinnig: los "bus", "kaartje", "rails"): moet met een ander
-    r"|\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\b"
+#   OV-woord combineren. Weert zo "VW-bus"/"schoolbus", "weer op de rails" en
-    r"|\b(?:ov-staking|conducteursstaking)\b"
+#   112-berichten met "bussen rijden om" in de lead, zonder eenduidige
-)
+#   bushalte/overweg/NS/trein-berichten te raken.
+MATCH_FILTER_FAMILIES: list[tuple[str, int]] = [
+    # sterk (3)
+    (r"\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\b", 3),  # vervoerders
+    (r"\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\b", 3),  # spoorinfra
+    (r"\bstation\w*\b", 3),  # station(s)
+    (r"\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking)\b", 3),  # voertuig-samenstelling
+    (r"\btrein\w*\b", 3),  # trein(verkeer/staking/ongeluk/...)
+    # medium (2)
+    (r"\btram\b", 2),
+    (r"\bmetro\b", 2),
+    (r"\b(?:openbaar vervoer|dienstregeling|inchecken|uitchecken|lightrail|pendelbus|shuttlebus)\b", 2),
+    (r"\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\b", 2),
+    (r"\b(?:ov-staking|conducteursstaking)\b", 2),
+    (r"\b(?:pont|ponten)\b", 2),  # waterbus/veerdienst
+    # zwak (1) — dubbelzinnig, moet combineren
+    (r"\bbus(?:sen)?\b", 1),
+    (r"\bkaart(?:je|jes)\b", 1),
+    (r"\brails\b", 1),
+]
+
+# Drempel voor doorlaten (score = 2×titel + 1×samenvatting).
+MATCH_FILTER_THRESHOLD = 3
+
+_compiled_match_families: list[tuple[re.Pattern, int]] = [
+    (re.compile(pat, re.IGNORECASE), w) for pat, w in MATCH_FILTER_FAMILIES
+]
 
 
 class Config(BaseProxyConfig):
@@ -69,10 +95,11 @@ class Config(BaseProxyConfig):
         helper.copy("spam_sleep")
         helper.copy("command_prefix")
         helper.copy("notification_template")
-        helper.copy("allow_filter")
         helper.copy("exclude_paywalled")
-        helper.copy("match_filter")
         helper.copy("run_log_file")
+        helper.copy("whatsapp_api")
+        helper.copy("whatsapp_session")
+        helper.copy("whatsapp_chat_id")
         helper.copy("admins")
 
 
@@ -162,46 +189,41 @@ class RSSBot(Plugin):
             self.log.exception("Fatal error while polling feeds")
 
     @staticmethod
-    def _matches_in(text: str, match_filter: str) -> int:
+    def _fam_score(text: str) -> int:
-        """Aantal aaneengesloten match-clusters in text (0 als geen match).
+        """Som van gewichten over de OV-families die in text voorkomen (1× per familie)."""
-        Aaneengesloten matches (bv 'NS-trein', 'openbaar vervoer') tellen als 1."""
+        return sum(weight for family, weight in _compiled_match_families if family.search(text))
-        spans = sorted((m.start(), m.end()) for m in re.finditer(match_filter, text))
-        if not spans:
-            return 0
-        clusters, prev_end = 1, spans[0][1]
-        for s, e in spans[1:]:
-            if s - prev_end <= 1:
-                prev_end = max(prev_end, e)
-            else:
-                clusters += 1
-                prev_end = e
-        return clusters
 
     @staticmethod
-    def _filter_score(title: str, summary: str, match_filter: str) -> int:
+    def _filter_passes(title: str, summary: str) -> bool:
-        """Score = 2× hits in titel + 1× hits in summary."""
+        """Gewogen familiescore: doorlaten bij 2×(titelscore) + 1×(samenvattingscore) >= drempel.
-        return 2 * RSSBot._matches_in(title, match_filter) + RSSBot._matches_in(summary, match_filter)
+
+        Sterke families (gewicht 3) halen de drempel al met één woord in titel óf
+        samenvatting. Medium (2) in de titel ook; in de samenvatting moet een tweede
+        OV-woord bijkomen. Zwakke woorden (losse 'bus', 'kaartje', 'rails') moeten
+        combineren met een ander OV-woord. Weert oa 'VW-bus', 'schoolbus',
+        'bussen rijden om' in een woningbrand-lead en figuratief 'weer op de rails'.
+        Bewust doorgelaten: krant 'De Metro'/café 'De Tram' in de titel (medium)."""
+        return 2 * RSSBot._fam_score(title) + RSSBot._fam_score(summary) >= MATCH_FILTER_THRESHOLD
 
     @staticmethod
-    def _filter_passes(title: str, summary: str, match_filter: str) -> bool:
+    def _filter_summary() -> str:
-        """Regel C: doorlaten bij (titel-match >= 1) of (totaalscore >= 3).
+        """Mensvriendelijke weergave van het vaste gewogen match-filter."""
-
+        return (
-        Een OV-trefwoord in de titel is een sterke intentieverklaring (score 2
+            "**OV-matchfilter (vast in code, gewogen families):**\n"
-        volstaat). Treffers alleen in de samenvatting moeten dubbel bevestigd
+            "- Sterk (3): vervoerders (NS, Arriva, Qbuzz, ...), spoorinfra (perron, overweg, sein), "
-        worden (>= 3): zo houdt 'bussen rijden om' in de lead van een
+            "station(s), bushalte/buslijn/treinstel, trein\n"
-        woningbrand-bericht het artikel niet meer binnen."""
+            "- Medium (2): tram, metro, openbaar vervoer/dienstregeling/inchecken, "
-        if RSSBot._matches_in(title, match_filter) >= 1:
+            "ov-chipkaart/ovpay/reisproduct, ov-staking, pont\n"
-            return True
+            "- Zwak (1): losse 'bus', 'kaartje', 'rails' — moet combineren met een ander OV-woord\n\n"
-        return RSSBot._filter_score(title, summary, match_filter) >= 3
+            "Doorlaten: `2×(titel-score) + 1×(samenvatting-score) ≥ 3`"
+        )
 
     async def _send(
         self, feed: Feed, entry: Entry, sub: Subscription, *, check_filter: bool = True
     ) -> EventID | None:
         if check_filter:
-            match_filter = self.config["match_filter"]
-            if match_filter:
             summary = html.unescape(re.sub(r"<[^>]+>", "", entry.summary))
-                if not self._filter_passes(entry.title, summary, match_filter):
+            if not self._filter_passes(entry.title, summary):
                 return None
 
         message = sub.notification_template.safe_substitute(
@@ -222,20 +244,76 @@ class RSSBot(Plugin):
             self.log.warning(f"Failed to send {entry.id} of {feed.id} to {sub.room_id}: {e}")
             return None
 
+    async def _send_whatsapp(self, feed: Feed, entry: Entry) -> bool:
+        """Post entry.link naar de WhatsApp-groep via wwebjs-api (met dedup-check). True = nieuw geplaatst."""
+        api = self.config.get("whatsapp_api", "")
+        chat_id = self.config.get("whatsapp_chat_id", "")
+        if not api or not chat_id:
+            self.log.debug("WhatsApp forwarding disabled")
+            return False
+        link = entry.link
+        if not link:
+            self.log.warning(f"No link for {entry.id}, skipping WhatsApp")
+            return False
+        if await self.dbm.is_whatsapp_posted(link):
+            self.log.info(f"{entry.id} already posted to WhatsApp before, skipping")
+            return False
+        session = self.config.get("whatsapp_session", "EdGpt")
+        base = api.rstrip("/")
+        # Dedup: sla over als de URL al in de doelgroep staat. Eerst de lokale
+        # tabel (eigen posts; daarna de globale search (vreemde posts), gefilterd
+        # op chat-id van het gevonden bericht.
+
+        try:
+            resp = await self.http.post(
+                f"{base}/client/searchMessages/{session}",
+                json={"query": link, "options": {"limit": 1}},
+                timeout=aiohttp.ClientTimeout(total=20),
+            )
+            data = await resp.json()
+            if data.get("success"):
+                for msg in data.get("messages", []):
+                    msg_data = msg.get("_data", {})
+                    if (
+                        msg_data.get("id", {}).get("remote") == chat_id
+                        and link in msg_data.get("body", "")
+                    ):
+                        self.log.info(f"{entry.id} already in WhatsApp group, skipping")
+                        return False
+        except Exception as e:
+            self.log.warning(f"WhatsApp dedup check failed for {entry.id}: {e}")
+            return False
+        try:
+            resp = await self.http.post(
+                f"{base}/client/sendMessage/{session}",
+                json={"chatId": chat_id, "content": link, "contentType": "string"},
+                timeout=aiohttp.ClientTimeout(total=20),
+            )
+            data = await resp.json()
+            ok = bool(data.get("success"))
+            if ok:
+                try:
+                    await self.dbm.mark_whatsapp_posted(link)
+                except Exception as e:
+                    self.log.warning(f"Failed to record WhatsApp post {entry.id}: {e}")
+            self.log.info(f"WhatsApp post {entry.id} to {chat_id}: {'ok' if ok else 'failed: ' + str(data)}")
+            return ok
+        except Exception as e:
+            self.log.warning(f"Failed to send {entry.id} to WhatsApp {chat_id}: {e}")
+            return False
+
     async def _broadcast(
         self, feed: Feed, entry: Entry, subscriptions: list[Subscription]
     ) -> tuple[int, int, int]:
         """Send entry to all subs. Returns (placed, filtered, failed).
 
-        placed   = 1 when at least one room received the message
+        placed   = 1 when the entry was posted to at least one channel (Matrix room or newly posted WhatsApp link)
         filtered = 1 when the global match filter rejected the entry
         failed   = number of subscriptions where sending failed
         """
         self.log.debug(f"Broadcasting {entry.id} of {feed.id}")
-        match_filter = self.config["match_filter"]
-        if match_filter:
         summary = html.unescape(re.sub(r"<[^>]+>", "", entry.summary))
-            if not self._filter_passes(entry.title, summary, match_filter):
+        if not self._filter_passes(entry.title, summary):
             return 0, 1, 0
         spam_sleep = self.config["spam_sleep"]
         if spam_sleep >= 0:
@@ -248,7 +326,8 @@ class RSSBot(Plugin):
                 *(self._send(feed, entry, sub, check_filter=False) for sub in subscriptions)
             )
         sent = sum(1 for result in results if result is not None)
-        return (1 if sent else 0, 0, len(results) - sent)
+        whatsapp_ok = await self._send_whatsapp(feed, entry)
+        return (1 if (sent or whatsapp_ok) else 0, 0, len(results) - sent)
 
     async def _poll_once(self) -> None:
         subs = await self.dbm.get_feeds()
@@ -583,56 +662,16 @@ class RSSBot(Plugin):
             await evt.reply(f"Subscribed to {feed_info}")
 
     @rss.subcommand(
-        "set-filter", aliases=("f",), help="Set the global match filter: !rss set-filter <regex> | !rss set-filter clear"
+        "set-filter", aliases=("f",), help="Show the fixed global OV match filter."
-    )
-    @command.argument("match_filter", "match filter", pass_raw=True, required=False)
-    async def set_filter(self, evt: MessageEvent, match_filter: str = "") -> None:
-        if evt.sender in self.config["admins"]:
-            pass
-        elif not self.config["allow_filter"]:
-            await evt.reply("This bot does not allow non-admin users to set the match filter")
-            return
-        elif not await self.can_manage(evt):
-            return
-        if not match_filter:
-            # Lege aanroep: nooit het filter wissen. Toon usage + huidige filter.
-            f = self.config["match_filter"]
-            if f:
-                await evt.reply(
-                    f"Usage: `!rss set-filter <regex>` or `!rss set-filter clear`\n\n"
-                    f"**Current filter:**\n```\n{f}\n```"
-                )
-            else:
-                await evt.reply(
-                    "Usage: `!rss set-filter <regex>` or `!rss set-filter clear`\n\n"
-                    "No filter is currently set (all posts are sent)"
-                )
-            return
-        if match_filter.strip().lower() in ("clear", "none", "off", "remove"):
-            self.config["match_filter"] = ""
-            self.config.save()
-            await evt.reply("Global filter removed (all posts will be sent)")
-            return
-        try:
-            re.compile(match_filter)
-        except re.error:
-            await evt.reply("Filter is not a valid regular expression")
-            return
-        self.config["match_filter"] = match_filter
-        self.config.save()
-        await evt.reply(
-            f"Global filter updated: only posts matching `{match_filter}` will be sent"
     )
+    async def set_filter(self, evt: MessageEvent) -> None:
+        await evt.reply(self._filter_summary())
 
     @rss.subcommand(
         "filter", aliases=("show-filter", "get-filter"), help="Show the current global match filter."
     )
     async def command_show_filter(self, evt: MessageEvent) -> None:
-        f = self.config["match_filter"]
+        await evt.reply(self._filter_summary())
-        if f:
-            await evt.reply(f"**Current global filter:**\n```\n{f}\n```")
-        else:
-            await evt.reply("No global filter is set (all posts are sent)")
 
     @rss.subcommand(
         "unsubscribe", aliases=("u", "unsub"), help="Unsubscribe this room from a feed."
@@ -723,15 +762,10 @@ class RSSBot(Plugin):
     )
     async def command_subscriptions(self, evt: MessageEvent) -> None:
         subscriptions = await self.dbm.get_feeds_by_room(evt.room_id)
-        match_filter = self.config["match_filter"]
         if len(subscriptions) == 0:
             await evt.reply("There are no RSS subscriptions in this room")
             return
-        filter_line = (
+        filter_line = self._filter_summary().replace("\n", "  \n")
-            f"**Global filter:** `{match_filter}`"
-            if match_filter
-            else "**Global filter:** none (all posts are sent)"
-        )
         await evt.reply(
             "**Subscriptions in this room:**\n\n"
             + "\n".join(

buzzrss/db.py

@@ -232,3 +232,10 @@ class DBManager:
     async def set_send_notice(self, feed_id: int, room_id: RoomID, send_notice: bool) -> None:
         q = "UPDATE subscription SET send_notice=$3 WHERE feed_id=$1 AND room_id=$2"
         await self.db.execute(q, feed_id, room_id, send_notice)
+
+    async def is_whatsapp_posted(self, link: str) -> bool:
+        row = await self.db.fetchrow("SELECT 1 FROM whatsapp_posted WHERE link = $1", link)
+        return row is not None
+
+    async def mark_whatsapp_posted(self, link: str) -> None:
+        await self.db.execute("INSERT INTO whatsapp_posted (link) VALUES ($1) ON CONFLICT DO NOTHING", link)

buzzrss/migrations.py

@@ -105,3 +105,15 @@ async def upgrade_v8(conn: Connection) -> None:
 async def upgrade_v9(conn: Connection) -> None:
     await conn.execute("ALTER TABLE feed ADD COLUMN etag TEXT DEFAULT ''")
     await conn.execute("ALTER TABLE feed ADD COLUMN last_modified TEXT DEFAULT ''")
+
+
+@upgrade_table.register(description="Add WhatsApp posted-link dedup table")
+async def upgrade_v10(conn: Connection, scheme: Scheme) -> None:
+    default = "now()" if scheme == Scheme.POSTGRES else "CURRENT_TIMESTAMP"
+    await conn.execute(
+        f"CREATE TABLE IF NOT EXISTS whatsapp_posted ("
+        f"link TEXT PRIMARY KEY, "
+        f"posted_at {'TIMESTAMP WITH TIME ZONE' if scheme == Scheme.POSTGRES else 'TIMESTAMP'} "
+        f"NOT NULL DEFAULT {default}"
+        f")"
+    )

maubot.yaml

@@ -1,5 +1,5 @@
 id: nl.eddydevink.rss
-version: 1.5.4
+version: 1.7.0
 modules:
 - buzzrss
 main_class: RSSBot