From d7ba8a06de4173631d4c7b094af34f12f91b28f7 Mon Sep 17 00:00:00 2001 From: Eddy de Vink Date: Tue, 8 Sep 2026 20:50:52 +0200 Subject: [PATCH] v1.7.0: gewogen OV-families-filter vervangt match_filter-regex Filter is nu vast in code (MATCH_FILTER_FAMILIES + MATCH_FILTER_THRESHOLD): doorlaten als 2x(titelfamiliescore) + 1x(samenvattingscore) >= 3. Sterk(3): vervoerders, spoorinfra, station, voertuig-samenstellingen, trein. Medium(2): tram, metro, dienst, vervoersbewijs/staking/pont. Zwak(1): losse bus/kaartje/rails - moet combineren met ander OV-woord. Weert 'VW-bus'/'schoolbus', woningbrand-leads met 'bussen rijden om' en figuratief 'op de rails', zonder bushalte/overweg/NS/trein-berichten te raken. Gewogen per familie ipv per cluster dicht ook het 'bus x3 in lead'-gat. !rss set-filter en !rss filter zijn read-only geworden (tonen het vaste filter); config-keys match_filter/allow_filter verwijderd uit do_update. V1.7.0 gedeployed op nextcloud (instance rss, enabled+started, run-log actief). --- base-config.yaml | 13 ++- buzzrss/bot.py | 236 ++++++++++++++++++++++++------------------ buzzrss/db.py | 7 ++ buzzrss/migrations.py | 12 +++ maubot.yaml | 2 +- 5 files changed, 164 insertions(+), 106 deletions(-) diff --git a/base-config.yaml b/base-config.yaml index a0f595f..9f4b52b 100644 --- a/base-config.yaml +++ b/base-config.yaml @@ -16,12 +16,17 @@ notification_template: "New post in $feed_title: [$title]($link)" # Absoluut pad in de container aanraden: /data is de persistente maubot-mount (host: /matrix/maubot/data). # Leeg = uitgeschakeld. run_log_file: /data/buzzrss-runs.log +# Doorsturen naar WhatsApp (wwebjs-api op nextcloud :3001). Leeg whatsapp_api/whatsapp_chat_id = uit. Alleen de link wordt gepost (geen titel. + +whatsapp_api: http://192.168.5.252:3001 +whatsapp_session: EdGpt +whatsapp_chat_id: 120363429643087130@g.us # Exclude paywalled articles (Telegraaf/NHD markeren dit met true) exclude_paywalled: true -# Global match filter (whitelist): applies to ALL feeds in this instance. -# A post is ONLY sent when the regex matches the title OR the (HTML-stripped) summary. -# Set it live with `!rss set-filter `. Empty = send everything. -match_filter: "(?i)\\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\\b|\\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking|sen)?\\b|\\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\\b|\\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\\b|\\b(?:openbaar vervoer|dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\\b|\\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\\b|\\b(?:ov-staking|conducteursstaking)\\b" +# OV-matchfilter: VAST in code (gewogen families), niet via config te wijzigen. +# Doorlaten als 2×(titelscore) + 1×(samenvattingscore) >= 3. Zie de families in +# buzzrss/bot.py (MATCH_FILTER_FAMILIES). `!rss set-filter` en `!rss filter` +# tonen alleen de huidige (vaste) filter. # Users who can bypass room permission checks admins: - "@user:example.com" \ No newline at end of file diff --git a/buzzrss/bot.py b/buzzrss/bot.py index 8022205..dd38b0b 100644 --- a/buzzrss/bot.py +++ b/buzzrss/bot.py @@ -49,16 +49,42 @@ from .migrations import upgrade_table rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STATE) -DEFAULT_MATCH_FILTER = ( - r"(?i)" - r"\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\b" - r"|\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking|sen)?\b" - r"|\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\b" - r"|\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\b" - r"|\b(?:openbaar vervoer|dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\b" - r"|\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\b" - r"|\b(?:ov-staking|conducteursstaking)\b" -) +# Gewogen OV-trefwoord-families voor het match-filter. Elke tuple = (regex, gewicht). +# Per familie wordt 1× geteld per tekst (niet per cluster/herhaling). Drempel (3): +# doorlaten als 2×(titelscore) + 1×(samenvattingscore) >= MATCH_FILTER_THRESHOLD +# Gewicht 3 (sterk, onmiskenbaar OV): één woord in titel óf samenvatting volstaat. +# Gewicht 2 (medium): één woord in de titel volstaat; in de samenvatting moet er +# een tweede OV-woord bijkomen. +# Gewicht 1 (zwak/dubbelzinnig: los "bus", "kaartje", "rails"): moet met een ander +# OV-woord combineren. Weert zo "VW-bus"/"schoolbus", "weer op de rails" en +# 112-berichten met "bussen rijden om" in de lead, zonder eenduidige +# bushalte/overweg/NS/trein-berichten te raken. +MATCH_FILTER_FAMILIES: list[tuple[str, int]] = [ + # sterk (3) + (r"\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\b", 3), # vervoerders + (r"\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\b", 3), # spoorinfra + (r"\bstation\w*\b", 3), # station(s) + (r"\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking)\b", 3), # voertuig-samenstelling + (r"\btrein\w*\b", 3), # trein(verkeer/staking/ongeluk/...) + # medium (2) + (r"\btram\b", 2), + (r"\bmetro\b", 2), + (r"\b(?:openbaar vervoer|dienstregeling|inchecken|uitchecken|lightrail|pendelbus|shuttlebus)\b", 2), + (r"\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\b", 2), + (r"\b(?:ov-staking|conducteursstaking)\b", 2), + (r"\b(?:pont|ponten)\b", 2), # waterbus/veerdienst + # zwak (1) — dubbelzinnig, moet combineren + (r"\bbus(?:sen)?\b", 1), + (r"\bkaart(?:je|jes)\b", 1), + (r"\brails\b", 1), +] + +# Drempel voor doorlaten (score = 2×titel + 1×samenvatting). +MATCH_FILTER_THRESHOLD = 3 + +_compiled_match_families: list[tuple[re.Pattern, int]] = [ + (re.compile(pat, re.IGNORECASE), w) for pat, w in MATCH_FILTER_FAMILIES +] class Config(BaseProxyConfig): @@ -69,10 +95,11 @@ class Config(BaseProxyConfig): helper.copy("spam_sleep") helper.copy("command_prefix") helper.copy("notification_template") - helper.copy("allow_filter") helper.copy("exclude_paywalled") - helper.copy("match_filter") helper.copy("run_log_file") + helper.copy("whatsapp_api") + helper.copy("whatsapp_session") + helper.copy("whatsapp_chat_id") helper.copy("admins") @@ -162,47 +189,42 @@ class RSSBot(Plugin): self.log.exception("Fatal error while polling feeds") @staticmethod - def _matches_in(text: str, match_filter: str) -> int: - """Aantal aaneengesloten match-clusters in text (0 als geen match). - Aaneengesloten matches (bv 'NS-trein', 'openbaar vervoer') tellen als 1.""" - spans = sorted((m.start(), m.end()) for m in re.finditer(match_filter, text)) - if not spans: - return 0 - clusters, prev_end = 1, spans[0][1] - for s, e in spans[1:]: - if s - prev_end <= 1: - prev_end = max(prev_end, e) - else: - clusters += 1 - prev_end = e - return clusters + def _fam_score(text: str) -> int: + """Som van gewichten over de OV-families die in text voorkomen (1× per familie).""" + return sum(weight for family, weight in _compiled_match_families if family.search(text)) @staticmethod - def _filter_score(title: str, summary: str, match_filter: str) -> int: - """Score = 2× hits in titel + 1× hits in summary.""" - return 2 * RSSBot._matches_in(title, match_filter) + RSSBot._matches_in(summary, match_filter) + def _filter_passes(title: str, summary: str) -> bool: + """Gewogen familiescore: doorlaten bij 2×(titelscore) + 1×(samenvattingscore) >= drempel. + + Sterke families (gewicht 3) halen de drempel al met één woord in titel óf + samenvatting. Medium (2) in de titel ook; in de samenvatting moet een tweede + OV-woord bijkomen. Zwakke woorden (losse 'bus', 'kaartje', 'rails') moeten + combineren met een ander OV-woord. Weert oa 'VW-bus', 'schoolbus', + 'bussen rijden om' in een woningbrand-lead en figuratief 'weer op de rails'. + Bewust doorgelaten: krant 'De Metro'/café 'De Tram' in de titel (medium).""" + return 2 * RSSBot._fam_score(title) + RSSBot._fam_score(summary) >= MATCH_FILTER_THRESHOLD @staticmethod - def _filter_passes(title: str, summary: str, match_filter: str) -> bool: - """Regel C: doorlaten bij (titel-match >= 1) of (totaalscore >= 3). - - Een OV-trefwoord in de titel is een sterke intentieverklaring (score 2 - volstaat). Treffers alleen in de samenvatting moeten dubbel bevestigd - worden (>= 3): zo houdt 'bussen rijden om' in de lead van een - woningbrand-bericht het artikel niet meer binnen.""" - if RSSBot._matches_in(title, match_filter) >= 1: - return True - return RSSBot._filter_score(title, summary, match_filter) >= 3 + def _filter_summary() -> str: + """Mensvriendelijke weergave van het vaste gewogen match-filter.""" + return ( + "**OV-matchfilter (vast in code, gewogen families):**\n" + "- Sterk (3): vervoerders (NS, Arriva, Qbuzz, ...), spoorinfra (perron, overweg, sein), " + "station(s), bushalte/buslijn/treinstel, trein\n" + "- Medium (2): tram, metro, openbaar vervoer/dienstregeling/inchecken, " + "ov-chipkaart/ovpay/reisproduct, ov-staking, pont\n" + "- Zwak (1): losse 'bus', 'kaartje', 'rails' — moet combineren met een ander OV-woord\n\n" + "Doorlaten: `2×(titel-score) + 1×(samenvatting-score) ≥ 3`" + ) async def _send( self, feed: Feed, entry: Entry, sub: Subscription, *, check_filter: bool = True ) -> EventID | None: if check_filter: - match_filter = self.config["match_filter"] - if match_filter: - summary = html.unescape(re.sub(r"<[^>]+>", "", entry.summary)) - if not self._filter_passes(entry.title, summary, match_filter): - return None + summary = html.unescape(re.sub(r"<[^>]+>", "", entry.summary)) + if not self._filter_passes(entry.title, summary): + return None message = sub.notification_template.safe_substitute( { @@ -222,21 +244,77 @@ class RSSBot(Plugin): self.log.warning(f"Failed to send {entry.id} of {feed.id} to {sub.room_id}: {e}") return None + async def _send_whatsapp(self, feed: Feed, entry: Entry) -> bool: + """Post entry.link naar de WhatsApp-groep via wwebjs-api (met dedup-check). True = nieuw geplaatst.""" + api = self.config.get("whatsapp_api", "") + chat_id = self.config.get("whatsapp_chat_id", "") + if not api or not chat_id: + self.log.debug("WhatsApp forwarding disabled") + return False + link = entry.link + if not link: + self.log.warning(f"No link for {entry.id}, skipping WhatsApp") + return False + if await self.dbm.is_whatsapp_posted(link): + self.log.info(f"{entry.id} already posted to WhatsApp before, skipping") + return False + session = self.config.get("whatsapp_session", "EdGpt") + base = api.rstrip("/") + # Dedup: sla over als de URL al in de doelgroep staat. Eerst de lokale + # tabel (eigen posts; daarna de globale search (vreemde posts), gefilterd + # op chat-id van het gevonden bericht. + + try: + resp = await self.http.post( + f"{base}/client/searchMessages/{session}", + json={"query": link, "options": {"limit": 1}}, + timeout=aiohttp.ClientTimeout(total=20), + ) + data = await resp.json() + if data.get("success"): + for msg in data.get("messages", []): + msg_data = msg.get("_data", {}) + if ( + msg_data.get("id", {}).get("remote") == chat_id + and link in msg_data.get("body", "") + ): + self.log.info(f"{entry.id} already in WhatsApp group, skipping") + return False + except Exception as e: + self.log.warning(f"WhatsApp dedup check failed for {entry.id}: {e}") + return False + try: + resp = await self.http.post( + f"{base}/client/sendMessage/{session}", + json={"chatId": chat_id, "content": link, "contentType": "string"}, + timeout=aiohttp.ClientTimeout(total=20), + ) + data = await resp.json() + ok = bool(data.get("success")) + if ok: + try: + await self.dbm.mark_whatsapp_posted(link) + except Exception as e: + self.log.warning(f"Failed to record WhatsApp post {entry.id}: {e}") + self.log.info(f"WhatsApp post {entry.id} to {chat_id}: {'ok' if ok else 'failed: ' + str(data)}") + return ok + except Exception as e: + self.log.warning(f"Failed to send {entry.id} to WhatsApp {chat_id}: {e}") + return False + async def _broadcast( self, feed: Feed, entry: Entry, subscriptions: list[Subscription] ) -> tuple[int, int, int]: """Send entry to all subs. Returns (placed, filtered, failed). - placed = 1 when at least one room received the message + placed = 1 when the entry was posted to at least one channel (Matrix room or newly posted WhatsApp link) filtered = 1 when the global match filter rejected the entry failed = number of subscriptions where sending failed """ self.log.debug(f"Broadcasting {entry.id} of {feed.id}") - match_filter = self.config["match_filter"] - if match_filter: - summary = html.unescape(re.sub(r"<[^>]+>", "", entry.summary)) - if not self._filter_passes(entry.title, summary, match_filter): - return 0, 1, 0 + summary = html.unescape(re.sub(r"<[^>]+>", "", entry.summary)) + if not self._filter_passes(entry.title, summary): + return 0, 1, 0 spam_sleep = self.config["spam_sleep"] if spam_sleep >= 0: results: list[EventID | None] = [] @@ -248,7 +326,8 @@ class RSSBot(Plugin): *(self._send(feed, entry, sub, check_filter=False) for sub in subscriptions) ) sent = sum(1 for result in results if result is not None) - return (1 if sent else 0, 0, len(results) - sent) + whatsapp_ok = await self._send_whatsapp(feed, entry) + return (1 if (sent or whatsapp_ok) else 0, 0, len(results) - sent) async def _poll_once(self) -> None: subs = await self.dbm.get_feeds() @@ -583,56 +662,16 @@ class RSSBot(Plugin): await evt.reply(f"Subscribed to {feed_info}") @rss.subcommand( - "set-filter", aliases=("f",), help="Set the global match filter: !rss set-filter | !rss set-filter clear" + "set-filter", aliases=("f",), help="Show the fixed global OV match filter." ) - @command.argument("match_filter", "match filter", pass_raw=True, required=False) - async def set_filter(self, evt: MessageEvent, match_filter: str = "") -> None: - if evt.sender in self.config["admins"]: - pass - elif not self.config["allow_filter"]: - await evt.reply("This bot does not allow non-admin users to set the match filter") - return - elif not await self.can_manage(evt): - return - if not match_filter: - # Lege aanroep: nooit het filter wissen. Toon usage + huidige filter. - f = self.config["match_filter"] - if f: - await evt.reply( - f"Usage: `!rss set-filter ` or `!rss set-filter clear`\n\n" - f"**Current filter:**\n```\n{f}\n```" - ) - else: - await evt.reply( - "Usage: `!rss set-filter ` or `!rss set-filter clear`\n\n" - "No filter is currently set (all posts are sent)" - ) - return - if match_filter.strip().lower() in ("clear", "none", "off", "remove"): - self.config["match_filter"] = "" - self.config.save() - await evt.reply("Global filter removed (all posts will be sent)") - return - try: - re.compile(match_filter) - except re.error: - await evt.reply("Filter is not a valid regular expression") - return - self.config["match_filter"] = match_filter - self.config.save() - await evt.reply( - f"Global filter updated: only posts matching `{match_filter}` will be sent" - ) + async def set_filter(self, evt: MessageEvent) -> None: + await evt.reply(self._filter_summary()) @rss.subcommand( "filter", aliases=("show-filter", "get-filter"), help="Show the current global match filter." ) async def command_show_filter(self, evt: MessageEvent) -> None: - f = self.config["match_filter"] - if f: - await evt.reply(f"**Current global filter:**\n```\n{f}\n```") - else: - await evt.reply("No global filter is set (all posts are sent)") + await evt.reply(self._filter_summary()) @rss.subcommand( "unsubscribe", aliases=("u", "unsub"), help="Unsubscribe this room from a feed." @@ -723,15 +762,10 @@ class RSSBot(Plugin): ) async def command_subscriptions(self, evt: MessageEvent) -> None: subscriptions = await self.dbm.get_feeds_by_room(evt.room_id) - match_filter = self.config["match_filter"] if len(subscriptions) == 0: await evt.reply("There are no RSS subscriptions in this room") return - filter_line = ( - f"**Global filter:** `{match_filter}`" - if match_filter - else "**Global filter:** none (all posts are sent)" - ) + filter_line = self._filter_summary().replace("\n", " \n") await evt.reply( "**Subscriptions in this room:**\n\n" + "\n".join( diff --git a/buzzrss/db.py b/buzzrss/db.py index b043edb..1c10433 100644 --- a/buzzrss/db.py +++ b/buzzrss/db.py @@ -232,3 +232,10 @@ class DBManager: async def set_send_notice(self, feed_id: int, room_id: RoomID, send_notice: bool) -> None: q = "UPDATE subscription SET send_notice=$3 WHERE feed_id=$1 AND room_id=$2" await self.db.execute(q, feed_id, room_id, send_notice) + + async def is_whatsapp_posted(self, link: str) -> bool: + row = await self.db.fetchrow("SELECT 1 FROM whatsapp_posted WHERE link = $1", link) + return row is not None + + async def mark_whatsapp_posted(self, link: str) -> None: + await self.db.execute("INSERT INTO whatsapp_posted (link) VALUES ($1) ON CONFLICT DO NOTHING", link) diff --git a/buzzrss/migrations.py b/buzzrss/migrations.py index 9432cf9..a31a8da 100644 --- a/buzzrss/migrations.py +++ b/buzzrss/migrations.py @@ -105,3 +105,15 @@ async def upgrade_v8(conn: Connection) -> None: async def upgrade_v9(conn: Connection) -> None: await conn.execute("ALTER TABLE feed ADD COLUMN etag TEXT DEFAULT ''") await conn.execute("ALTER TABLE feed ADD COLUMN last_modified TEXT DEFAULT ''") + + +@upgrade_table.register(description="Add WhatsApp posted-link dedup table") +async def upgrade_v10(conn: Connection, scheme: Scheme) -> None: + default = "now()" if scheme == Scheme.POSTGRES else "CURRENT_TIMESTAMP" + await conn.execute( + f"CREATE TABLE IF NOT EXISTS whatsapp_posted (" + f"link TEXT PRIMARY KEY, " + f"posted_at {'TIMESTAMP WITH TIME ZONE' if scheme == Scheme.POSTGRES else 'TIMESTAMP'} " + f"NOT NULL DEFAULT {default}" + f")" + ) diff --git a/maubot.yaml b/maubot.yaml index 677eca2..9ebea5d 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,5 +1,5 @@ id: nl.eddydevink.rss -version: 1.5.4 +version: 1.7.0 modules: - buzzrss main_class: RSSBot