v1.3.0: paywall-filter (Telegraaf/NHD premium) + staking-fix

- exclude_paywalled config (default true): verwijdert feedparser-entries met
  <premium>true</premium> vóór opslag (Telegraaf en NHD markeren dit; AD/
  Parool/Trouw/VK/NRC niet)
- v8-migratie: entry.paywalled BOOLEAN
- staking-fix (v1.2.3): losse 'staking/stakingen' uit filter; alleen ov-
  specifiek: ov-staking, bus/tram/metro/spoor/conducteurs-staking,
  'openbaar vervoer'
- config-proof: match_filter altijd enkelregelig YAML

base-config.yaml

@@ -15,10 +15,12 @@ notification_template: "New post in $feed_title: [$title]($link)"
 # Should non-admin users be allowed to set regex filters to drop posts from feeds?
 # Regexes are dangerous and may lead to DoS attacks.
 allow_filter: false
+# Exclude paywalled articles (Telegraaf/NHD markeren dit met <premium>true</premium>)
+exclude_paywalled: true
 # Global match filter (whitelist): applies to ALL feeds in this instance.
 # A post is ONLY sent when the regex matches the title OR the (HTML-stripped) summary.
 # Set it live with `!rss set-filter <regex>`. Empty = send everything.
-match_filter: "(?i)\\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\\b|\\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net)?\\b|\\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\\b|\\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord)\\b|\\b(?:dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\\b|\\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\\b|\\b(?:ov-staking|staking|stakingen)\\b"
+match_filter: "(?i)\\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\\b|\\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking)?\\b|\\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\\b|\\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\\b|\\b(?:openbaar vervoer|dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\\b|\\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\\b|\\b(?:ov-staking|conducteursstaking)\\b"
 # Users who can bypass room permission checks
 admins:
 - "@user:example.com"

buzzrss/bot.py

@@ -51,12 +51,12 @@ rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STAT
 DEFAULT_MATCH_FILTER = (
     r"(?i)"
     r"\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\b"
-    r"|\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net)?\b"
+    r"|\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking)?\b"
     r"|\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\b"
-    r"|\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord)\b"
+    r"|\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\b"
-    r"|\b(?:dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\b"
+    r"|\b(?:openbaar vervoer|dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\b"
     r"|\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\b"
-    r"|\b(?:ov-staking|staking|stakingen)\b"
+    r"|\b(?:ov-staking|conducteursstaking)\b"
 )
 
 
@@ -69,6 +69,7 @@ class Config(BaseProxyConfig):
         helper.copy("command_prefix")
         helper.copy("notification_template")
         helper.copy("allow_filter")
+        helper.copy("exclude_paywalled")
         helper.copy("match_filter")
         helper.copy("admins")
 
@@ -90,6 +91,7 @@ class BoolArgument(command.Argument):
 
 class RSSBot(Plugin):
     dbm: DBManager
+    exclude_paywalled: bool = False
     poll_task: asyncio.Future
     poll_sema: asyncio.Semaphore | None
     http: aiohttp.ClientSession
@@ -117,6 +119,8 @@ class RSSBot(Plugin):
         self.poll_sema = (
             asyncio.Semaphore(poll_parallelism_limit) if poll_parallelism_limit > 0 else None
         )
+        # single-instance fork: class-level since _parse_rss is a classmethod
+        type(self).exclude_paywalled = bool(self.config["exclude_paywalled"])
 
     async def stop(self) -> None:
         await super().stop()
@@ -311,7 +315,10 @@ class RSSBot(Plugin):
         feed.title = feed_data.get("title", feed.url)
         feed.subtitle = feed_data.get("description", "")
         feed.link = feed_data.get("link", "")
-        return feed, [cls._parse_rss_entry(feed.id, entry) for entry in parsed_data.entries]
+        entries = [cls._parse_rss_entry(feed.id, entry) for entry in parsed_data.entries]
+        if cls.exclude_paywalled:
+            entries = [e for e in entries if not e.paywalled]
+        return feed, entries
 
     @classmethod
     def _parse_rss_entry(cls, feed_id: int, entry: Any) -> Entry:
@@ -333,6 +340,7 @@ class RSSBot(Plugin):
             title=getattr(entry, "title", ""),
             summary=getattr(entry, "description", "").strip(),
             link=getattr(entry, "link", ""),
+            paywalled=str(getattr(entry, "premium", "")).lower() == "true",
         )
 
     @staticmethod

buzzrss/db.py

@@ -91,6 +91,7 @@ class Entry:
     title: str
     summary: str
     link: str
+    paywalled: bool = False
 
     def __attrs_post_init__(self):
         if self.date.tzinfo is None:
@@ -161,14 +162,14 @@ class DBManager:
             for entry in entries:
                 entry.feed_id = override_feed_id
         records = [attr.astuple(entry) for entry in entries]
-        columns = ("feed_id", "id", "date", "title", "summary", "link")
+        columns = ("feed_id", "id", "date", "title", "summary", "link", "paywalled")
         async with self.db.acquire() as conn:
             if self.db.scheme == Scheme.POSTGRES:
                 await conn.copy_records_to_table("entry", records=records, columns=columns)
             else:
                 q = (
-                    "INSERT INTO entry (feed_id, id, date, title, summary, link) "
+                    "INSERT INTO entry (feed_id, id, date, title, summary, link, paywalled) "
-                    "VALUES ($1, $2, $3, $4, $5, $6)"
+                    "VALUES ($1, $2, $3, $4, $5, $6, $7)"
                 )
                 await conn.executemany(q, records)
 

buzzrss/migrations.py

@@ -94,3 +94,8 @@ async def upgrade_v6(conn: Connection) -> None:
 @upgrade_table.register(description="Remove per-subscription filter (single global config filter)")
 async def upgrade_v7(conn: Connection) -> None:
     await conn.execute("ALTER TABLE subscription DROP COLUMN IF EXISTS match_filter")
+
+
+@upgrade_table.register(description="Add paywalled flag to entries")
+async def upgrade_v8(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE entry ADD COLUMN paywalled BOOLEAN NOT NULL DEFAULT false")

maubot.yaml

@@ -1,5 +1,5 @@
 id: nl.eddydevink.rss
-version: 1.2.2
+version: 1.3.0
 modules:
 - buzzrss
 main_class: RSSBot