diff --git a/base-config.yaml b/base-config.yaml
index 53c1bfe..f556482 100644
--- a/base-config.yaml
+++ b/base-config.yaml
@@ -15,10 +15,12 @@ notification_template: "New post in $feed_title: [$title]($link)"
# Should non-admin users be allowed to set regex filters to drop posts from feeds?
# Regexes are dangerous and may lead to DoS attacks.
allow_filter: false
+# Exclude paywalled articles (Telegraaf/NHD markeren dit met true)
+exclude_paywalled: true
# Global match filter (whitelist): applies to ALL feeds in this instance.
# A post is ONLY sent when the regex matches the title OR the (HTML-stripped) summary.
# Set it live with `!rss set-filter `. Empty = send everything.
-match_filter: "(?i)\\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\\b|\\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net)?\\b|\\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\\b|\\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord)\\b|\\b(?:dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\\b|\\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\\b|\\b(?:ov-staking|staking|stakingen)\\b"
+match_filter: "(?i)\\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\\b|\\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking)?\\b|\\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\\b|\\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\\b|\\b(?:openbaar vervoer|dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\\b|\\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\\b|\\b(?:ov-staking|conducteursstaking)\\b"
# Users who can bypass room permission checks
admins:
-- "@user:example.com"
+- "@user:example.com"
\ No newline at end of file
diff --git a/buzzrss/bot.py b/buzzrss/bot.py
index 3fce734..1068dda 100644
--- a/buzzrss/bot.py
+++ b/buzzrss/bot.py
@@ -51,12 +51,12 @@ rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STAT
DEFAULT_MATCH_FILTER = (
r"(?i)"
r"\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\b"
- r"|\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net)?\b"
+ r"|\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking)?\b"
r"|\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\b"
- r"|\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord)\b"
- r"|\b(?:dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\b"
+ r"|\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\b"
+ r"|\b(?:openbaar vervoer|dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\b"
r"|\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\b"
- r"|\b(?:ov-staking|staking|stakingen)\b"
+ r"|\b(?:ov-staking|conducteursstaking)\b"
)
@@ -69,6 +69,7 @@ class Config(BaseProxyConfig):
helper.copy("command_prefix")
helper.copy("notification_template")
helper.copy("allow_filter")
+ helper.copy("exclude_paywalled")
helper.copy("match_filter")
helper.copy("admins")
@@ -90,6 +91,7 @@ class BoolArgument(command.Argument):
class RSSBot(Plugin):
dbm: DBManager
+ exclude_paywalled: bool = False
poll_task: asyncio.Future
poll_sema: asyncio.Semaphore | None
http: aiohttp.ClientSession
@@ -117,6 +119,8 @@ class RSSBot(Plugin):
self.poll_sema = (
asyncio.Semaphore(poll_parallelism_limit) if poll_parallelism_limit > 0 else None
)
+ # single-instance fork: class-level since _parse_rss is a classmethod
+ type(self).exclude_paywalled = bool(self.config["exclude_paywalled"])
async def stop(self) -> None:
await super().stop()
@@ -311,7 +315,10 @@ class RSSBot(Plugin):
feed.title = feed_data.get("title", feed.url)
feed.subtitle = feed_data.get("description", "")
feed.link = feed_data.get("link", "")
- return feed, [cls._parse_rss_entry(feed.id, entry) for entry in parsed_data.entries]
+ entries = [cls._parse_rss_entry(feed.id, entry) for entry in parsed_data.entries]
+ if cls.exclude_paywalled:
+ entries = [e for e in entries if not e.paywalled]
+ return feed, entries
@classmethod
def _parse_rss_entry(cls, feed_id: int, entry: Any) -> Entry:
@@ -333,6 +340,7 @@ class RSSBot(Plugin):
title=getattr(entry, "title", ""),
summary=getattr(entry, "description", "").strip(),
link=getattr(entry, "link", ""),
+ paywalled=str(getattr(entry, "premium", "")).lower() == "true",
)
@staticmethod
diff --git a/buzzrss/db.py b/buzzrss/db.py
index 52bd08f..65e33b3 100644
--- a/buzzrss/db.py
+++ b/buzzrss/db.py
@@ -91,6 +91,7 @@ class Entry:
title: str
summary: str
link: str
+ paywalled: bool = False
def __attrs_post_init__(self):
if self.date.tzinfo is None:
@@ -161,14 +162,14 @@ class DBManager:
for entry in entries:
entry.feed_id = override_feed_id
records = [attr.astuple(entry) for entry in entries]
- columns = ("feed_id", "id", "date", "title", "summary", "link")
+ columns = ("feed_id", "id", "date", "title", "summary", "link", "paywalled")
async with self.db.acquire() as conn:
if self.db.scheme == Scheme.POSTGRES:
await conn.copy_records_to_table("entry", records=records, columns=columns)
else:
q = (
- "INSERT INTO entry (feed_id, id, date, title, summary, link) "
- "VALUES ($1, $2, $3, $4, $5, $6)"
+ "INSERT INTO entry (feed_id, id, date, title, summary, link, paywalled) "
+ "VALUES ($1, $2, $3, $4, $5, $6, $7)"
)
await conn.executemany(q, records)
diff --git a/buzzrss/migrations.py b/buzzrss/migrations.py
index 2d9f3b6..6d15ec8 100644
--- a/buzzrss/migrations.py
+++ b/buzzrss/migrations.py
@@ -94,3 +94,8 @@ async def upgrade_v6(conn: Connection) -> None:
@upgrade_table.register(description="Remove per-subscription filter (single global config filter)")
async def upgrade_v7(conn: Connection) -> None:
await conn.execute("ALTER TABLE subscription DROP COLUMN IF EXISTS match_filter")
+
+
+@upgrade_table.register(description="Add paywalled flag to entries")
+async def upgrade_v8(conn: Connection) -> None:
+ await conn.execute("ALTER TABLE entry ADD COLUMN paywalled BOOLEAN NOT NULL DEFAULT false")
diff --git a/maubot.yaml b/maubot.yaml
index 7db05d9..a401b08 100644
--- a/maubot.yaml
+++ b/maubot.yaml
@@ -1,5 +1,5 @@
id: nl.eddydevink.rss
-version: 1.2.2
+version: 1.3.0
modules:
- buzzrss
main_class: RSSBot