diff --git a/base-config.yaml b/base-config.yaml index 53c1bfe..f556482 100644 --- a/base-config.yaml +++ b/base-config.yaml @@ -15,10 +15,12 @@ notification_template: "New post in $feed_title: [$title]($link)" # Should non-admin users be allowed to set regex filters to drop posts from feeds? # Regexes are dangerous and may lead to DoS attacks. allow_filter: false +# Exclude paywalled articles (Telegraaf/NHD markeren dit met true) +exclude_paywalled: true # Global match filter (whitelist): applies to ALL feeds in this instance. # A post is ONLY sent when the regex matches the title OR the (HTML-stripped) summary. # Set it live with `!rss set-filter `. Empty = send everything. -match_filter: "(?i)\\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\\b|\\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net)?\\b|\\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\\b|\\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord)\\b|\\b(?:dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\\b|\\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\\b|\\b(?:ov-staking|staking|stakingen)\\b" +match_filter: "(?i)\\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\\b|\\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking)?\\b|\\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\\b|\\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\\b|\\b(?:openbaar vervoer|dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\\b|\\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\\b|\\b(?:ov-staking|conducteursstaking)\\b" # Users who can bypass room permission checks admins: -- "@user:example.com" +- "@user:example.com" \ No newline at end of file diff --git a/buzzrss/bot.py b/buzzrss/bot.py index 3fce734..1068dda 100644 --- a/buzzrss/bot.py +++ b/buzzrss/bot.py @@ -51,12 +51,12 @@ rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STAT DEFAULT_MATCH_FILTER = ( r"(?i)" r"\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|breng|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|ov)\b" - r"|\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net)?\b" + r"|\b(?:bus|tram|metro)(?:halte|haltes|lijn|lijnen|station|stations|verkeer|net|staking)?\b" r"|\btrein(?:en|verkeer|stel(?:len)?|staking|ongeluk|reizigers?|station|stations)?\b" - r"|\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord)\b" - r"|\b(?:dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\b" + r"|\b(?:spoorweg|spoorbaan|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|perrons|overweg|overwegen|seinstoring|wisselstoring|sein|ontsporing|ontspoord|spoorstaking)\b" + r"|\b(?:openbaar vervoer|dienstregeling|reizigers?|inchecken|uitchecken|pont|ponten|lightrail|pendelbus|shuttlebus)\b" r"|\b(?:ov-chipkaart|chipkaart|ovpay|ov-pay|reisproduct|studentenreisproduct|vervoersbewijs|vervoerbewijs|opwaarderen)\b" - r"|\b(?:ov-staking|staking|stakingen)\b" + r"|\b(?:ov-staking|conducteursstaking)\b" ) @@ -69,6 +69,7 @@ class Config(BaseProxyConfig): helper.copy("command_prefix") helper.copy("notification_template") helper.copy("allow_filter") + helper.copy("exclude_paywalled") helper.copy("match_filter") helper.copy("admins") @@ -90,6 +91,7 @@ class BoolArgument(command.Argument): class RSSBot(Plugin): dbm: DBManager + exclude_paywalled: bool = False poll_task: asyncio.Future poll_sema: asyncio.Semaphore | None http: aiohttp.ClientSession @@ -117,6 +119,8 @@ class RSSBot(Plugin): self.poll_sema = ( asyncio.Semaphore(poll_parallelism_limit) if poll_parallelism_limit > 0 else None ) + # single-instance fork: class-level since _parse_rss is a classmethod + type(self).exclude_paywalled = bool(self.config["exclude_paywalled"]) async def stop(self) -> None: await super().stop() @@ -311,7 +315,10 @@ class RSSBot(Plugin): feed.title = feed_data.get("title", feed.url) feed.subtitle = feed_data.get("description", "") feed.link = feed_data.get("link", "") - return feed, [cls._parse_rss_entry(feed.id, entry) for entry in parsed_data.entries] + entries = [cls._parse_rss_entry(feed.id, entry) for entry in parsed_data.entries] + if cls.exclude_paywalled: + entries = [e for e in entries if not e.paywalled] + return feed, entries @classmethod def _parse_rss_entry(cls, feed_id: int, entry: Any) -> Entry: @@ -333,6 +340,7 @@ class RSSBot(Plugin): title=getattr(entry, "title", ""), summary=getattr(entry, "description", "").strip(), link=getattr(entry, "link", ""), + paywalled=str(getattr(entry, "premium", "")).lower() == "true", ) @staticmethod diff --git a/buzzrss/db.py b/buzzrss/db.py index 52bd08f..65e33b3 100644 --- a/buzzrss/db.py +++ b/buzzrss/db.py @@ -91,6 +91,7 @@ class Entry: title: str summary: str link: str + paywalled: bool = False def __attrs_post_init__(self): if self.date.tzinfo is None: @@ -161,14 +162,14 @@ class DBManager: for entry in entries: entry.feed_id = override_feed_id records = [attr.astuple(entry) for entry in entries] - columns = ("feed_id", "id", "date", "title", "summary", "link") + columns = ("feed_id", "id", "date", "title", "summary", "link", "paywalled") async with self.db.acquire() as conn: if self.db.scheme == Scheme.POSTGRES: await conn.copy_records_to_table("entry", records=records, columns=columns) else: q = ( - "INSERT INTO entry (feed_id, id, date, title, summary, link) " - "VALUES ($1, $2, $3, $4, $5, $6)" + "INSERT INTO entry (feed_id, id, date, title, summary, link, paywalled) " + "VALUES ($1, $2, $3, $4, $5, $6, $7)" ) await conn.executemany(q, records) diff --git a/buzzrss/migrations.py b/buzzrss/migrations.py index 2d9f3b6..6d15ec8 100644 --- a/buzzrss/migrations.py +++ b/buzzrss/migrations.py @@ -94,3 +94,8 @@ async def upgrade_v6(conn: Connection) -> None: @upgrade_table.register(description="Remove per-subscription filter (single global config filter)") async def upgrade_v7(conn: Connection) -> None: await conn.execute("ALTER TABLE subscription DROP COLUMN IF EXISTS match_filter") + + +@upgrade_table.register(description="Add paywalled flag to entries") +async def upgrade_v8(conn: Connection) -> None: + await conn.execute("ALTER TABLE entry ADD COLUMN paywalled BOOLEAN NOT NULL DEFAULT false") diff --git a/maubot.yaml b/maubot.yaml index 7db05d9..a401b08 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,5 +1,5 @@ id: nl.eddydevink.rss -version: 1.2.2 +version: 1.3.0 modules: - buzzrss main_class: RSSBot