v1.11.0: HTML-scrape-modus — feeds zonder RSS/Atom/JSON als pagina polled

Nieuwe module buzzrss/html_scrape.py (stdlib-only DOM + simpele selectors:
tag/.class/tag.class/#id) die een server-rendered nieuwspagina omzet in
entries. Per feed via feed.scrape_config (JSON; migratie v13): item/title/
link/date(+date_format, NL-maandnamen locale-onafhankelijk)/summary.
Scrape-entries lopen door de bestaande pipeline (OV-filter, titel-dedup,
WhatsApp-forward, run-log). Eerste scrape-feed: CNV Stads- en streekvervoer
(feed 102).

buzzrss/bot.py

@@ -24,6 +24,7 @@ import hashlib
 import html
 import io
 import logging
+import json
 import re
 
 import aiohttp
@@ -44,6 +45,8 @@ from mautrix.util.async_db import UpgradeTable
 from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper
 
 from .db import DBManager, Entry, Feed, Subscription, normalize_title
+from .html_scrape import extract_entries as _scrape_entries
+from .html_scrape import page_title as _scrape_page_title
 from .migrations import upgrade_table
 
 rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STATE)
@@ -787,7 +790,10 @@ class RSSBot(Plugin):
         etag = resp.headers.get("ETag", "")
         last_modified = resp.headers.get("Last-Modified", "")
         ct = resp.headers["Content-Type"].split(";")[0].strip()
-        if ct == "application/json" or ct == "application/feed+json":
+        scrape_cfg = self._load_scrape_config(feed)
+        if scrape_cfg:
+            feed, entries = await self._parse_html(feed, resp, scrape_cfg)
+        elif ct == "application/json" or ct == "application/feed+json":
             feed, entries = await self._parse_json(feed, resp)
         else:
             feed, entries = await self._parse_rss(feed, resp)
@@ -802,6 +808,49 @@ class RSSBot(Plugin):
                 self.log.debug(f"Failed to store fetch headers of {feed.id}")
         return feed, entries
 
+    @staticmethod
+    def _load_scrape_config(feed: Feed) -> dict | None:
+        """Parsed scrape_config JSON for HTML-page feeds; None for normal feeds.
+
+        Invalid JSON silently falls back to normal feed parsing instead of
+        taking the feed down; the validation happens when the config is set."""
+        raw = (feed.scrape_config or "").strip()
+        if not raw:
+            return None
+        try:
+            cfg = json.loads(raw)
+        except (ValueError, TypeError):
+            return None
+        return cfg if isinstance(cfg, dict) else None
+
+    async def _parse_html(
+        self, feed: Feed, resp: aiohttp.ClientResponse, cfg: dict
+    ) -> tuple[Feed, list[Entry]]:
+        content = await resp.read()
+        html_text = content.decode("utf-8", errors="replace")
+        raw_entries, skipped = _scrape_entries(html_text, feed.url, cfg)
+        self.log.debug(
+            f"Scraped {feed.url}: {len(raw_entries)} items, {skipped} skipped"
+        )
+        feed.title = _scrape_page_title(html_text) or feed.url
+        feed.subtitle = feed.url
+        feed.link = feed.url
+        entries = [
+            Entry(
+                feed_id=feed.id,
+                id=str(e["id"]),
+                date=e["date"],
+                title=e["title"],
+                summary=e["summary"],
+                link=e["link"],
+                paywalled=e["paywalled"],
+            )
+            for e in raw_entries
+        ]
+        if self.exclude_paywalled:
+            entries = [e for e in entries if not e.paywalled]
+        return feed, entries
+
     @classmethod
     async def _parse_json(
         cls, feed: Feed, resp: aiohttp.ClientResponse