v1.11.0: HTML-scrape-modus — feeds zonder RSS/Atom/JSON als pagina polled

Nieuwe module buzzrss/html_scrape.py (stdlib-only DOM + simpele selectors:
tag/.class/tag.class/#id) die een server-rendered nieuwspagina omzet in
entries. Per feed via feed.scrape_config (JSON; migratie v13): item/title/
link/date(+date_format, NL-maandnamen locale-onafhankelijk)/summary.
Scrape-entries lopen door de bestaande pipeline (OV-filter, titel-dedup,
WhatsApp-forward, run-log). Eerste scrape-feed: CNV Stads- en streekvervoer
(feed 102).

buzzrss/bot.py

@@ -24,6 +24,7 @@ import hashlib
 import html
 import io
 import logging
+import json
 import re
 
 import aiohttp
@@ -44,6 +45,8 @@ from mautrix.util.async_db import UpgradeTable
 from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper
 
 from .db import DBManager, Entry, Feed, Subscription, normalize_title
+from .html_scrape import extract_entries as _scrape_entries
+from .html_scrape import page_title as _scrape_page_title
 from .migrations import upgrade_table
 
 rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STATE)
@@ -78,6 +81,10 @@ rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STAT
 #   is geen OV-nieuws). Zwartgelijste woorden leveren geen score op, maar
 #   blokkeren de familie niet: een écht "station"-woord ernaast scoort gewoon.
 # Bewust geaccepteerde botsingen: krant "De Metro"/café "De Tram" (medium).
+# 'spoorbrug' is MEDIUM (niet sterk/whitelist): een brug is vaak louter locatie
+# ('bij de spoorbrug tussen Liempde en Boxtel', Stuifmail 2026-09-12) — één
+# spoorbrug-hit in de samenvatting is geen OV-nieuws; écht spoorbrug-nieuws
+# combineert met trein/spoor/verkeer en haalt de drempel alsnog (besluit 2026-09-12).
 # Losse algemene woorden zijn zwak (moeten combineren met een andere familie):
 # "trein", "sprinter" (atleet), "vervoerder" (logistiek), "reiziger",
 # "inchecken"/"uitchecken" (Schiphol/hotel), "station(s)" (omroepen), "bus",
@@ -148,7 +155,7 @@ rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STAT
 MATCH_FILTER_FAMILIES: list[tuple[str, int]] = [
     # sterk (3)
     (r"\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|prorail|hermes|rrreis|qliner|ov)\b", 3),  # vervoerders ('Breng' staat in de zwakke groep: botst met het werkwoord "brengen")
-    (r"\b(?:spoorweg|spoorba(?:an|nen)|spoorlijn|spoorverkeer|spoorwegovergang|spoorbrug|spoorstoring|perron|seinstoring|wisselstoring|spoorstaking)\w*\b", 3),  # spoorinfra; let op de klinkerwisseling: 'ontsporing' (open lettergreep, één o) vs 'ontspoord'/'ontspoorde' (dicht, dubbel o) — vandaar beide stammen
+    (r"\b(?:spoorweg|spoorba(?:an|nen)|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|seinstoring|wisselstoring|spoorstaking)\w*\b", 3),  # spoorinfra; let op de klinkerwisseling: 'ontsporing' (open lettergreep, één o) vs 'ontspoord'/'ontspoorde' (dicht, dubbel o) — vandaar beide stammen. Géén 'spoorbrug': die is medium (vaak locatie-referentie, besluit 2026-09-12)
     (r"\b(?:treinstation|treinstations|metrostation|metrostations|busstation|busstations)\b", 2),  # station-samenstellingen: onmiskenbaar OV, maar MEDIUM — één station-woord náást een ander onderwerp ('pand naast busstation', besluit 2026-09-11) is geen OV-bericht; in de samenvatting moet een tweede OV-woord bijkomen
     (r"(?<![\w-])\bstations?\b", 1),  # los 'station(s)' is ZWAK: ook tv-/radio-/regionale omroepstations ('de lokale stations'), moet combineren met een andere OV-familie
     (r"\b(?:bus|tram|metro)-?(?:halte|haltes|lijn|lijnen|verkeer|net|staking|ba(?:an|nen)|hok(?:je|jes)|chauffeur)\w*\b", 3),  # voertuig-samenstelling (bushalte/buslijn/bushokje/busbanen/…); géén 'station': *station-samenstellingen zitten in de station-familie, anders telde 'busstation' dubbel mee (via 'bus' én 'station') — fix 2026-09-11
@@ -164,6 +171,7 @@ MATCH_FILTER_FAMILIES: list[tuple[str, int]] = [
     (r"\b(?:ov-staking|conducteursstaking)\w*\b", 2),
     (r"\bconduct(?:eur|eurs)\b", 2),  # trein-/buspersoneel: onmiskenbaar OV-jargon
     (r"\b(?:pont|ponten)\b", 2),  # waterbus/veerdienst
+    (r"\bspoorbrug(?:gen)?\b", 2),  # spoorinfra, maar MEDIUM: vaak locatie-referentie, geen OV-nieuws ('bij de spoorbrug', Stuifmail 2026-09-12); moet combineren — zelfde regel als station-samenstellingen (besluit 2026-09-11)
     # zwak (1) — dubbelzinnig, moet combineren
     (r"(?<!-)\bbus(?:sen)?\b", 1),  # ná een koppelteken geen OV: 'VW-bus', 'school-bus' 
     (r"\b(?:pendelbus|shuttlebus)\w*\b", 1),  # evenement-/hotel-/parkeerpendel ('pendelbus naar de golfbaan') is geen OV; synoniemen bewust in één familie
@@ -231,7 +239,7 @@ MATCH_FILTER_WHITELIST: frozenset[str] = frozenset({
     "qliner", "flixbus", "eurostar", "thalys", "gvu", "ovpay",
     # Spoorinfra-jargon (samenvoegingen, onmiskenbaar OV)
     "spoorstoring", "spoorverkeer", "spoorwegovergang", "spoorlijn", "spoorbaan",
-    "spoorbrug", "spoorstaking", "perron", "wisselstoring", "seinstoring", "treinstoring", "treinverkeer",
+    "spoorstaking", "perron", "wisselstoring", "seinstoring", "treinstoring", "treinverkeer",
     "treinstaking", "treinongeval", "treinbotsing", "treinstel",
     # Bus/tram/metro-jargon
     "bushalte", "bushaltes", "busbaan", "buslijn", "busnet",
@@ -472,7 +480,7 @@ class RSSBot(Plugin):
             "- Sterk (3): vervoerders (NS, Arriva, ProRail, Qbuzz, ...), spoorinfra (spoorlijn, spoorwegovergang, perron, spoorstoring), "
             "samenstellingen als bushalte/buslijn/bushokje/busbaan/busverkeer, treinstoring/treinverkeer/treinstaking, "
             "intercity/streekvervoer/veerdienst/buurtbus/belbus, 'lijn 12'\n"
-            "- Medium (2): tram, metro, openbaar vervoer, lightrail, halte, pont, ov-chipkaart/ovpay, ov-staking, conducteur, "
+            "- Medium (2): tram, metro, openbaar vervoer, lightrail, halte, pont, spoorbrug, ov-chipkaart/ovpay, ov-staking, conducteur, "
             "trein-/bus-/metrostation — één medium-hit volstaat niet, moet combineren met een andere OV-familie; "
             "medium + zwak uitsluitend in de samenvatting (geen hit in de titel) telt niet\n"
             "- Zwak (1): losse 'bus' (niet ná een koppelteken), 'pendelbus'/'shuttlebus', 'trein', 'station(s)', 'breng', 'overweg', 'overwegen', "
@@ -782,7 +790,10 @@ class RSSBot(Plugin):
         etag = resp.headers.get("ETag", "")
         last_modified = resp.headers.get("Last-Modified", "")
         ct = resp.headers["Content-Type"].split(";")[0].strip()
-        if ct == "application/json" or ct == "application/feed+json":
+        scrape_cfg = self._load_scrape_config(feed)
+        if scrape_cfg:
+            feed, entries = await self._parse_html(feed, resp, scrape_cfg)
+        elif ct == "application/json" or ct == "application/feed+json":
             feed, entries = await self._parse_json(feed, resp)
         else:
             feed, entries = await self._parse_rss(feed, resp)
@@ -797,6 +808,49 @@ class RSSBot(Plugin):
                 self.log.debug(f"Failed to store fetch headers of {feed.id}")
         return feed, entries
 
+    @staticmethod
+    def _load_scrape_config(feed: Feed) -> dict | None:
+        """Parsed scrape_config JSON for HTML-page feeds; None for normal feeds.
+
+        Invalid JSON silently falls back to normal feed parsing instead of
+        taking the feed down; the validation happens when the config is set."""
+        raw = (feed.scrape_config or "").strip()
+        if not raw:
+            return None
+        try:
+            cfg = json.loads(raw)
+        except (ValueError, TypeError):
+            return None
+        return cfg if isinstance(cfg, dict) else None
+
+    async def _parse_html(
+        self, feed: Feed, resp: aiohttp.ClientResponse, cfg: dict
+    ) -> tuple[Feed, list[Entry]]:
+        content = await resp.read()
+        html_text = content.decode("utf-8", errors="replace")
+        raw_entries, skipped = _scrape_entries(html_text, feed.url, cfg)
+        self.log.debug(
+            f"Scraped {feed.url}: {len(raw_entries)} items, {skipped} skipped"
+        )
+        feed.title = _scrape_page_title(html_text) or feed.url
+        feed.subtitle = feed.url
+        feed.link = feed.url
+        entries = [
+            Entry(
+                feed_id=feed.id,
+                id=str(e["id"]),
+                date=e["date"],
+                title=e["title"],
+                summary=e["summary"],
+                link=e["link"],
+                paywalled=e["paywalled"],
+            )
+            for e in raw_entries
+        ]
+        if self.exclude_paywalled:
+            entries = [e for e in entries if not e.paywalled]
+        return feed, entries
+
     @classmethod
     async def _parse_json(
         cls, feed: Feed, resp: aiohttp.ClientResponse

buzzrss/db.py

@@ -68,6 +68,7 @@ class Feed:
     error_count: int = 0
     etag: str = ""
     last_modified: str = ""
+    scrape_config: str = ""
 
     subscriptions: list[Subscription] = attr.ib(factory=lambda: [])
 
@@ -137,7 +138,7 @@ class DBManager:
     async def get_feeds(self) -> list[Feed]:
         q = """
         SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified,
-               room_id, user_id, notification_template, send_notice
+               scrape_config, room_id, user_id, notification_template, send_notice
         FROM feed INNER JOIN subscription ON feed.id = subscription.feed_id
         """
         rows = await self.db.fetch(q)
@@ -153,7 +154,7 @@ class DBManager:
     async def get_feeds_by_room(self, room_id: RoomID) -> list[tuple[Feed, UserID]]:
         q = """
         SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified,
-               user_id FROM feed
+               scrape_config, user_id FROM feed
         INNER JOIN subscription ON feed.id = subscription.feed_id AND subscription.room_id = $1
         """
         rows = await self.db.fetch(q, room_id)
@@ -201,7 +202,7 @@ class DBManager:
                 await conn.executemany(q, records)
 
     async def get_feed_by_url(self, url: str) -> Feed | None:
-        q = "SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified FROM feed WHERE url=$1"
+        q = "SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified, scrape_config FROM feed WHERE url=$1"
         return Feed.from_row(await self.db.fetchrow(q, url))
 
     async def get_subscription(
@@ -209,7 +210,7 @@ class DBManager:
     ) -> tuple[Subscription | None, Feed | None]:
         q = """
         SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified,
-               room_id, user_id, notification_template, send_notice
+               scrape_config, room_id, user_id, notification_template, send_notice
         FROM feed LEFT JOIN subscription ON feed.id = subscription.feed_id AND room_id = $2
         WHERE feed.id = $1
         """

buzzrss/html_scrape.py

@@ -0,0 +1,214 @@
+# HTML page scraping for feeds without RSS/Atom/JSON.
+#
+# Used when a feed row carries a `scrape_config` JSON blob. The scrape mode
+# turns a plain server-rendered news list page into entries, so sources that
+# publish no feed at all (vakbonden, gemeenten, OV-bedrijven) can still be
+# followed with the exact same pipeline (OV-filter, title dedup, WhatsApp).
+#
+# Selector syntax is deliberately small: one element per selector, no
+# combinators. Supported:  "tag"  ".class"  "tag.class"  "#id"  "*"
+# Config keys:
+#   item        (required) element selector for each news card/item
+#   title       (required) element selector for the headline inside the item
+#   link        (required) element selector for the <a href> inside the item
+#   date        (optional) element selector for the date text; the element's
+#               datetime/content attributes win over its text
+#   date_format (optional) strptime format for the date text. Dutch month
+#               names are translated to numbers first, so "%d %B %Y" works
+#               for "11 september 2026" regardless of the process locale.
+#   summary     (optional) element selector for a short text inside the item
+#
+# Items without a resolvable link or a non-empty title are skipped; the caller
+# counts skips for sanity checking.
+
+from __future__ import annotations
+
+import re
+import urllib.parse
+from datetime import datetime, timezone
+from html.parser import HTMLParser
+
+_NL_MONTHS = {
+    "januari": 1, "februari": 2, "maart": 3, "april": 4, "mei": 5, "juni": 6,
+    "juli": 7, "augustus": 8, "september": 9, "oktober": 10, "november": 11,
+    "december": 12,
+}
+
+_WS = re.compile(r"\s+")
+
+
+class _Node:
+    __slots__ = ("tag", "attrs", "children", "text")
+
+    def __init__(self, tag: str, attrs: dict[str, str]) -> None:
+        self.tag = tag
+        self.attrs = attrs
+        self.children: list[_Node] = []
+        self.text = ""
+
+
+class _Builder(HTMLParser):
+    """Turns an HTML document into a small DOM tree (stdlib only)."""
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.root = _Node("#root", {})
+        self._stack = [self.root]
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        node = _Node(tag, {k: (v or "") for k, v in attrs})
+        self._stack[-1].children.append(node)
+        self._stack.append(node)
+
+    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        node = _Node(tag, {k: (v or "") for k, v in attrs})
+        self._stack[-1].children.append(node)
+
+    def handle_endtag(self, tag: str) -> None:
+        if len(self._stack) > 1:
+            self._stack.pop()
+
+    def handle_data(self, data: str) -> None:
+        self._stack[-1].text += data
+
+
+def _match(node: _Node, sel: str) -> bool:
+    if sel == "*":
+        return True
+    cls = None
+    if sel.startswith("."):
+        tag, cls = None, sel[1:]
+    elif sel.startswith("#"):
+        return node.attrs.get("id", "") == sel[1:]
+    elif "." in sel:
+        tag, cls = sel.split(".", 1)
+    else:
+        tag = sel
+    if tag and node.tag != tag:
+        return False
+    if cls:
+        classes = node.attrs.get("class", "").split()
+        if cls not in classes:
+            return False
+    return True
+
+
+def _iter_tree(node: _Node):
+    for child in node.children:
+        yield child
+        yield from _iter_tree(child)
+
+
+def _node_text(node: _Node) -> str:
+    parts = [node.text]
+
+    def walk(n: _Node) -> None:
+        for c in n.children:
+            parts.append(c.text)
+            walk(c)
+
+    walk(node)
+    return _WS.sub(" ", "".join(parts)).strip()
+
+
+def _parse_date(text: str, fmt: str | None, attrs: dict[str, str]) -> datetime | None:
+    raw = attrs.get("datetime") or attrs.get("content") or ""
+    for candidate in (raw.strip(), text):
+        if not candidate:
+            continue
+        try:
+            return datetime.fromisoformat(candidate).astimezone(timezone.utc)
+        except ValueError:
+            pass
+        if fmt:
+            # Dutch month names -> numbers so %B works under any locale.
+            # The format's %B is translated to %m alongside (the text now
+            # carries the number, not the name).
+            translated = candidate
+            for name, num in _NL_MONTHS.items():
+                translated = re.sub(
+                    rf"\b{name}\b", str(num), translated, flags=re.IGNORECASE
+                )
+            translated_fmt = fmt.replace("%B", "%m")
+            try:
+                return datetime.strptime(translated, translated_fmt).replace(
+                    tzinfo=timezone.utc
+                )
+            except ValueError:
+                pass
+    return None
+
+
+def _first(node: _Node, sel: str) -> _Node | None:
+    for n in _iter_tree(node):
+        if _match(n, sel):
+            return n
+    return None
+
+
+def _href(node: _Node, sel: str, base: str) -> str:
+    # The item node itself may be the <a> (whole card is a link): match it first.
+    targets = [node]
+    targets.extend(_iter_tree(node))
+    for n in targets:
+        if _match(n, sel):
+            href = (n.attrs.get("href") or "").strip()
+            if href:
+                return urllib.parse.urljoin(base, href)
+    return ""
+
+
+def page_title(html_text: str) -> str:
+    """First <title> text, if any."""
+    m = re.search(r"<title[^>]*>(.*?)</title>", html_text, re.IGNORECASE | re.DOTALL)
+    return _WS.sub(" ", m.group(1)).strip() if m else ""
+
+
+def extract_entries(
+    html_text: str, base_url: str, cfg: dict[str, object]
+) -> tuple[list[dict[str, object]], int]:
+    """Returns (entries, skipped). Each entry dict:
+    {id, date, title, summary, link, paywalled} with date as tz-aware datetime."""
+    item_sel = str(cfg.get("item") or "").strip()
+    title_sel = str(cfg.get("title") or "").strip()
+    link_sel = str(cfg.get("link") or "").strip()
+    if not (item_sel and title_sel and link_sel):
+        raise ValueError("scrape_config requires item, title and link selectors")
+    date_sel = str(cfg.get("date") or "").strip()
+    date_format = str(cfg.get("date_format") or "").strip() or None
+    summary_sel = str(cfg.get("summary") or "").strip()
+
+    builder = _Builder()
+    try:
+        builder.feed(html_text)
+        builder.close()
+    except Exception as e:  # malformed HTML must not kill the whole poll
+        raise ValueError(f"HTML parse failed: {e}") from e
+
+    entries: list[dict[str, object]] = []
+    skipped = 0
+    for node in _iter_tree(builder.root):
+        if not _match(node, item_sel):
+            continue
+        link = _href(node, link_sel, base_url)
+        title_node = _first(node, title_sel)
+        title = _node_text(title_node) if title_node else ""
+        if not link or not title:
+            skipped += 1
+            continue
+        date = None
+        if date_sel:
+            date_node = _first(node, date_sel)
+            if date_node:
+                date = _parse_date(_node_text(date_node), date_format, date_node.attrs)
+        entries.append(
+            {
+                "id": link,
+                "date": date or datetime.now(timezone.utc),
+                "title": title,
+                "summary": _node_text(_first(node, summary_sel)) if summary_sel else "",
+                "link": link,
+                "paywalled": False,
+            }
+        )
+    return entries, skipped

buzzrss/migrations.py

@@ -144,3 +144,10 @@ async def upgrade_v12(conn: Connection, scheme: Scheme) -> None:
     # entry_date_idx diende alleen de oude dedup-query.
     await conn.execute("DROP INDEX IF EXISTS entry_date_idx")
     await conn.execute("CREATE INDEX entry_posted_at_idx ON entry (posted_at)")
+
+
+@upgrade_table.register(description="Add HTML scrape config to feeds")
+async def upgrade_v13(conn: Connection) -> None:
+    # scrape_config = JSON met per-feed HTML-selectors voor nieuwspagina's zonder
+    # RSS/Atom/JSON-feed. Leeg = normale feed. Zie buzzrss/html_scrape.py.
+    await conn.execute("ALTER TABLE feed ADD COLUMN scrape_config TEXT NOT NULL DEFAULT ''")

maubot.yaml

@@ -1,5 +1,5 @@
 id: nl.eddydevink.rss
-version: 1.10.0
+version: 1.11.0
 modules:
 - buzzrss
 main_class: RSSBot

validate_filter.py

@@ -92,7 +92,7 @@ UNAMBIGUOUS = [
     "spoorstoring", "spoorlijn", "perron", "bushalte", "bushokje",
     "conducteur", "ovpay", "ns", "prorail",
     "treinstoring", "halte", "openbaar vervoer", "intercity", "buurtbus",
-    "veerdienst", "lijn 12", "ov-chipkaart",
+    "veerdienst", "lijn 12", "ov-chipkaart", "spoorbrug",
 ]
 print(f"== invariant 2: {len(UNAMBIGUOUS)} onmiskenbare OV-termen -> sterk/medium of whitelist ==")
 for w in UNAMBIGUOUS:
@@ -166,6 +166,14 @@ GOLDEN = [
      "De gemeente stopt vanaf januari met de gratis reizen voor 65-plussers.", True),
     # busstation mag niet dubbel meetellen via 'bus' én 'station' (was 2×3=6→nu 1×2)
     ("Metrostation Centraal tijdelijk gesloten", "De metro's rijden niet.", True),
+    # fix 2026-09-12: 'spoorbrug' is medium (niet meer sterk/whitelist). Echte
+    # case: Omroep Brabant Stuifmail (12-09-2026) — een natuurverhaal waarin iemand
+    # 'bij de spoorbrug' een waterschildpad zag; dat passeerde voorheen op één
+    # sterke spoorbrug-hit in de samenvatting.
+    ("Frans weet wat voor warrig beestje Mony heeft gefilmd",
+     "Ruud Klomp zag in de Dommel bij de spoorbrug tussen Liempde en Boxtel een waterschildpad.", False),
+    ("Spoorbrug in Boxtel wordt aangepakt", "De werkzaamheden duren tot het najaar.", False),  # één medium-hit in de titel: 2 < 3
+    ("Spoorbrug wordt verhoogd voor treinen", "De brug gaat omhoog zodat er hogere treinen onderdoor kunnen.", True),  # spoorbrug(2) + trein(1) = 3 → door
     ("NS verlengt noodregeling op traject Zwolle", "Reizigers moeten rekening houden met minder treinen.", True),
     ("Spoorstoring bij Utrecht opgeheven", "ProRail heeft de wisselstoring verholpen.", True),
     ("Nieuwe intercity naar Berlijn", "De trein rijdt vanaf december vier keer per dag.", True),