diff --git a/buzzrss/bot.py b/buzzrss/bot.py
index d431a2a..d0d78a3 100644
--- a/buzzrss/bot.py
+++ b/buzzrss/bot.py
@@ -24,6 +24,7 @@ import hashlib
import html
import io
import logging
+import json
import re
import aiohttp
@@ -44,6 +45,8 @@ from mautrix.util.async_db import UpgradeTable
from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper
from .db import DBManager, Entry, Feed, Subscription, normalize_title
+from .html_scrape import extract_entries as _scrape_entries
+from .html_scrape import page_title as _scrape_page_title
from .migrations import upgrade_table
rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STATE)
@@ -78,6 +81,10 @@ rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STAT
# is geen OV-nieuws). Zwartgelijste woorden leveren geen score op, maar
# blokkeren de familie niet: een écht "station"-woord ernaast scoort gewoon.
# Bewust geaccepteerde botsingen: krant "De Metro"/café "De Tram" (medium).
+# 'spoorbrug' is MEDIUM (niet sterk/whitelist): een brug is vaak louter locatie
+# ('bij de spoorbrug tussen Liempde en Boxtel', Stuifmail 2026-09-12) — één
+# spoorbrug-hit in de samenvatting is geen OV-nieuws; écht spoorbrug-nieuws
+# combineert met trein/spoor/verkeer en haalt de drempel alsnog (besluit 2026-09-12).
# Losse algemene woorden zijn zwak (moeten combineren met een andere familie):
# "trein", "sprinter" (atleet), "vervoerder" (logistiek), "reiziger",
# "inchecken"/"uitchecken" (Schiphol/hotel), "station(s)" (omroepen), "bus",
@@ -148,7 +155,7 @@ rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STAT
MATCH_FILTER_FAMILIES: list[tuple[str, int]] = [
# sterk (3)
(r"\b(?:ns|arriva|qbuzz|connexxion|cxx|transdev|keolis|ret|gvb|ebs|htm|syntus|u-ov|allgo|flixbus|blauwnet|r-net|valleilijn|waterbus|eurostar|thalys|randstadrail|gvu|prorail|hermes|rrreis|qliner|ov)\b", 3), # vervoerders ('Breng' staat in de zwakke groep: botst met het werkwoord "brengen")
- (r"\b(?:spoorweg|spoorba(?:an|nen)|spoorlijn|spoorverkeer|spoorwegovergang|spoorbrug|spoorstoring|perron|seinstoring|wisselstoring|spoorstaking)\w*\b", 3), # spoorinfra; let op de klinkerwisseling: 'ontsporing' (open lettergreep, één o) vs 'ontspoord'/'ontspoorde' (dicht, dubbel o) — vandaar beide stammen
+ (r"\b(?:spoorweg|spoorba(?:an|nen)|spoorlijn|spoorverkeer|spoorwegovergang|spoorstoring|perron|seinstoring|wisselstoring|spoorstaking)\w*\b", 3), # spoorinfra; let op de klinkerwisseling: 'ontsporing' (open lettergreep, één o) vs 'ontspoord'/'ontspoorde' (dicht, dubbel o) — vandaar beide stammen. Géén 'spoorbrug': die is medium (vaak locatie-referentie, besluit 2026-09-12)
(r"\b(?:treinstation|treinstations|metrostation|metrostations|busstation|busstations)\b", 2), # station-samenstellingen: onmiskenbaar OV, maar MEDIUM — één station-woord náást een ander onderwerp ('pand naast busstation', besluit 2026-09-11) is geen OV-bericht; in de samenvatting moet een tweede OV-woord bijkomen
(r"(? dict | None:
+ """Parsed scrape_config JSON for HTML-page feeds; None for normal feeds.
+
+ Invalid JSON silently falls back to normal feed parsing instead of
+ taking the feed down; the validation happens when the config is set."""
+ raw = (feed.scrape_config or "").strip()
+ if not raw:
+ return None
+ try:
+ cfg = json.loads(raw)
+ except (ValueError, TypeError):
+ return None
+ return cfg if isinstance(cfg, dict) else None
+
+ async def _parse_html(
+ self, feed: Feed, resp: aiohttp.ClientResponse, cfg: dict
+ ) -> tuple[Feed, list[Entry]]:
+ content = await resp.read()
+ html_text = content.decode("utf-8", errors="replace")
+ raw_entries, skipped = _scrape_entries(html_text, feed.url, cfg)
+ self.log.debug(
+ f"Scraped {feed.url}: {len(raw_entries)} items, {skipped} skipped"
+ )
+ feed.title = _scrape_page_title(html_text) or feed.url
+ feed.subtitle = feed.url
+ feed.link = feed.url
+ entries = [
+ Entry(
+ feed_id=feed.id,
+ id=str(e["id"]),
+ date=e["date"],
+ title=e["title"],
+ summary=e["summary"],
+ link=e["link"],
+ paywalled=e["paywalled"],
+ )
+ for e in raw_entries
+ ]
+ if self.exclude_paywalled:
+ entries = [e for e in entries if not e.paywalled]
+ return feed, entries
+
@classmethod
async def _parse_json(
cls, feed: Feed, resp: aiohttp.ClientResponse
diff --git a/buzzrss/db.py b/buzzrss/db.py
index 728fdf3..def1f29 100644
--- a/buzzrss/db.py
+++ b/buzzrss/db.py
@@ -68,6 +68,7 @@ class Feed:
error_count: int = 0
etag: str = ""
last_modified: str = ""
+ scrape_config: str = ""
subscriptions: list[Subscription] = attr.ib(factory=lambda: [])
@@ -137,7 +138,7 @@ class DBManager:
async def get_feeds(self) -> list[Feed]:
q = """
SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified,
- room_id, user_id, notification_template, send_notice
+ scrape_config, room_id, user_id, notification_template, send_notice
FROM feed INNER JOIN subscription ON feed.id = subscription.feed_id
"""
rows = await self.db.fetch(q)
@@ -153,7 +154,7 @@ class DBManager:
async def get_feeds_by_room(self, room_id: RoomID) -> list[tuple[Feed, UserID]]:
q = """
SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified,
- user_id FROM feed
+ scrape_config, user_id FROM feed
INNER JOIN subscription ON feed.id = subscription.feed_id AND subscription.room_id = $1
"""
rows = await self.db.fetch(q, room_id)
@@ -201,7 +202,7 @@ class DBManager:
await conn.executemany(q, records)
async def get_feed_by_url(self, url: str) -> Feed | None:
- q = "SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified FROM feed WHERE url=$1"
+ q = "SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified, scrape_config FROM feed WHERE url=$1"
return Feed.from_row(await self.db.fetchrow(q, url))
async def get_subscription(
@@ -209,7 +210,7 @@ class DBManager:
) -> tuple[Subscription | None, Feed | None]:
q = """
SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified,
- room_id, user_id, notification_template, send_notice
+ scrape_config, room_id, user_id, notification_template, send_notice
FROM feed LEFT JOIN subscription ON feed.id = subscription.feed_id AND room_id = $2
WHERE feed.id = $1
"""
diff --git a/buzzrss/html_scrape.py b/buzzrss/html_scrape.py
new file mode 100644
index 0000000..93f664b
--- /dev/null
+++ b/buzzrss/html_scrape.py
@@ -0,0 +1,214 @@
+# HTML page scraping for feeds without RSS/Atom/JSON.
+#
+# Used when a feed row carries a `scrape_config` JSON blob. The scrape mode
+# turns a plain server-rendered news list page into entries, so sources that
+# publish no feed at all (vakbonden, gemeenten, OV-bedrijven) can still be
+# followed with the exact same pipeline (OV-filter, title dedup, WhatsApp).
+#
+# Selector syntax is deliberately small: one element per selector, no
+# combinators. Supported: "tag" ".class" "tag.class" "#id" "*"
+# Config keys:
+# item (required) element selector for each news card/item
+# title (required) element selector for the headline inside the item
+# link (required) element selector for the inside the item
+# date (optional) element selector for the date text; the element's
+# datetime/content attributes win over its text
+# date_format (optional) strptime format for the date text. Dutch month
+# names are translated to numbers first, so "%d %B %Y" works
+# for "11 september 2026" regardless of the process locale.
+# summary (optional) element selector for a short text inside the item
+#
+# Items without a resolvable link or a non-empty title are skipped; the caller
+# counts skips for sanity checking.
+
+from __future__ import annotations
+
+import re
+import urllib.parse
+from datetime import datetime, timezone
+from html.parser import HTMLParser
+
+_NL_MONTHS = {
+ "januari": 1, "februari": 2, "maart": 3, "april": 4, "mei": 5, "juni": 6,
+ "juli": 7, "augustus": 8, "september": 9, "oktober": 10, "november": 11,
+ "december": 12,
+}
+
+_WS = re.compile(r"\s+")
+
+
+class _Node:
+ __slots__ = ("tag", "attrs", "children", "text")
+
+ def __init__(self, tag: str, attrs: dict[str, str]) -> None:
+ self.tag = tag
+ self.attrs = attrs
+ self.children: list[_Node] = []
+ self.text = ""
+
+
+class _Builder(HTMLParser):
+ """Turns an HTML document into a small DOM tree (stdlib only)."""
+
+ def __init__(self) -> None:
+ super().__init__(convert_charrefs=True)
+ self.root = _Node("#root", {})
+ self._stack = [self.root]
+
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ node = _Node(tag, {k: (v or "") for k, v in attrs})
+ self._stack[-1].children.append(node)
+ self._stack.append(node)
+
+ def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ node = _Node(tag, {k: (v or "") for k, v in attrs})
+ self._stack[-1].children.append(node)
+
+ def handle_endtag(self, tag: str) -> None:
+ if len(self._stack) > 1:
+ self._stack.pop()
+
+ def handle_data(self, data: str) -> None:
+ self._stack[-1].text += data
+
+
+def _match(node: _Node, sel: str) -> bool:
+ if sel == "*":
+ return True
+ cls = None
+ if sel.startswith("."):
+ tag, cls = None, sel[1:]
+ elif sel.startswith("#"):
+ return node.attrs.get("id", "") == sel[1:]
+ elif "." in sel:
+ tag, cls = sel.split(".", 1)
+ else:
+ tag = sel
+ if tag and node.tag != tag:
+ return False
+ if cls:
+ classes = node.attrs.get("class", "").split()
+ if cls not in classes:
+ return False
+ return True
+
+
+def _iter_tree(node: _Node):
+ for child in node.children:
+ yield child
+ yield from _iter_tree(child)
+
+
+def _node_text(node: _Node) -> str:
+ parts = [node.text]
+
+ def walk(n: _Node) -> None:
+ for c in n.children:
+ parts.append(c.text)
+ walk(c)
+
+ walk(node)
+ return _WS.sub(" ", "".join(parts)).strip()
+
+
+def _parse_date(text: str, fmt: str | None, attrs: dict[str, str]) -> datetime | None:
+ raw = attrs.get("datetime") or attrs.get("content") or ""
+ for candidate in (raw.strip(), text):
+ if not candidate:
+ continue
+ try:
+ return datetime.fromisoformat(candidate).astimezone(timezone.utc)
+ except ValueError:
+ pass
+ if fmt:
+ # Dutch month names -> numbers so %B works under any locale.
+ # The format's %B is translated to %m alongside (the text now
+ # carries the number, not the name).
+ translated = candidate
+ for name, num in _NL_MONTHS.items():
+ translated = re.sub(
+ rf"\b{name}\b", str(num), translated, flags=re.IGNORECASE
+ )
+ translated_fmt = fmt.replace("%B", "%m")
+ try:
+ return datetime.strptime(translated, translated_fmt).replace(
+ tzinfo=timezone.utc
+ )
+ except ValueError:
+ pass
+ return None
+
+
+def _first(node: _Node, sel: str) -> _Node | None:
+ for n in _iter_tree(node):
+ if _match(n, sel):
+ return n
+ return None
+
+
+def _href(node: _Node, sel: str, base: str) -> str:
+ # The item node itself may be the (whole card is a link): match it first.
+ targets = [node]
+ targets.extend(_iter_tree(node))
+ for n in targets:
+ if _match(n, sel):
+ href = (n.attrs.get("href") or "").strip()
+ if href:
+ return urllib.parse.urljoin(base, href)
+ return ""
+
+
+def page_title(html_text: str) -> str:
+ """First text, if any."""
+ m = re.search(r"<title[^>]*>(.*?)", html_text, re.IGNORECASE | re.DOTALL)
+ return _WS.sub(" ", m.group(1)).strip() if m else ""
+
+
+def extract_entries(
+ html_text: str, base_url: str, cfg: dict[str, object]
+) -> tuple[list[dict[str, object]], int]:
+ """Returns (entries, skipped). Each entry dict:
+ {id, date, title, summary, link, paywalled} with date as tz-aware datetime."""
+ item_sel = str(cfg.get("item") or "").strip()
+ title_sel = str(cfg.get("title") or "").strip()
+ link_sel = str(cfg.get("link") or "").strip()
+ if not (item_sel and title_sel and link_sel):
+ raise ValueError("scrape_config requires item, title and link selectors")
+ date_sel = str(cfg.get("date") or "").strip()
+ date_format = str(cfg.get("date_format") or "").strip() or None
+ summary_sel = str(cfg.get("summary") or "").strip()
+
+ builder = _Builder()
+ try:
+ builder.feed(html_text)
+ builder.close()
+ except Exception as e: # malformed HTML must not kill the whole poll
+ raise ValueError(f"HTML parse failed: {e}") from e
+
+ entries: list[dict[str, object]] = []
+ skipped = 0
+ for node in _iter_tree(builder.root):
+ if not _match(node, item_sel):
+ continue
+ link = _href(node, link_sel, base_url)
+ title_node = _first(node, title_sel)
+ title = _node_text(title_node) if title_node else ""
+ if not link or not title:
+ skipped += 1
+ continue
+ date = None
+ if date_sel:
+ date_node = _first(node, date_sel)
+ if date_node:
+ date = _parse_date(_node_text(date_node), date_format, date_node.attrs)
+ entries.append(
+ {
+ "id": link,
+ "date": date or datetime.now(timezone.utc),
+ "title": title,
+ "summary": _node_text(_first(node, summary_sel)) if summary_sel else "",
+ "link": link,
+ "paywalled": False,
+ }
+ )
+ return entries, skipped
\ No newline at end of file
diff --git a/buzzrss/migrations.py b/buzzrss/migrations.py
index 635fcd7..448578f 100644
--- a/buzzrss/migrations.py
+++ b/buzzrss/migrations.py
@@ -144,3 +144,10 @@ async def upgrade_v12(conn: Connection, scheme: Scheme) -> None:
# entry_date_idx diende alleen de oude dedup-query.
await conn.execute("DROP INDEX IF EXISTS entry_date_idx")
await conn.execute("CREATE INDEX entry_posted_at_idx ON entry (posted_at)")
+
+
+@upgrade_table.register(description="Add HTML scrape config to feeds")
+async def upgrade_v13(conn: Connection) -> None:
+ # scrape_config = JSON met per-feed HTML-selectors voor nieuwspagina's zonder
+ # RSS/Atom/JSON-feed. Leeg = normale feed. Zie buzzrss/html_scrape.py.
+ await conn.execute("ALTER TABLE feed ADD COLUMN scrape_config TEXT NOT NULL DEFAULT ''")
diff --git a/maubot.yaml b/maubot.yaml
index e6a42e7..28b5889 100644
--- a/maubot.yaml
+++ b/maubot.yaml
@@ -1,5 +1,5 @@
id: nl.eddydevink.rss
-version: 1.10.0
+version: 1.11.0
modules:
- buzzrss
main_class: RSSBot
diff --git a/validate_filter.py b/validate_filter.py
index 85a7cfb..5637491 100755
--- a/validate_filter.py
+++ b/validate_filter.py
@@ -92,7 +92,7 @@ UNAMBIGUOUS = [
"spoorstoring", "spoorlijn", "perron", "bushalte", "bushokje",
"conducteur", "ovpay", "ns", "prorail",
"treinstoring", "halte", "openbaar vervoer", "intercity", "buurtbus",
- "veerdienst", "lijn 12", "ov-chipkaart",
+ "veerdienst", "lijn 12", "ov-chipkaart", "spoorbrug",
]
print(f"== invariant 2: {len(UNAMBIGUOUS)} onmiskenbare OV-termen -> sterk/medium of whitelist ==")
for w in UNAMBIGUOUS:
@@ -166,6 +166,14 @@ GOLDEN = [
"De gemeente stopt vanaf januari met de gratis reizen voor 65-plussers.", True),
# busstation mag niet dubbel meetellen via 'bus' én 'station' (was 2×3=6→nu 1×2)
("Metrostation Centraal tijdelijk gesloten", "De metro's rijden niet.", True),
+ # fix 2026-09-12: 'spoorbrug' is medium (niet meer sterk/whitelist). Echte
+ # case: Omroep Brabant Stuifmail (12-09-2026) — een natuurverhaal waarin iemand
+ # 'bij de spoorbrug' een waterschildpad zag; dat passeerde voorheen op één
+ # sterke spoorbrug-hit in de samenvatting.
+ ("Frans weet wat voor warrig beestje Mony heeft gefilmd",
+ "Ruud Klomp zag in de Dommel bij de spoorbrug tussen Liempde en Boxtel een waterschildpad.", False),
+ ("Spoorbrug in Boxtel wordt aangepakt", "De werkzaamheden duren tot het najaar.", False), # één medium-hit in de titel: 2 < 3
+ ("Spoorbrug wordt verhoogd voor treinen", "De brug gaat omhoog zodat er hogere treinen onderdoor kunnen.", True), # spoorbrug(2) + trein(1) = 3 → door
("NS verlengt noodregeling op traject Zwolle", "Reizigers moeten rekening houden met minder treinen.", True),
("Spoorstoring bij Utrecht opgeheven", "ProRail heeft de wisselstoring verholpen.", True),
("Nieuwe intercity naar Berlijn", "De trein rijdt vanaf december vier keer per dag.", True),