diff --git a/buzzrss/bot.py b/buzzrss/bot.py
index 3316adb..d0d78a3 100644
--- a/buzzrss/bot.py
+++ b/buzzrss/bot.py
@@ -24,6 +24,7 @@ import hashlib
import html
import io
import logging
+import json
import re
import aiohttp
@@ -44,6 +45,8 @@ from mautrix.util.async_db import UpgradeTable
from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper
from .db import DBManager, Entry, Feed, Subscription, normalize_title
+from .html_scrape import extract_entries as _scrape_entries
+from .html_scrape import page_title as _scrape_page_title
from .migrations import upgrade_table
rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STATE)
@@ -787,7 +790,10 @@ class RSSBot(Plugin):
etag = resp.headers.get("ETag", "")
last_modified = resp.headers.get("Last-Modified", "")
ct = resp.headers["Content-Type"].split(";")[0].strip()
- if ct == "application/json" or ct == "application/feed+json":
+ scrape_cfg = self._load_scrape_config(feed)
+ if scrape_cfg:
+ feed, entries = await self._parse_html(feed, resp, scrape_cfg)
+ elif ct == "application/json" or ct == "application/feed+json":
feed, entries = await self._parse_json(feed, resp)
else:
feed, entries = await self._parse_rss(feed, resp)
@@ -802,6 +808,49 @@ class RSSBot(Plugin):
self.log.debug(f"Failed to store fetch headers of {feed.id}")
return feed, entries
+ @staticmethod
+ def _load_scrape_config(feed: Feed) -> dict | None:
+ """Parsed scrape_config JSON for HTML-page feeds; None for normal feeds.
+
+ Invalid JSON silently falls back to normal feed parsing instead of
+ taking the feed down; the validation happens when the config is set."""
+ raw = (feed.scrape_config or "").strip()
+ if not raw:
+ return None
+ try:
+ cfg = json.loads(raw)
+ except (ValueError, TypeError):
+ return None
+ return cfg if isinstance(cfg, dict) else None
+
+ async def _parse_html(
+ self, feed: Feed, resp: aiohttp.ClientResponse, cfg: dict
+ ) -> tuple[Feed, list[Entry]]:
+ content = await resp.read()
+ html_text = content.decode("utf-8", errors="replace")
+ raw_entries, skipped = _scrape_entries(html_text, feed.url, cfg)
+ self.log.debug(
+ f"Scraped {feed.url}: {len(raw_entries)} items, {skipped} skipped"
+ )
+ feed.title = _scrape_page_title(html_text) or feed.url
+ feed.subtitle = feed.url
+ feed.link = feed.url
+ entries = [
+ Entry(
+ feed_id=feed.id,
+ id=str(e["id"]),
+ date=e["date"],
+ title=e["title"],
+ summary=e["summary"],
+ link=e["link"],
+ paywalled=e["paywalled"],
+ )
+ for e in raw_entries
+ ]
+ if self.exclude_paywalled:
+ entries = [e for e in entries if not e.paywalled]
+ return feed, entries
+
@classmethod
async def _parse_json(
cls, feed: Feed, resp: aiohttp.ClientResponse
diff --git a/buzzrss/db.py b/buzzrss/db.py
index 728fdf3..def1f29 100644
--- a/buzzrss/db.py
+++ b/buzzrss/db.py
@@ -68,6 +68,7 @@ class Feed:
error_count: int = 0
etag: str = ""
last_modified: str = ""
+ scrape_config: str = ""
subscriptions: list[Subscription] = attr.ib(factory=lambda: [])
@@ -137,7 +138,7 @@ class DBManager:
async def get_feeds(self) -> list[Feed]:
q = """
SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified,
- room_id, user_id, notification_template, send_notice
+ scrape_config, room_id, user_id, notification_template, send_notice
FROM feed INNER JOIN subscription ON feed.id = subscription.feed_id
"""
rows = await self.db.fetch(q)
@@ -153,7 +154,7 @@ class DBManager:
async def get_feeds_by_room(self, room_id: RoomID) -> list[tuple[Feed, UserID]]:
q = """
SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified,
- user_id FROM feed
+ scrape_config, user_id FROM feed
INNER JOIN subscription ON feed.id = subscription.feed_id AND subscription.room_id = $1
"""
rows = await self.db.fetch(q, room_id)
@@ -201,7 +202,7 @@ class DBManager:
await conn.executemany(q, records)
async def get_feed_by_url(self, url: str) -> Feed | None:
- q = "SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified FROM feed WHERE url=$1"
+ q = "SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified, scrape_config FROM feed WHERE url=$1"
return Feed.from_row(await self.db.fetchrow(q, url))
async def get_subscription(
@@ -209,7 +210,7 @@ class DBManager:
) -> tuple[Subscription | None, Feed | None]:
q = """
SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified,
- room_id, user_id, notification_template, send_notice
+ scrape_config, room_id, user_id, notification_template, send_notice
FROM feed LEFT JOIN subscription ON feed.id = subscription.feed_id AND room_id = $2
WHERE feed.id = $1
"""
diff --git a/buzzrss/html_scrape.py b/buzzrss/html_scrape.py
new file mode 100644
index 0000000..93f664b
--- /dev/null
+++ b/buzzrss/html_scrape.py
@@ -0,0 +1,214 @@
+# HTML page scraping for feeds without RSS/Atom/JSON.
+#
+# Used when a feed row carries a `scrape_config` JSON blob. The scrape mode
+# turns a plain server-rendered news list page into entries, so sources that
+# publish no feed at all (vakbonden, gemeenten, OV-bedrijven) can still be
+# followed with the exact same pipeline (OV-filter, title dedup, WhatsApp).
+#
+# Selector syntax is deliberately small: one element per selector, no
+# combinators. Supported: "tag" ".class" "tag.class" "#id" "*"
+# Config keys:
+# item (required) element selector for each news card/item
+# title (required) element selector for the headline inside the item
+# link (required) element selector for the inside the item
+# date (optional) element selector for the date text; the element's
+# datetime/content attributes win over its text
+# date_format (optional) strptime format for the date text. Dutch month
+# names are translated to numbers first, so "%d %B %Y" works
+# for "11 september 2026" regardless of the process locale.
+# summary (optional) element selector for a short text inside the item
+#
+# Items without a resolvable link or a non-empty title are skipped; the caller
+# counts skips for sanity checking.
+
+from __future__ import annotations
+
+import re
+import urllib.parse
+from datetime import datetime, timezone
+from html.parser import HTMLParser
+
+_NL_MONTHS = {
+ "januari": 1, "februari": 2, "maart": 3, "april": 4, "mei": 5, "juni": 6,
+ "juli": 7, "augustus": 8, "september": 9, "oktober": 10, "november": 11,
+ "december": 12,
+}
+
+_WS = re.compile(r"\s+")
+
+
+class _Node:
+ __slots__ = ("tag", "attrs", "children", "text")
+
+ def __init__(self, tag: str, attrs: dict[str, str]) -> None:
+ self.tag = tag
+ self.attrs = attrs
+ self.children: list[_Node] = []
+ self.text = ""
+
+
+class _Builder(HTMLParser):
+ """Turns an HTML document into a small DOM tree (stdlib only)."""
+
+ def __init__(self) -> None:
+ super().__init__(convert_charrefs=True)
+ self.root = _Node("#root", {})
+ self._stack = [self.root]
+
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ node = _Node(tag, {k: (v or "") for k, v in attrs})
+ self._stack[-1].children.append(node)
+ self._stack.append(node)
+
+ def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+ node = _Node(tag, {k: (v or "") for k, v in attrs})
+ self._stack[-1].children.append(node)
+
+ def handle_endtag(self, tag: str) -> None:
+ if len(self._stack) > 1:
+ self._stack.pop()
+
+ def handle_data(self, data: str) -> None:
+ self._stack[-1].text += data
+
+
+def _match(node: _Node, sel: str) -> bool:
+ if sel == "*":
+ return True
+ cls = None
+ if sel.startswith("."):
+ tag, cls = None, sel[1:]
+ elif sel.startswith("#"):
+ return node.attrs.get("id", "") == sel[1:]
+ elif "." in sel:
+ tag, cls = sel.split(".", 1)
+ else:
+ tag = sel
+ if tag and node.tag != tag:
+ return False
+ if cls:
+ classes = node.attrs.get("class", "").split()
+ if cls not in classes:
+ return False
+ return True
+
+
+def _iter_tree(node: _Node):
+ for child in node.children:
+ yield child
+ yield from _iter_tree(child)
+
+
+def _node_text(node: _Node) -> str:
+ parts = [node.text]
+
+ def walk(n: _Node) -> None:
+ for c in n.children:
+ parts.append(c.text)
+ walk(c)
+
+ walk(node)
+ return _WS.sub(" ", "".join(parts)).strip()
+
+
+def _parse_date(text: str, fmt: str | None, attrs: dict[str, str]) -> datetime | None:
+ raw = attrs.get("datetime") or attrs.get("content") or ""
+ for candidate in (raw.strip(), text):
+ if not candidate:
+ continue
+ try:
+ return datetime.fromisoformat(candidate).astimezone(timezone.utc)
+ except ValueError:
+ pass
+ if fmt:
+ # Dutch month names -> numbers so %B works under any locale.
+ # The format's %B is translated to %m alongside (the text now
+ # carries the number, not the name).
+ translated = candidate
+ for name, num in _NL_MONTHS.items():
+ translated = re.sub(
+ rf"\b{name}\b", str(num), translated, flags=re.IGNORECASE
+ )
+ translated_fmt = fmt.replace("%B", "%m")
+ try:
+ return datetime.strptime(translated, translated_fmt).replace(
+ tzinfo=timezone.utc
+ )
+ except ValueError:
+ pass
+ return None
+
+
+def _first(node: _Node, sel: str) -> _Node | None:
+ for n in _iter_tree(node):
+ if _match(n, sel):
+ return n
+ return None
+
+
+def _href(node: _Node, sel: str, base: str) -> str:
+ # The item node itself may be the (whole card is a link): match it first.
+ targets = [node]
+ targets.extend(_iter_tree(node))
+ for n in targets:
+ if _match(n, sel):
+ href = (n.attrs.get("href") or "").strip()
+ if href:
+ return urllib.parse.urljoin(base, href)
+ return ""
+
+
+def page_title(html_text: str) -> str:
+ """First text, if any."""
+ m = re.search(r"]*>(.*?)", html_text, re.IGNORECASE | re.DOTALL)
+ return _WS.sub(" ", m.group(1)).strip() if m else ""
+
+
+def extract_entries(
+ html_text: str, base_url: str, cfg: dict[str, object]
+) -> tuple[list[dict[str, object]], int]:
+ """Returns (entries, skipped). Each entry dict:
+ {id, date, title, summary, link, paywalled} with date as tz-aware datetime."""
+ item_sel = str(cfg.get("item") or "").strip()
+ title_sel = str(cfg.get("title") or "").strip()
+ link_sel = str(cfg.get("link") or "").strip()
+ if not (item_sel and title_sel and link_sel):
+ raise ValueError("scrape_config requires item, title and link selectors")
+ date_sel = str(cfg.get("date") or "").strip()
+ date_format = str(cfg.get("date_format") or "").strip() or None
+ summary_sel = str(cfg.get("summary") or "").strip()
+
+ builder = _Builder()
+ try:
+ builder.feed(html_text)
+ builder.close()
+ except Exception as e: # malformed HTML must not kill the whole poll
+ raise ValueError(f"HTML parse failed: {e}") from e
+
+ entries: list[dict[str, object]] = []
+ skipped = 0
+ for node in _iter_tree(builder.root):
+ if not _match(node, item_sel):
+ continue
+ link = _href(node, link_sel, base_url)
+ title_node = _first(node, title_sel)
+ title = _node_text(title_node) if title_node else ""
+ if not link or not title:
+ skipped += 1
+ continue
+ date = None
+ if date_sel:
+ date_node = _first(node, date_sel)
+ if date_node:
+ date = _parse_date(_node_text(date_node), date_format, date_node.attrs)
+ entries.append(
+ {
+ "id": link,
+ "date": date or datetime.now(timezone.utc),
+ "title": title,
+ "summary": _node_text(_first(node, summary_sel)) if summary_sel else "",
+ "link": link,
+ "paywalled": False,
+ }
+ )
+ return entries, skipped
\ No newline at end of file
diff --git a/buzzrss/migrations.py b/buzzrss/migrations.py
index 635fcd7..448578f 100644
--- a/buzzrss/migrations.py
+++ b/buzzrss/migrations.py
@@ -144,3 +144,10 @@ async def upgrade_v12(conn: Connection, scheme: Scheme) -> None:
# entry_date_idx diende alleen de oude dedup-query.
await conn.execute("DROP INDEX IF EXISTS entry_date_idx")
await conn.execute("CREATE INDEX entry_posted_at_idx ON entry (posted_at)")
+
+
+@upgrade_table.register(description="Add HTML scrape config to feeds")
+async def upgrade_v13(conn: Connection) -> None:
+ # scrape_config = JSON met per-feed HTML-selectors voor nieuwspagina's zonder
+ # RSS/Atom/JSON-feed. Leeg = normale feed. Zie buzzrss/html_scrape.py.
+ await conn.execute("ALTER TABLE feed ADD COLUMN scrape_config TEXT NOT NULL DEFAULT ''")
diff --git a/maubot.yaml b/maubot.yaml
index d244909..28b5889 100644
--- a/maubot.yaml
+++ b/maubot.yaml
@@ -1,5 +1,5 @@
id: nl.eddydevink.rss
-version: 1.10.1
+version: 1.11.0
modules:
- buzzrss
main_class: RSSBot