From 4adfff5b2a019927111d00cf50f499407840eb20 Mon Sep 17 00:00:00 2001 From: Eddy de Vink Date: Sat, 12 Sep 2026 18:59:14 +0200 Subject: [PATCH] =?UTF-8?q?v1.11.0:=20HTML-scrape-modus=20=E2=80=94=20feed?= =?UTF-8?q?s=20zonder=20RSS/Atom/JSON=20als=20pagina=20polled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nieuwe module buzzrss/html_scrape.py (stdlib-only DOM + simpele selectors: tag/.class/tag.class/#id) die een server-rendered nieuwspagina omzet in entries. Per feed via feed.scrape_config (JSON; migratie v13): item/title/ link/date(+date_format, NL-maandnamen locale-onafhankelijk)/summary. Scrape-entries lopen door de bestaande pipeline (OV-filter, titel-dedup, WhatsApp-forward, run-log). Eerste scrape-feed: CNV Stads- en streekvervoer (feed 102). --- buzzrss/bot.py | 51 +++++++++- buzzrss/db.py | 9 +- buzzrss/html_scrape.py | 214 +++++++++++++++++++++++++++++++++++++++++ buzzrss/migrations.py | 7 ++ maubot.yaml | 2 +- 5 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 buzzrss/html_scrape.py diff --git a/buzzrss/bot.py b/buzzrss/bot.py index 3316adb..d0d78a3 100644 --- a/buzzrss/bot.py +++ b/buzzrss/bot.py @@ -24,6 +24,7 @@ import hashlib import html import io import logging +import json import re import aiohttp @@ -44,6 +45,8 @@ from mautrix.util.async_db import UpgradeTable from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper from .db import DBManager, Entry, Feed, Subscription, normalize_title +from .html_scrape import extract_entries as _scrape_entries +from .html_scrape import page_title as _scrape_page_title from .migrations import upgrade_table rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STATE) @@ -787,7 +790,10 @@ class RSSBot(Plugin): etag = resp.headers.get("ETag", "") last_modified = resp.headers.get("Last-Modified", "") ct = resp.headers["Content-Type"].split(";")[0].strip() - if ct == "application/json" or ct == "application/feed+json": + scrape_cfg = self._load_scrape_config(feed) + if scrape_cfg: + feed, entries = await self._parse_html(feed, resp, scrape_cfg) + elif ct == "application/json" or ct == "application/feed+json": feed, entries = await self._parse_json(feed, resp) else: feed, entries = await self._parse_rss(feed, resp) @@ -802,6 +808,49 @@ class RSSBot(Plugin): self.log.debug(f"Failed to store fetch headers of {feed.id}") return feed, entries + @staticmethod + def _load_scrape_config(feed: Feed) -> dict | None: + """Parsed scrape_config JSON for HTML-page feeds; None for normal feeds. + + Invalid JSON silently falls back to normal feed parsing instead of + taking the feed down; the validation happens when the config is set.""" + raw = (feed.scrape_config or "").strip() + if not raw: + return None + try: + cfg = json.loads(raw) + except (ValueError, TypeError): + return None + return cfg if isinstance(cfg, dict) else None + + async def _parse_html( + self, feed: Feed, resp: aiohttp.ClientResponse, cfg: dict + ) -> tuple[Feed, list[Entry]]: + content = await resp.read() + html_text = content.decode("utf-8", errors="replace") + raw_entries, skipped = _scrape_entries(html_text, feed.url, cfg) + self.log.debug( + f"Scraped {feed.url}: {len(raw_entries)} items, {skipped} skipped" + ) + feed.title = _scrape_page_title(html_text) or feed.url + feed.subtitle = feed.url + feed.link = feed.url + entries = [ + Entry( + feed_id=feed.id, + id=str(e["id"]), + date=e["date"], + title=e["title"], + summary=e["summary"], + link=e["link"], + paywalled=e["paywalled"], + ) + for e in raw_entries + ] + if self.exclude_paywalled: + entries = [e for e in entries if not e.paywalled] + return feed, entries + @classmethod async def _parse_json( cls, feed: Feed, resp: aiohttp.ClientResponse diff --git a/buzzrss/db.py b/buzzrss/db.py index 728fdf3..def1f29 100644 --- a/buzzrss/db.py +++ b/buzzrss/db.py @@ -68,6 +68,7 @@ class Feed: error_count: int = 0 etag: str = "" last_modified: str = "" + scrape_config: str = "" subscriptions: list[Subscription] = attr.ib(factory=lambda: []) @@ -137,7 +138,7 @@ class DBManager: async def get_feeds(self) -> list[Feed]: q = """ SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified, - room_id, user_id, notification_template, send_notice + scrape_config, room_id, user_id, notification_template, send_notice FROM feed INNER JOIN subscription ON feed.id = subscription.feed_id """ rows = await self.db.fetch(q) @@ -153,7 +154,7 @@ class DBManager: async def get_feeds_by_room(self, room_id: RoomID) -> list[tuple[Feed, UserID]]: q = """ SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified, - user_id FROM feed + scrape_config, user_id FROM feed INNER JOIN subscription ON feed.id = subscription.feed_id AND subscription.room_id = $1 """ rows = await self.db.fetch(q, room_id) @@ -201,7 +202,7 @@ class DBManager: await conn.executemany(q, records) async def get_feed_by_url(self, url: str) -> Feed | None: - q = "SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified FROM feed WHERE url=$1" + q = "SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified, scrape_config FROM feed WHERE url=$1" return Feed.from_row(await self.db.fetchrow(q, url)) async def get_subscription( @@ -209,7 +210,7 @@ class DBManager: ) -> tuple[Subscription | None, Feed | None]: q = """ SELECT id, url, title, subtitle, link, next_retry, error_count, etag, last_modified, - room_id, user_id, notification_template, send_notice + scrape_config, room_id, user_id, notification_template, send_notice FROM feed LEFT JOIN subscription ON feed.id = subscription.feed_id AND room_id = $2 WHERE feed.id = $1 """ diff --git a/buzzrss/html_scrape.py b/buzzrss/html_scrape.py new file mode 100644 index 0000000..93f664b --- /dev/null +++ b/buzzrss/html_scrape.py @@ -0,0 +1,214 @@ +# HTML page scraping for feeds without RSS/Atom/JSON. +# +# Used when a feed row carries a `scrape_config` JSON blob. The scrape mode +# turns a plain server-rendered news list page into entries, so sources that +# publish no feed at all (vakbonden, gemeenten, OV-bedrijven) can still be +# followed with the exact same pipeline (OV-filter, title dedup, WhatsApp). +# +# Selector syntax is deliberately small: one element per selector, no +# combinators. Supported: "tag" ".class" "tag.class" "#id" "*" +# Config keys: +# item (required) element selector for each news card/item +# title (required) element selector for the headline inside the item +# link (required) element selector for the inside the item +# date (optional) element selector for the date text; the element's +# datetime/content attributes win over its text +# date_format (optional) strptime format for the date text. Dutch month +# names are translated to numbers first, so "%d %B %Y" works +# for "11 september 2026" regardless of the process locale. +# summary (optional) element selector for a short text inside the item +# +# Items without a resolvable link or a non-empty title are skipped; the caller +# counts skips for sanity checking. + +from __future__ import annotations + +import re +import urllib.parse +from datetime import datetime, timezone +from html.parser import HTMLParser + +_NL_MONTHS = { + "januari": 1, "februari": 2, "maart": 3, "april": 4, "mei": 5, "juni": 6, + "juli": 7, "augustus": 8, "september": 9, "oktober": 10, "november": 11, + "december": 12, +} + +_WS = re.compile(r"\s+") + + +class _Node: + __slots__ = ("tag", "attrs", "children", "text") + + def __init__(self, tag: str, attrs: dict[str, str]) -> None: + self.tag = tag + self.attrs = attrs + self.children: list[_Node] = [] + self.text = "" + + +class _Builder(HTMLParser): + """Turns an HTML document into a small DOM tree (stdlib only).""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.root = _Node("#root", {}) + self._stack = [self.root] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + node = _Node(tag, {k: (v or "") for k, v in attrs}) + self._stack[-1].children.append(node) + self._stack.append(node) + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + node = _Node(tag, {k: (v or "") for k, v in attrs}) + self._stack[-1].children.append(node) + + def handle_endtag(self, tag: str) -> None: + if len(self._stack) > 1: + self._stack.pop() + + def handle_data(self, data: str) -> None: + self._stack[-1].text += data + + +def _match(node: _Node, sel: str) -> bool: + if sel == "*": + return True + cls = None + if sel.startswith("."): + tag, cls = None, sel[1:] + elif sel.startswith("#"): + return node.attrs.get("id", "") == sel[1:] + elif "." in sel: + tag, cls = sel.split(".", 1) + else: + tag = sel + if tag and node.tag != tag: + return False + if cls: + classes = node.attrs.get("class", "").split() + if cls not in classes: + return False + return True + + +def _iter_tree(node: _Node): + for child in node.children: + yield child + yield from _iter_tree(child) + + +def _node_text(node: _Node) -> str: + parts = [node.text] + + def walk(n: _Node) -> None: + for c in n.children: + parts.append(c.text) + walk(c) + + walk(node) + return _WS.sub(" ", "".join(parts)).strip() + + +def _parse_date(text: str, fmt: str | None, attrs: dict[str, str]) -> datetime | None: + raw = attrs.get("datetime") or attrs.get("content") or "" + for candidate in (raw.strip(), text): + if not candidate: + continue + try: + return datetime.fromisoformat(candidate).astimezone(timezone.utc) + except ValueError: + pass + if fmt: + # Dutch month names -> numbers so %B works under any locale. + # The format's %B is translated to %m alongside (the text now + # carries the number, not the name). + translated = candidate + for name, num in _NL_MONTHS.items(): + translated = re.sub( + rf"\b{name}\b", str(num), translated, flags=re.IGNORECASE + ) + translated_fmt = fmt.replace("%B", "%m") + try: + return datetime.strptime(translated, translated_fmt).replace( + tzinfo=timezone.utc + ) + except ValueError: + pass + return None + + +def _first(node: _Node, sel: str) -> _Node | None: + for n in _iter_tree(node): + if _match(n, sel): + return n + return None + + +def _href(node: _Node, sel: str, base: str) -> str: + # The item node itself may be the (whole card is a link): match it first. + targets = [node] + targets.extend(_iter_tree(node)) + for n in targets: + if _match(n, sel): + href = (n.attrs.get("href") or "").strip() + if href: + return urllib.parse.urljoin(base, href) + return "" + + +def page_title(html_text: str) -> str: + """First text, if any.""" + m = re.search(r"<title[^>]*>(.*?)", html_text, re.IGNORECASE | re.DOTALL) + return _WS.sub(" ", m.group(1)).strip() if m else "" + + +def extract_entries( + html_text: str, base_url: str, cfg: dict[str, object] +) -> tuple[list[dict[str, object]], int]: + """Returns (entries, skipped). Each entry dict: + {id, date, title, summary, link, paywalled} with date as tz-aware datetime.""" + item_sel = str(cfg.get("item") or "").strip() + title_sel = str(cfg.get("title") or "").strip() + link_sel = str(cfg.get("link") or "").strip() + if not (item_sel and title_sel and link_sel): + raise ValueError("scrape_config requires item, title and link selectors") + date_sel = str(cfg.get("date") or "").strip() + date_format = str(cfg.get("date_format") or "").strip() or None + summary_sel = str(cfg.get("summary") or "").strip() + + builder = _Builder() + try: + builder.feed(html_text) + builder.close() + except Exception as e: # malformed HTML must not kill the whole poll + raise ValueError(f"HTML parse failed: {e}") from e + + entries: list[dict[str, object]] = [] + skipped = 0 + for node in _iter_tree(builder.root): + if not _match(node, item_sel): + continue + link = _href(node, link_sel, base_url) + title_node = _first(node, title_sel) + title = _node_text(title_node) if title_node else "" + if not link or not title: + skipped += 1 + continue + date = None + if date_sel: + date_node = _first(node, date_sel) + if date_node: + date = _parse_date(_node_text(date_node), date_format, date_node.attrs) + entries.append( + { + "id": link, + "date": date or datetime.now(timezone.utc), + "title": title, + "summary": _node_text(_first(node, summary_sel)) if summary_sel else "", + "link": link, + "paywalled": False, + } + ) + return entries, skipped \ No newline at end of file diff --git a/buzzrss/migrations.py b/buzzrss/migrations.py index 635fcd7..448578f 100644 --- a/buzzrss/migrations.py +++ b/buzzrss/migrations.py @@ -144,3 +144,10 @@ async def upgrade_v12(conn: Connection, scheme: Scheme) -> None: # entry_date_idx diende alleen de oude dedup-query. await conn.execute("DROP INDEX IF EXISTS entry_date_idx") await conn.execute("CREATE INDEX entry_posted_at_idx ON entry (posted_at)") + + +@upgrade_table.register(description="Add HTML scrape config to feeds") +async def upgrade_v13(conn: Connection) -> None: + # scrape_config = JSON met per-feed HTML-selectors voor nieuwspagina's zonder + # RSS/Atom/JSON-feed. Leeg = normale feed. Zie buzzrss/html_scrape.py. + await conn.execute("ALTER TABLE feed ADD COLUMN scrape_config TEXT NOT NULL DEFAULT ''") diff --git a/maubot.yaml b/maubot.yaml index d244909..28b5889 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,5 +1,5 @@ id: nl.eddydevink.rss -version: 1.10.1 +version: 1.11.0 modules: - buzzrss main_class: RSSBot