eddy/buzz-rss
Watch
1
0
Fork
You've already forked buzz-rss
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

v1.8.0: titel-dedup over feeds (zelfde artikel, andere url)

Browse source
This commit is contained in:
Eddy de Vink 2026-09-09 23:39:05 +02:00
commit 6264e73441
7 changed files with 77 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

34
buzzrss/bot.py
View file

@ -16,7 +16,7 @@
from __future__ import annotations
from typing import Any
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from string import Template
from time import mktime, monotonic, time
import asyncio
@ -43,7 +43,7 @@ from mautrix.types import (
from mautrix.util.async_db import UpgradeTable
from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper
from .db import DBManager, Entry, Feed, Subscription
from .db import DBManager, Entry, Feed, Subscription, normalize_title
from .migrations import upgrade_table
rss_change_level = EventType.find("xyz.maubot.rss", t_class=EventType.Class.STATE)
@ -96,6 +96,7 @@ class Config(BaseProxyConfig):
helper.copy("command_prefix")
helper.copy("notification_template")
helper.copy("exclude_paywalled")
helper.copy("title_dedup_days")
helper.copy("run_log_file")
helper.copy("whatsapp_api")
helper.copy("whatsapp_session")
@ -332,10 +333,15 @@ class RSSBot(Plugin):
async def _poll_once(self) -> None:
subs = await self.dbm.get_feeds()
if not subs:
self._log_run_summary(0, 0, 0, 0, 0, 0, 0.0)
self._log_run_summary(0, 0, 0, 0, 0, 0, 0, 0.0)
return
now = int(time())
start = monotonic()
title_dedup_days = int(self.config.get("title_dedup_days", 30))
posted_titles: set[str] = set()
if title_dedup_days > 0:
cutoff = datetime.now(timezone.utc) - timedelta(days=title_dedup_days)
posted_titles = await self.dbm.get_recent_posted_titles(cutoff)
tasks = [self.try_parse_feed(feed=feed) for feed in subs if feed.next_retry < now]
feed: Feed
entries: list[Entry] | None
@ -343,6 +349,7 @@ class RSSBot(Plugin):
evaluated_total = 0
placed_total = 0
filtered_total = 0
deduped_total = 0
failed_total = 0
unchanged_total = 0
for res in asyncio.as_completed(tasks):
@ -383,18 +390,36 @@ class RSSBot(Plugin):
new_entry_list.sort(key=lambda entry: (entry.date, entry.id))
await self.dbm.add_entries(new_entry_list)
for entry in new_entry_list:
norm = normalize_title(entry.title)
if title_dedup_days > 0 and norm and norm in posted_titles:
deduped_total += 1
self.log.info(
f"{entry.id} of feed {feed.id} title already posted ({norm[:60]}), skipping"
)
continue
placed, filtered, failed = await self._broadcast(
feed, entry, feed.subscriptions
)
placed_total += placed
filtered_total += filtered
failed_total += failed
if placed and norm:
# In-memory set: vangt ook duplicaten binnen dezelfde poll-run
# (twee sites van dezelfde uitgever in één run).
posted_titles.add(norm)
try:
await self.dbm.mark_entry_posted(feed.id, entry.id)
except Exception:
self.log.warning(
f"Failed to mark {entry.id} of feed {feed.id} as posted"
)
duration = monotonic() - start
self._log_run_summary(
len(tasks),
evaluated_total,
placed_total,
filtered_total,
deduped_total,
failed_total,
unchanged_total,
duration,
@ -406,13 +431,14 @@ class RSSBot(Plugin):
evaluated: int,
selected: int,
filtered: int,
deduped: int,
failed: int,
unchanged: int,
duration: float,
) -> None:
line = (
f"run complete feeds={feeds} evaluated={evaluated} selected={selected} "
f"filtered={filtered} failed={failed} unchanged={unchanged} "
f"filtered={filtered} deduped={deduped} failed={failed} unchanged={unchanged} "
f"duration={duration:.2f}s"
)
self.log.info(f"Finished polling {feeds} feeds in {duration:.2f} seconds: {line}")