146 lines
6.3 KiB
Python
146 lines
6.3 KiB
Python
# rss - A maubot plugin to subscribe to RSS/Atom feeds.
|
|
# Copyright (C) 2022 Tulir Asokan
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
from mautrix.util.async_db import Connection, Scheme, UpgradeTable
|
|
|
|
upgrade_table = UpgradeTable()
|
|
|
|
|
|
@upgrade_table.register(description="Latest revision", upgrades_to=5)
|
|
async def upgrade_latest(conn: Connection, scheme: Scheme) -> None:
|
|
gen = "GENERATED ALWAYS AS IDENTITY" if scheme != Scheme.SQLITE else ""
|
|
await conn.execute(f"""
|
|
CREATE TABLE IF NOT EXISTS feed (
|
|
id INTEGER {gen},
|
|
url TEXT NOT NULL,
|
|
title TEXT NOT NULL,
|
|
subtitle TEXT NOT NULL,
|
|
link TEXT NOT NULL,
|
|
|
|
next_retry BIGINT DEFAULT 0,
|
|
error_count BIGINT DEFAULT 0,
|
|
|
|
PRIMARY KEY (id),
|
|
UNIQUE (url)
|
|
)
|
|
""")
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS subscription (
|
|
feed_id INTEGER,
|
|
room_id TEXT,
|
|
user_id TEXT NOT NULL,
|
|
|
|
notification_template TEXT,
|
|
send_notice BOOLEAN DEFAULT true,
|
|
title_exclude_filter TEXT DEFAULT '',
|
|
|
|
PRIMARY KEY (feed_id, room_id),
|
|
FOREIGN KEY (feed_id) REFERENCES feed (id)
|
|
)
|
|
""")
|
|
await conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS entry (
|
|
feed_id INTEGER,
|
|
id TEXT,
|
|
date timestamp with time zone NOT NULL,
|
|
title TEXT NOT NULL,
|
|
summary TEXT NOT NULL,
|
|
link TEXT NOT NULL,
|
|
PRIMARY KEY (feed_id, id),
|
|
FOREIGN KEY (feed_id) REFERENCES feed (id)
|
|
)
|
|
""")
|
|
|
|
|
|
@upgrade_table.register(description="Add send_notice field to subscriptions")
|
|
async def upgrade_v2(conn: Connection) -> None:
|
|
await conn.execute("ALTER TABLE subscription ADD COLUMN send_notice BOOLEAN DEFAULT true")
|
|
|
|
|
|
@upgrade_table.register(description="Add error counts to feeds")
|
|
async def upgrade_v3(conn: Connection) -> None:
|
|
await conn.execute("ALTER TABLE feed ADD COLUMN next_retry BIGINT DEFAULT 0")
|
|
await conn.execute("ALTER TABLE feed ADD COLUMN error_count BIGINT DEFAULT 0")
|
|
|
|
|
|
@upgrade_table.register(description="Change entry date to timezone-aware")
|
|
async def upgrade_v4(conn: Connection, scheme: Scheme) -> None:
|
|
if scheme == Scheme.POSTGRES:
|
|
await conn.execute("ALTER TABLE entry ALTER COLUMN date TYPE timestamp with time zone")
|
|
|
|
|
|
@upgrade_table.register(description="Add title exclude filter to subscriptions")
|
|
async def upgrade_v5(conn: Connection) -> None:
|
|
await conn.execute("ALTER TABLE subscription ADD COLUMN title_exclude_filter TEXT DEFAULT ''")
|
|
|
|
|
|
@upgrade_table.register(description="Rename title exclude filter to match filter (whitelist mode)")
|
|
async def upgrade_v6(conn: Connection) -> None:
|
|
await conn.execute("ALTER TABLE subscription RENAME COLUMN title_exclude_filter TO match_filter")
|
|
|
|
|
|
@upgrade_table.register(description="Remove per-subscription filter (single global config filter)")
|
|
async def upgrade_v7(conn: Connection) -> None:
|
|
await conn.execute("ALTER TABLE subscription DROP COLUMN IF EXISTS match_filter")
|
|
|
|
|
|
@upgrade_table.register(description="Add paywalled flag to entries")
|
|
async def upgrade_v8(conn: Connection) -> None:
|
|
await conn.execute("ALTER TABLE entry ADD COLUMN paywalled BOOLEAN NOT NULL DEFAULT false")
|
|
|
|
|
|
@upgrade_table.register(description="Add conditional-fetch headers to feeds")
|
|
async def upgrade_v9(conn: Connection) -> None:
|
|
await conn.execute("ALTER TABLE feed ADD COLUMN etag TEXT DEFAULT ''")
|
|
await conn.execute("ALTER TABLE feed ADD COLUMN last_modified TEXT DEFAULT ''")
|
|
|
|
|
|
@upgrade_table.register(description="Add WhatsApp posted-link dedup table")
|
|
async def upgrade_v10(conn: Connection, scheme: Scheme) -> None:
|
|
default = "now()" if scheme == Scheme.POSTGRES else "CURRENT_TIMESTAMP"
|
|
await conn.execute(
|
|
f"CREATE TABLE IF NOT EXISTS whatsapp_posted ("
|
|
f"link TEXT PRIMARY KEY, "
|
|
f"posted_at {'TIMESTAMP WITH TIME ZONE' if scheme == Scheme.POSTGRES else 'TIMESTAMP'} "
|
|
f"NOT NULL DEFAULT {default}"
|
|
f")"
|
|
)
|
|
|
|
|
|
@upgrade_table.register(description="Add posted flag to entries for cross-feed title dedup")
|
|
async def upgrade_v11(conn: Connection, scheme: Scheme) -> None:
|
|
# posted = deze entry is daadwerkelijk gebroadcast (Matrix en/of WhatsApp).
|
|
# Nieuwe entries die alleen 'gezien' zijn (bv. feed-historis bij subscribe) en
|
|
# filter-afgewezen entries blijven false en blokkeren geen latere titel-match.
|
|
await conn.execute("ALTER TABLE entry ADD COLUMN posted BOOLEAN NOT NULL DEFAULT false")
|
|
# Venster-query voor de titel-dedup: recent gepostede titels ophalen.
|
|
await conn.execute("CREATE INDEX entry_date_idx ON entry (date)")
|
|
|
|
|
|
@upgrade_table.register(description="Measure the title-dedup window on post time, not publication date")
|
|
async def upgrade_v12(conn: Connection, scheme: Scheme) -> None:
|
|
# posted_at = moment waarop de entry gebroadcast is. v11 meette het venster op
|
|
# de publicatiedatum: een backdated item of een na een storing ingehaalde
|
|
# batch viel daardoor buiten het venster en werd alsnog dubbel gepost.
|
|
await conn.execute(
|
|
"ALTER TABLE entry ADD COLUMN posted_at TIMESTAMP WITH TIME ZONE"
|
|
if scheme == Scheme.POSTGRES
|
|
else "ALTER TABLE entry ADD COLUMN posted_at TIMESTAMP"
|
|
)
|
|
# Bestaande gepostede entries: publicatiedatum is de beste benadering.
|
|
await conn.execute("UPDATE entry SET posted_at = date WHERE posted AND posted_at IS NULL")
|
|
# entry_date_idx diende alleen de oude dedup-query.
|
|
await conn.execute("DROP INDEX IF EXISTS entry_date_idx")
|
|
await conn.execute("CREATE INDEX entry_posted_at_idx ON entry (posted_at)")
|