From 35f2fe63df4473e2aa8e8f82dd3f95a1ef7fc413 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 30 Apr 2022 21:14:19 +0300 Subject: [PATCH 01/17] Add support for old SQLites Closes #31 --- rss/db.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/rss/db.py b/rss/db.py index 69c930c..c7cea64 100644 --- a/rss/db.py +++ b/rss/db.py @@ -23,7 +23,7 @@ from attr import dataclass import attr from mautrix.types import RoomID, UserID -from mautrix.util.async_db import Database, Scheme +from mautrix.util.async_db import Database, Scheme, SQLiteCursor @dataclass @@ -182,9 +182,23 @@ class DBManager: "INSERT INTO feed (url, title, subtitle, link, next_retry) " "VALUES ($1, $2, $3, $4, $5) RETURNING (id)" ) - info.id = await self.db.fetchval( - q, info.url, info.title, info.subtitle, info.link, info.next_retry - ) + # SQLite only gained RETURNING support in v3.35 (2021-03-12) + # TODO remove this special case in a couple of years + if self.db.scheme == Scheme.SQLITE: + cur = await self.db.execute( + q.replace(" RETURNING (id)", ""), + info.url, + info.title, + info.subtitle, + info.link, + info.next_retry, + ) + assert isinstance(cur, SQLiteCursor) + info.id = cur.lastrowid + else: + info.id = await self.db.fetchval( + q, info.url, info.title, info.subtitle, info.link, info.next_retry + ) return info async def set_backoff(self, info: Feed, error_count: int, next_retry: int) -> None: From 70eb6efed52fb5e5a9a8e25398d16b219cf39883 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 30 Apr 2022 21:14:28 +0300 Subject: [PATCH 02/17] Fix Python 3.10 compatibility --- rss/bot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rss/bot.py b/rss/bot.py index f80de7e..d6048c5 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -90,7 +90,7 @@ class RSSBot(Plugin): self.dbm = DBManager(self.database) self.http = self.client.api.session self.power_level_cache = {} - self.poll_task = asyncio.ensure_future(self.poll_feeds(), loop=self.loop) + self.poll_task = asyncio.create_task(self.poll_feeds()) async def stop(self) -> None: await super().stop() @@ -131,7 +131,7 @@ class RSSBot(Plugin): if spam_sleep >= 0: for task in tasks: await task - await asyncio.sleep(spam_sleep, loop=self.loop) + await asyncio.sleep(spam_sleep) else: await asyncio.gather(*tasks) @@ -185,7 +185,7 @@ class RSSBot(Plugin): self.log.debug("Polling stopped") except Exception: self.log.exception("Error while polling feeds") - await asyncio.sleep(self.config["update_interval"] * 60, loop=self.loop) + await asyncio.sleep(self.config["update_interval"] * 60) async def try_parse_feed(self, feed: Feed | None = None) -> tuple[Feed, list[Entry]]: try: From fa34d80c4f131a1ecf6108e80c3623a4eac57ea2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 30 Apr 2022 21:14:35 +0300 Subject: [PATCH 03/17] Update and unpin black --- .github/workflows/python-lint.yml | 1 - .pre-commit-config.yaml | 13 +++++-------- pyproject.toml | 1 - 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 7deb4c8..fc28bdb 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -16,7 +16,6 @@ jobs: - uses: psf/black@stable with: src: "./rss" - version: "22.1.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a205c9..7f1b3e5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,17 +7,14 @@ repos: - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files - # TODO convert to use the upstream psf/black when - # https://github.com/psf/black/issues/2493 gets fixed - - repo: local + - repo: https://github.com/psf/black + rev: 22.3.0 hooks: - id: black - name: black - entry: black --check - language: system - files: ^rss/.*\.py$ + language_version: python3 + files: ^rss/.*\.pyi?$ - repo: https://github.com/PyCQA/isort rev: 5.10.1 hooks: - id: isort - files: ^rss/.*$ + files: ^rss/.*\.pyi?$ diff --git a/pyproject.toml b/pyproject.toml index be9cdda..3e608c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,4 +9,3 @@ line_length = 99 [tool.black] line-length = 99 target-version = ["py38"] -required-version = "22.1.0" From e87f332e0e243a04d6f8addd71ca14f1d40d40d7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 30 Apr 2022 21:24:02 +0300 Subject: [PATCH 04/17] Don't break on old mautrix-python versions --- rss/db.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/rss/db.py b/rss/db.py index c7cea64..e6faa88 100644 --- a/rss/db.py +++ b/rss/db.py @@ -23,7 +23,13 @@ from attr import dataclass import attr from mautrix.types import RoomID, UserID -from mautrix.util.async_db import Database, Scheme, SQLiteCursor +from mautrix.util.async_db import Database, Scheme + +# TODO make this import unconditional after updating mautrix-python +try: + from mautrix.util.async_db import SQLiteCursor +except ImportError: + SQLiteCursor = None @dataclass @@ -193,7 +199,8 @@ class DBManager: info.link, info.next_retry, ) - assert isinstance(cur, SQLiteCursor) + if SQLiteCursor is not None: + assert isinstance(cur, SQLiteCursor) info.id = cur.lastrowid else: info.id = await self.db.fetchval( From e7af4d2657b33402c1fe4ca7f388aa0f83a8affb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 2 May 2022 10:29:41 +0300 Subject: [PATCH 05/17] Bump version to 0.3.1 --- maubot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot.yaml b/maubot.yaml index f17f5e4..38fba7d 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,6 +1,6 @@ maubot: 0.3.0 id: xyz.maubot.rss -version: 0.3.0 +version: 0.3.1 license: AGPL-3.0-or-later modules: - rss From 877dcffb9c695ad0320fc6f5079f626d8bae452b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 18 Jun 2022 17:47:03 +0300 Subject: [PATCH 06/17] Use custom user agent --- rss/bot.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/rss/bot.py b/rss/bot.py index d6048c5..0352f4c 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -26,7 +26,7 @@ import aiohttp import attr import feedparser -from maubot import MessageEvent, Plugin +from maubot import MessageEvent, Plugin, __version__ as maubot_version from maubot.handlers import command, event from mautrix.types import ( EventID, @@ -198,6 +198,12 @@ class RSSBot(Plugin): self.log.warning(f"Failed to parse feed {feed.id} / {feed.url}: {e}") return feed, [] + @property + def _feed_get_headers(self) -> dict[str, str]: + return { + "User-Agent": f"maubot/{maubot_version} +https://github.com/maubot/rss", + } + async def parse_feed( self, *, feed: Feed | None = None, url: str | None = None ) -> tuple[Feed, list[Entry]]: @@ -207,7 +213,7 @@ class RSSBot(Plugin): feed = Feed(id=-1, url=url, title="", subtitle="", link="") elif url is not None: raise ValueError("Only one of feed or url must be set") - resp = await self.http.get(feed.url) + resp = await self.http.get(feed.url, headers=self._feed_get_headers) ct = resp.headers["Content-Type"].split(";")[0].strip() if ct == "application/json" or ct == "application/feed+json": return await self._parse_json(feed, resp) From 30ad459870470dec9986675278c70a66d7267952 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 19 Jun 2022 14:27:42 +0300 Subject: [PATCH 07/17] Move CI script to main maubot repo --- .gitlab-ci.yml | 32 +++----------------------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 45ef06b..7c690ef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,29 +1,3 @@ -image: dock.mau.dev/maubot/maubot - -stages: -- build - -variables: - PYTHONPATH: /opt/maubot - -build: - stage: build - except: - - tags - script: - - python3 -m maubot.cli build -o xyz.maubot.$CI_PROJECT_NAME-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA.mbp - artifacts: - paths: - - "*.mbp" - expire_in: 365 days - -build tags: - stage: build - only: - - tags - script: - - python3 -m maubot.cli build -o xyz.maubot.$CI_PROJECT_NAME-$CI_COMMIT_TAG.mbp - artifacts: - paths: - - "*.mbp" - expire_in: never +include: +- project: 'maubot/maubot' + file: '/.gitlab-ci-plugin.yml' From 03bb1280050e6fcb09405c8848cb58ec7807f8d9 Mon Sep 17 00:00:00 2001 From: Andrew Kvalheim Date: Thu, 14 Jul 2022 09:41:33 -0700 Subject: [PATCH 08/17] Key entries by link if missing ID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the problem of incorrectly duplicated entries in feeds that update content but don’t explicitly provide entry IDs. Example feed: - https://www.to-rss.xyz/wikipedia/current_events/ Example entry: Current events: 2022-07-13 https://en.wikipedia.org/wiki/Portal:Current_events/2022_July_13 [VARIABLE CONTENT] Wed, 13 Jul 2022 00:00:00 -0000 This behavior is suggested by the common practice of using an entry’s link as its ID value, and is consistent with typical feed aggregators such as Feedbin and Inoreader. --- rss/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rss/bot.py b/rss/bot.py index 0352f4c..42ca897 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -279,12 +279,12 @@ class RSSBot(Plugin): feed_id=feed_id, id=( getattr(entry, "id", None) + or getattr(entry, "link", None) or hashlib.sha1( " ".join( [ getattr(entry, "title", ""), getattr(entry, "description", ""), - getattr(entry, "link", ""), ] ).encode("utf-8") ).hexdigest() From f12d32ad3ccd3a4289df3860cbf65811420b42bf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 3 Oct 2022 09:25:35 +0300 Subject: [PATCH 09/17] Bump version to 0.3.2 --- maubot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot.yaml b/maubot.yaml index 38fba7d..4cefc3d 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,6 +1,6 @@ maubot: 0.3.0 id: xyz.maubot.rss -version: 0.3.1 +version: 0.3.2 license: AGPL-3.0-or-later modules: - rss From 1a52d18f5993e7c5e7c78729a396dbe686b0a560 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 21 Feb 2023 12:43:32 +0200 Subject: [PATCH 10/17] Show current template if ran without arguments --- rss/bot.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/rss/bot.py b/rss/bot.py index 0352f4c..6c6e2fc 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -21,6 +21,7 @@ from string import Template from time import mktime, time import asyncio import hashlib +import html import aiohttp import attr @@ -392,7 +393,7 @@ class RSSBot(Plugin): help="Change the notification template for a subscription in this room", ) @command.argument("feed_id", "feed ID", parser=int) - @command.argument("template", "new template", pass_raw=True) + @command.argument("template", "new template", pass_raw=True, required=False) async def command_template(self, evt: MessageEvent, feed_id: int, template: str) -> None: if not await self.can_manage(evt): return @@ -400,6 +401,13 @@ class RSSBot(Plugin): if not sub: await evt.reply("This room is not subscribed to that feed") return + if not template: + await evt.reply( + '

Current template in this room:

'
+                f"{html.escape(sub.notification_template.template)}"
+                "
", allow_html=True, markdown=False, + ) + return await self.dbm.update_template(feed.id, evt.room_id, template) sub = Subscription( feed_id=feed.id, From ef4915e43442f500f232d7f5067d6f9dbc73b013 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 21 Feb 2023 12:47:13 +0200 Subject: [PATCH 11/17] Add usage to readme --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index ee06f0c..aab772a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,30 @@ # rss A [maubot](https://github.com/maubot/maubot) that posts RSS feed updates to Matrix. + +## Usage +Basic commands: + +* `!rss subscribe ` - Subscribe the current room to a feed. +* `!rss unsubscribe ` - Unsubscribe the current room from a feed. +* `!rss subscriptions` - List subscriptions (and feed IDs) in the current room. +* `!rss notice [true/false]` - Set whether the bot should send new + posts as `m.notice` (if false, they're sent as `m.text`). +* `!rss template [new template]` - Change the post template for a + feed in the current room. If the new template is omitted, the bot replies + with the current template. + +### Templates +The default template is `New post in $feed_title: [$title]($link)`. + +Templates are interpreted as markdown with some simple variable substitution. +The following variables are available: + +* `$feed_url` - The URL that was used to subscribe to the feed. +* `$feed_link` - The home page of the feed. +* `$feed_title` - The title of the feed. +* `$feed_subtitle` - The subtitle of the feed. +* `$id` - The unique ID of the entry. +* `$date` - The date of the entry. +* `$title` - The title of the entry. +* `$summary` - The summary/description of the entry. +* `$link` - The link of the entry. From eeb71a008f39a2bcbe8b8bdba36ead433fd8eba0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 21 Feb 2023 13:22:19 +0200 Subject: [PATCH 12/17] Fix formatting --- rss/bot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rss/bot.py b/rss/bot.py index 6c6e2fc..4806f1f 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -405,7 +405,9 @@ class RSSBot(Plugin): await evt.reply( '

Current template in this room:

'
                 f"{html.escape(sub.notification_template.template)}"
-                "
", allow_html=True, markdown=False, + "", + allow_html=True, + markdown=False, ) return await self.dbm.update_template(feed.id, evt.room_id, template) From a8f134012515202e4be7fab0b0609f08b8b068f4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 30 Jan 2025 14:07:57 +0200 Subject: [PATCH 13/17] Update feedparser input --- rss/bot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rss/bot.py b/rss/bot.py index 945903b..423118c 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -22,6 +22,7 @@ from time import mktime, time import asyncio import hashlib import html +import io import aiohttp import attr @@ -264,7 +265,7 @@ class RSSBot(Plugin): except UnicodeDecodeError: content = str(await resp.read())[2:-1] headers = {"Content-Location": feed.url, **resp.headers, "Content-Encoding": "identity"} - parsed_data = feedparser.parse(content, response_headers=headers) + parsed_data = feedparser.parse(io.StringIO(content), response_headers=headers) if parsed_data.bozo: if not isinstance(parsed_data.bozo_exception, feedparser.ThingsNobodyCaresAboutButMe): raise parsed_data.bozo_exception From f62b0335dd0f12c95d873fe2729cca43d22f4b84 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 30 Jan 2025 14:08:04 +0200 Subject: [PATCH 14/17] Update linters --- .github/workflows/python-lint.yml | 2 +- .pre-commit-config.yaml | 6 +++--- pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index fc28bdb..18be560 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.13" - uses: isort/isort-action@master with: sortPaths: "./rss" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f1b3e5..caefdcb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v5.0.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 25.1.0 hooks: - id: black language_version: python3 files: ^rss/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 6.0.0 hooks: - id: isort files: ^rss/.*\.pyi?$ diff --git a/pyproject.toml b/pyproject.toml index 3e608c9..f143797 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,4 +8,4 @@ line_length = 99 [tool.black] line-length = 99 -target-version = ["py38"] +target-version = ["py310"] From 68e5a84096cfe26e0b95f870726912f94d540c8e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 30 Jan 2025 14:10:05 +0200 Subject: [PATCH 15/17] Bump version to v0.4.0 --- maubot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot.yaml b/maubot.yaml index 4cefc3d..c1ec735 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,6 +1,6 @@ maubot: 0.3.0 id: xyz.maubot.rss -version: 0.3.2 +version: 0.4.0 license: AGPL-3.0-or-later modules: - rss From 72d08096b7af74b1565d7958a1aac3f8e246e59d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 30 Jan 2025 15:47:09 +0200 Subject: [PATCH 16/17] Pass raw data to feedparser. Fixes #59 --- rss/bot.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/rss/bot.py b/rss/bot.py index 423118c..74c1681 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -257,15 +257,9 @@ class RSSBot(Plugin): async def _parse_rss( cls, feed: Feed, resp: aiohttp.ClientResponse ) -> tuple[Feed, list[Entry]]: - try: - content = await resp.text() - except UnicodeDecodeError: - try: - content = await resp.text(encoding="utf-8", errors="ignore") - except UnicodeDecodeError: - content = str(await resp.read())[2:-1] + content = await resp.read() headers = {"Content-Location": feed.url, **resp.headers, "Content-Encoding": "identity"} - parsed_data = feedparser.parse(io.StringIO(content), response_headers=headers) + parsed_data = feedparser.parse(io.BytesIO(content), response_headers=headers) if parsed_data.bozo: if not isinstance(parsed_data.bozo_exception, feedparser.ThingsNobodyCaresAboutButMe): raise parsed_data.bozo_exception From 81ec8ed86494fae5f1dd49628c316158067ffeab Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 30 Jan 2025 15:53:50 +0200 Subject: [PATCH 17/17] Bump version to 0.4.1 --- maubot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot.yaml b/maubot.yaml index c1ec735..b8c0836 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,6 +1,6 @@ maubot: 0.3.0 id: xyz.maubot.rss -version: 0.4.0 +version: 0.4.1 license: AGPL-3.0-or-later modules: - rss