From 03bb1280050e6fcb09405c8848cb58ec7807f8d9 Mon Sep 17 00:00:00 2001 From: Andrew Kvalheim Date: Thu, 14 Jul 2022 09:41:33 -0700 Subject: [PATCH 1/9] Key entries by link if missing ID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the problem of incorrectly duplicated entries in feeds that update content but don’t explicitly provide entry IDs. Example feed: - https://www.to-rss.xyz/wikipedia/current_events/ Example entry: Current events: 2022-07-13 https://en.wikipedia.org/wiki/Portal:Current_events/2022_July_13 [VARIABLE CONTENT] Wed, 13 Jul 2022 00:00:00 -0000 This behavior is suggested by the common practice of using an entry’s link as its ID value, and is consistent with typical feed aggregators such as Feedbin and Inoreader. --- rss/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rss/bot.py b/rss/bot.py index 0352f4c..42ca897 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -279,12 +279,12 @@ class RSSBot(Plugin): feed_id=feed_id, id=( getattr(entry, "id", None) + or getattr(entry, "link", None) or hashlib.sha1( " ".join( [ getattr(entry, "title", ""), getattr(entry, "description", ""), - getattr(entry, "link", ""), ] ).encode("utf-8") ).hexdigest() From 1a52d18f5993e7c5e7c78729a396dbe686b0a560 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 21 Feb 2023 12:43:32 +0200 Subject: [PATCH 2/9] Show current template if ran without arguments --- rss/bot.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/rss/bot.py b/rss/bot.py index 0352f4c..6c6e2fc 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -21,6 +21,7 @@ from string import Template from time import mktime, time import asyncio import hashlib +import html import aiohttp import attr @@ -392,7 +393,7 @@ class RSSBot(Plugin): help="Change the notification template for a subscription in this room", ) @command.argument("feed_id", "feed ID", parser=int) - @command.argument("template", "new template", pass_raw=True) + @command.argument("template", "new template", pass_raw=True, required=False) async def command_template(self, evt: MessageEvent, feed_id: int, template: str) -> None: if not await self.can_manage(evt): return @@ -400,6 +401,13 @@ class RSSBot(Plugin): if not sub: await evt.reply("This room is not subscribed to that feed") return + if not template: + await evt.reply( + '

Current template in this room:

'
+                f"{html.escape(sub.notification_template.template)}"
+                "
", allow_html=True, markdown=False, + ) + return await self.dbm.update_template(feed.id, evt.room_id, template) sub = Subscription( feed_id=feed.id, From ef4915e43442f500f232d7f5067d6f9dbc73b013 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 21 Feb 2023 12:47:13 +0200 Subject: [PATCH 3/9] Add usage to readme --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index ee06f0c..aab772a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,30 @@ # rss A [maubot](https://github.com/maubot/maubot) that posts RSS feed updates to Matrix. + +## Usage +Basic commands: + +* `!rss subscribe ` - Subscribe the current room to a feed. +* `!rss unsubscribe ` - Unsubscribe the current room from a feed. +* `!rss subscriptions` - List subscriptions (and feed IDs) in the current room. +* `!rss notice [true/false]` - Set whether the bot should send new + posts as `m.notice` (if false, they're sent as `m.text`). +* `!rss template [new template]` - Change the post template for a + feed in the current room. If the new template is omitted, the bot replies + with the current template. + +### Templates +The default template is `New post in $feed_title: [$title]($link)`. + +Templates are interpreted as markdown with some simple variable substitution. +The following variables are available: + +* `$feed_url` - The URL that was used to subscribe to the feed. +* `$feed_link` - The home page of the feed. +* `$feed_title` - The title of the feed. +* `$feed_subtitle` - The subtitle of the feed. +* `$id` - The unique ID of the entry. +* `$date` - The date of the entry. +* `$title` - The title of the entry. +* `$summary` - The summary/description of the entry. +* `$link` - The link of the entry. From eeb71a008f39a2bcbe8b8bdba36ead433fd8eba0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 21 Feb 2023 13:22:19 +0200 Subject: [PATCH 4/9] Fix formatting --- rss/bot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rss/bot.py b/rss/bot.py index 6c6e2fc..4806f1f 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -405,7 +405,9 @@ class RSSBot(Plugin): await evt.reply( '

Current template in this room:

'
                 f"{html.escape(sub.notification_template.template)}"
-                "
", allow_html=True, markdown=False, + "", + allow_html=True, + markdown=False, ) return await self.dbm.update_template(feed.id, evt.room_id, template) From a8f134012515202e4be7fab0b0609f08b8b068f4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 30 Jan 2025 14:07:57 +0200 Subject: [PATCH 5/9] Update feedparser input --- rss/bot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rss/bot.py b/rss/bot.py index 945903b..423118c 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -22,6 +22,7 @@ from time import mktime, time import asyncio import hashlib import html +import io import aiohttp import attr @@ -264,7 +265,7 @@ class RSSBot(Plugin): except UnicodeDecodeError: content = str(await resp.read())[2:-1] headers = {"Content-Location": feed.url, **resp.headers, "Content-Encoding": "identity"} - parsed_data = feedparser.parse(content, response_headers=headers) + parsed_data = feedparser.parse(io.StringIO(content), response_headers=headers) if parsed_data.bozo: if not isinstance(parsed_data.bozo_exception, feedparser.ThingsNobodyCaresAboutButMe): raise parsed_data.bozo_exception From f62b0335dd0f12c95d873fe2729cca43d22f4b84 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 30 Jan 2025 14:08:04 +0200 Subject: [PATCH 6/9] Update linters --- .github/workflows/python-lint.yml | 2 +- .pre-commit-config.yaml | 6 +++--- pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index fc28bdb..18be560 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.13" - uses: isort/isort-action@master with: sortPaths: "./rss" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f1b3e5..caefdcb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v5.0.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 25.1.0 hooks: - id: black language_version: python3 files: ^rss/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 6.0.0 hooks: - id: isort files: ^rss/.*\.pyi?$ diff --git a/pyproject.toml b/pyproject.toml index 3e608c9..f143797 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,4 +8,4 @@ line_length = 99 [tool.black] line-length = 99 -target-version = ["py38"] +target-version = ["py310"] From 68e5a84096cfe26e0b95f870726912f94d540c8e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 30 Jan 2025 14:10:05 +0200 Subject: [PATCH 7/9] Bump version to v0.4.0 --- maubot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot.yaml b/maubot.yaml index 4cefc3d..c1ec735 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,6 +1,6 @@ maubot: 0.3.0 id: xyz.maubot.rss -version: 0.3.2 +version: 0.4.0 license: AGPL-3.0-or-later modules: - rss From 72d08096b7af74b1565d7958a1aac3f8e246e59d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 30 Jan 2025 15:47:09 +0200 Subject: [PATCH 8/9] Pass raw data to feedparser. Fixes #59 --- rss/bot.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/rss/bot.py b/rss/bot.py index 423118c..74c1681 100644 --- a/rss/bot.py +++ b/rss/bot.py @@ -257,15 +257,9 @@ class RSSBot(Plugin): async def _parse_rss( cls, feed: Feed, resp: aiohttp.ClientResponse ) -> tuple[Feed, list[Entry]]: - try: - content = await resp.text() - except UnicodeDecodeError: - try: - content = await resp.text(encoding="utf-8", errors="ignore") - except UnicodeDecodeError: - content = str(await resp.read())[2:-1] + content = await resp.read() headers = {"Content-Location": feed.url, **resp.headers, "Content-Encoding": "identity"} - parsed_data = feedparser.parse(io.StringIO(content), response_headers=headers) + parsed_data = feedparser.parse(io.BytesIO(content), response_headers=headers) if parsed_data.bozo: if not isinstance(parsed_data.bozo_exception, feedparser.ThingsNobodyCaresAboutButMe): raise parsed_data.bozo_exception From 81ec8ed86494fae5f1dd49628c316158067ffeab Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 30 Jan 2025 15:53:50 +0200 Subject: [PATCH 9/9] Bump version to 0.4.1 --- maubot.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maubot.yaml b/maubot.yaml index c1ec735..b8c0836 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,6 +1,6 @@ maubot: 0.3.0 id: xyz.maubot.rss -version: 0.4.0 +version: 0.4.1 license: AGPL-3.0-or-later modules: - rss