Merge branch 'master' of https://github.com/LukeLR/rss

README.md

@@ -12,6 +12,8 @@ Basic commands:
 * `!rss template <feed ID> [new template]` - Change the post template for a
   feed in the current room. If the new template is omitted, the bot replies
   with the current template.
+* `!rss postall <feed ID>` - Post all entries in the specified feed to the
+  current room
 
 ### Templates
 The default template is `New post in $feed_title: [$title]($link)`.

rss/bot.py

@@ -67,6 +67,8 @@ class BoolArgument(command.Argument):
             res = False
         elif part in ("t", "true", "y", "yes", "1"):
             res = True
+        elif self.required == False:
+            res = None
         else:
             raise ValueError("invalid boolean")
         return val[len(part) :], res
@@ -125,10 +127,14 @@ class RSSBot(Plugin):
             message = message[:match.span()[0]] + value + message[match.span()[1]:]
             
         msgtype = MessageType.NOTICE if sub.send_notice else MessageType.TEXT
+        msgchunks = [message[i:i + 30000] for i in range(0, len(message), 30000)]
+        self.log.debug(f"Message length: {len(message)} Content length: {len(entry.content)} Chunks: {len(msgchunks)}")
         try:
-            return await self.client.send_markdown(
-                sub.room_id, message, msgtype=msgtype, allow_html=True
+            for chunk in msgchunks:
+                returnval = await self.client.send_markdown(
+                    sub.room_id, chunk, msgtype=msgtype, allow_html=True
                 )
+            return returnval
         except Exception as e:
             self.log.warning(f"Failed to send {entry.id} of {feed.id} to {sub.room_id}: {e}")
 
@@ -303,6 +309,9 @@ class RSSBot(Plugin):
             title=getattr(entry, "title", ""),
             summary=getattr(entry, "description", "").strip(),
             link=getattr(entry, "link", ""),
+            content=entry["content"][0]["value"].strip() if hasattr(entry, "content")
+                                                            and len(entry["content"]) > 0
+                                                            and hasattr(entry["content"][0], "value") else "",
         )
 
     @staticmethod
@@ -434,6 +443,7 @@ class RSSBot(Plugin):
             title="Sample entry",
             summary="This is a sample entry to demonstrate your new template",
             link="http://example.com",
+            content="<b>Sample formatted content</b>"
         )
         await evt.reply(f"Template for feed ID {feed.id} updated. Sample notification:")
         await self._send(feed, sample_entry, sub)
@@ -454,6 +464,20 @@ class RSSBot(Plugin):
         send_type = "m.notice" if setting else "m.text"
         await evt.reply(f"Updates for feed ID {feed.id} will now be sent as `{send_type}`")
 
+    @rss.subcommand(
+        "postall", aliases=("p",), help="Post all previously seen entries from the given feed to this room"
+    )
+    @command.argument("feed_id", "feed ID", parser=int)
+    async def command_postall(self, evt: MessageEvent, feed_id: int) -> None:
+        if not await self.can_manage(evt):
+            return
+        sub, feed = await self.dbm.get_subscription(feed_id, evt.room_id)
+        if not sub:
+            await evt.reply("This room is not subscribed to that feed")
+            return
+        for entry in await self.dbm.get_entries(feed.id):
+            await self._broadcast(feed, entry, [sub])
+
     @staticmethod
     def _format_subscription(feed: Feed, subscriber: str) -> str:
         msg = (

rss/db.py

@@ -97,6 +97,7 @@ class Entry:
     title: str
     summary: str
     link: str
+    content: str
 
     @classmethod
     def from_row(cls, row: Record | None) -> Entry | None:
@@ -143,7 +144,7 @@ class DBManager:
         return [(Feed.from_row(row), row["user_id"]) for row in rows]
 
     async def get_entries(self, feed_id: int) -> list[Entry]:
-        q = "SELECT feed_id, id, date, title, summary, link FROM entry WHERE feed_id = $1"
+        q = "SELECT feed_id, id, date, title, summary, link, content FROM entry WHERE feed_id = $1"
         return [Entry.from_row(row) for row in await self.db.fetch(q, feed_id)]
 
     async def add_entries(self, entries: list[Entry], override_feed_id: int | None = None) -> None:
@@ -153,14 +154,14 @@ class DBManager:
             for entry in entries:
                 entry.feed_id = override_feed_id
         records = [attr.astuple(entry) for entry in entries]
-        columns = ("feed_id", "id", "date", "title", "summary", "link")
+        columns = ("feed_id", "id", "date", "title", "summary", "link", "content")
         async with self.db.acquire() as conn:
             if self.db.scheme == Scheme.POSTGRES:
                 await conn.copy_records_to_table("entry", records=records, columns=columns)
             else:
                 q = (
-                    "INSERT INTO entry (feed_id, id, date, title, summary, link) "
-                    "VALUES ($1, $2, $3, $4, $5, $6)"
+                    "INSERT INTO entry (feed_id, id, date, title, summary, link, content) "
+                    "VALUES ($1, $2, $3, $4, $5, $6, $7)"
                 )
                 await conn.executemany(q, records)
 
@@ -218,14 +219,18 @@ class DBManager:
         room_id: RoomID,
         user_id: UserID,
         template: str | None = None,
-        send_notice: bool = True,
+        send_notice: bool = True
     ) -> None:
         q = """
-        INSERT INTO subscription (feed_id, room_id, user_id, notification_template, send_notice)
+        INSERT INTO subscription (
+            feed_id, room_id, user_id, notification_template,
+            send_notice)
         VALUES ($1, $2, $3, $4, $5)
         """
         template = template or "New post in $feed_title: [$title]($link)"
-        await self.db.execute(q, feed_id, room_id, user_id, template, send_notice)
+        await self.db.execute(
+            q, feed_id, room_id, user_id, template, send_notice,
+        )
 
     async def unsubscribe(self, feed_id: int, room_id: RoomID) -> None:
         q = "DELETE FROM subscription WHERE feed_id = $1 AND room_id = $2"

rss/migrations.py

@@ -17,7 +17,6 @@ from mautrix.util.async_db import Connection, Scheme, UpgradeTable
 
 upgrade_table = UpgradeTable()
 
-
 @upgrade_table.register(description="Latest revision", upgrades_to=3)
 async def upgrade_latest(conn: Connection, scheme: Scheme) -> None:
     gen = "GENERATED ALWAYS AS IDENTITY" if scheme != Scheme.SQLITE else ""
@@ -72,3 +71,8 @@ async def upgrade_v2(conn: Connection) -> None:
 async def upgrade_v3(conn: Connection) -> None:
     await conn.execute("ALTER TABLE feed ADD COLUMN next_retry BIGINT DEFAULT 0")
     await conn.execute("ALTER TABLE feed ADD COLUMN error_count BIGINT DEFAULT 0")
+
+
+@upgrade_table.register(description="Add support for encoded content")
+async def upgrade_v4(conn: Connection) -> None:
+    await conn.execute("ALTER TABLE entry ADD COLUMN content TEXT")