Matrix-Mirrors/maubot-rss
1
0
Fork 0
mirror of https://github.com/maubot/rss.git synced 2025-01-20 20:41:37 -05:00
Code Issues Packages Projects Releases Wiki Activity

Pass feed URL to feedparser. Fixes #7

Browse Source
This commit is contained in:
Tulir Asokan 2019-09-17 22:19:30 +03:00
parent 8fcc2e8fb1
commit bcbab52c0c
1 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
rss/bot.py
View File

@ -94,8 +94,8 @@ class RSSBot(Plugin):
if not subs: if not subs:
return return
datas = await asyncio.gather(*[self.read_feed(feed.url) for feed in subs], loop=self.loop) datas = await asyncio.gather(*[self.read_feed(feed.url) for feed in subs], loop=self.loop)
for feed, data in zip(subs, datas): for feed, (data, headers) in zip(subs, datas):
parsed_data = feedparser.parse(data) parsed_data = feedparser.parse(data, response_headers=headers)
entries = parsed_data.entries entries = parsed_data.entries
new_entries = {entry.id: entry for entry in self.find_entries(feed.id, entries)} new_entries = {entry.id: entry for entry in self.find_entries(feed.id, entries)}
for old_entry in self.db.get_entries(feed.id): for old_entry in self.db.get_entries(feed.id):
@ -115,12 +115,12 @@ class RSSBot(Plugin):
self.log.exception("Error while polling feeds") self.log.exception("Error while polling feeds")
await asyncio.sleep(self.config["update_interval"] * 60, loop=self.loop) await asyncio.sleep(self.config["update_interval"] * 60, loop=self.loop)
async def read_feed(self, url: str) -> str: async def read_feed(self, url: str) -> Tuple[str, Dict[str, str]]:
try: try:
resp = await self.http.get(url) resp = await self.http.get(url)
except Exception: except Exception:
self.log.exception(f"Error fetching {url}") self.log.exception(f"Error fetching {url}")
return "" return "", {}
try: try:
content = await resp.text() content = await resp.text()
except UnicodeDecodeError: except UnicodeDecodeError:
@ -128,7 +128,7 @@ class RSSBot(Plugin):
content = await resp.text(encoding="utf-8") content = await resp.text(encoding="utf-8")
except: except:
content = str(await resp.read())[2:-1] content = str(await resp.read())[2:-1]
return content return content, {"Content-Location": url, **resp.headers}
@staticmethod @staticmethod
def get_date(entry: Any) -> datetime: def get_date(entry: Any) -> datetime:
@ -173,7 +173,8 @@ class RSSBot(Plugin):
if type(state_level) != int: if type(state_level) != int:
state_level = 50 state_level = 50
if user_level < state_level: if user_level < state_level:
await evt.reply("You don't have the permission to manage the subscriptions of this room.") await evt.reply(
"You don't have the permission to manage the subscriptions of this room.")
return False return False
return True return True
@ -190,7 +191,8 @@ class RSSBot(Plugin):
return return
feed = self.db.get_feed_by_url(url) feed = self.db.get_feed_by_url(url)
if not feed: if not feed:
metadata = feedparser.parse(await self.read_feed(url)) data, headers = await self.read_feed(url)
metadata = feedparser.parse(data, response_headers=headers)
if metadata.bozo: if metadata.bozo:
await evt.reply("That doesn't look like a valid feed.") await evt.reply("That doesn't look like a valid feed.")
return return