Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

trace logging again

Browse Source
This commit is contained in:
Noah Levitt 2019-10-03 11:41:34 -07:00
parent fdb24f2893
commit 1c51396111
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
brozzler/frontier.py
View File

@ -315,7 +315,7 @@ class RethinkDbFrontier:
existing_page.priority += fresh_page.priority
self.logger.trace(
'adding hashtags %r to existing hashtags %r for page %s',
existing_page.hashtags, fresh_page.hashtags)
existing_page.hashtags, fresh_page.hashtags, fresh_page.url)
existing_page.hashtags = list(set(
existing_page.hashtags + fresh_page.hashtags))
existing_page.hops_off = min(
@ -371,6 +371,8 @@ class RethinkDbFrontier:
# get existing pages from rethinkdb
results = self.rr.table('pages').get_all(*fresh_pages.keys()).run()
pages = {doc['id']: brozzler.Page(self.rr, doc) for doc in results}
self.logger.trace('fresh_pages.keys()=%r', fresh_pages.keys())
self.logger.trace('existing pages.keys()=%r', pages.keys())
# build list of pages to save, consisting of new pages, and existing
# pages updated with higher priority and new hashtags