Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

add some trace logging to debug test failure

Browse Source
This commit is contained in:
Noah Levitt 2019-10-02 15:45:59 -07:00
parent 8a51f28c3d
commit fdb24f2893
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
brozzler/frontier.py
View File

@ -313,6 +313,9 @@ class RethinkDbFrontier:
representing the same url but with possibly different metadata.
'''
existing_page.priority += fresh_page.priority
self.logger.trace(
'adding hashtags %r to existing hashtags %r for page %s',
existing_page.hashtags, fresh_page.hashtags)
existing_page.hashtags = list(set(
existing_page.hashtags + fresh_page.hashtags))
existing_page.hops_off = min(
@ -375,9 +378,7 @@ class RethinkDbFrontier:
decisions['accepted'].add(fresh_page.url)
if fresh_page.id in pages:
page = pages[fresh_page.id]
page.hashtags = list(set((page.hashtags or [])
+ fresh_page.hashtags))
page.priority += fresh_page.priority
self._merge_page(page, fresh_page)
counts['updated'] += 1
else:
pages[fresh_page.id] = fresh_page