Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 15:55:49 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fix problem where each hashtag visited causes a page load if page url redirects

Browse Source
This commit is contained in:
Noah Levitt 2017-09-27 14:11:20 -07:00
parent 384c877e9a
commit ec847e48bc
2 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
brozzler/browser.py
View File

@ -454,7 +454,7 @@ class Browser:
else:
outlinks = self.extract_outlinks()
if not skip_visit_hashtags:
self.visit_hashtags(page_url, hashtags, outlinks)
self.visit_hashtags(self.url(), hashtags, outlinks)
final_page_url = self.url()
return final_page_url, outlinks
except brozzler.ReachedLimit:
@ -599,8 +599,7 @@ class Browser:
def try_login(self, username, password, timeout=300):
try_login_js = brozzler.jinja2_environment().get_template(
'try-login.js.j2').render(
username=username, password=password)
'try-login.js.j2').render(username=username, password=password)
self.websock_thread.got_page_load_event = None
self.send_to_chrome(

2
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup(
name='brozzler',
version='1.1b12.dev267',
version='1.1b12.dev268',
description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler',
author='Noah Levitt',