Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-23 16:19:49 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

reppy 0.4.1 has a significantly different api apparently, so for now let's go back to 0.3.4

Browse Source
This commit is contained in:
Noah Levitt 2016-11-08 14:11:46 -08:00
parent fe18d915f5
commit 125a31165a
2 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
brozzler/robots.py
View File

@ -44,8 +44,7 @@ def _robots_cache(site):
req_sesh.headers.update(site.extra_headers())
if site.user_agent:
req_sesh.headers['User-Agent'] = site.user_agent
_robots_caches[site.id] = reppy.cache.RobotsCache(
session=req_sesh, capacity=5)
_robots_caches[site.id] = reppy.cache.RobotsCache(session=req_sesh)
return _robots_caches[site.id]

4
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup(
name='brozzler',
version='1.1b7.dev116',
version='1.1b7.dev117',
description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler',
author='Noah Levitt',
@ -59,7 +59,7 @@ setuptools.setup(
install_requires=[
'PyYAML',
'youtube-dl',
'reppy==0.4.1',
'reppy==0.3.4',
'requests',
'websocket-client',
'pillow==3.3.0',