Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

reppy 0.4.1 has a significantly different api apparently, so for now let's go back to 0.3.4

Browse Source
This commit is contained in:
Noah Levitt 2016-11-08 14:11:46 -08:00
parent fe18d915f5
commit 125a31165a
2 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
brozzler/robots.py
View File

@ -44,8 +44,7 @@ def _robots_cache(site):
req_sesh.headers.update(site.extra_headers()) req_sesh.headers.update(site.extra_headers())
if site.user_agent: if site.user_agent:
req_sesh.headers['User-Agent'] = site.user_agent req_sesh.headers['User-Agent'] = site.user_agent
_robots_caches[site.id] = reppy.cache.RobotsCache( _robots_caches[site.id] = reppy.cache.RobotsCache(session=req_sesh)
session=req_sesh, capacity=5)
return _robots_caches[site.id] return _robots_caches[site.id]

4
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b7.dev116', version='1.1b7.dev117',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',
@ -59,7 +59,7 @@ setuptools.setup(
install_requires=[ install_requires=[
'PyYAML', 'PyYAML',
'youtube-dl', 'youtube-dl',
'reppy==0.4.1', 'reppy==0.3.4',
'requests', 'requests',
'websocket-client', 'websocket-client',
'pillow==3.3.0', 'pillow==3.3.0',