Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

debugging travis-ci test failure

Browse Source
This commit is contained in:
Noah Levitt 2017-04-18 10:23:45 -07:00
parent dc43794363
commit 0880ab615b
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
brozzler/robots.py
View File

@ -102,6 +102,9 @@ def is_permitted_by_robots(site, url, proxy=None):
url, site.user_agent or "brozzler") url, site.user_agent or "brozzler")
return result return result
except Exception as e: except Exception as e:
logging.warn(
"caught exception fetching robots.txt: e=%s vars(e)=%s",
repr(e), vars(e))
if isinstance(e, reppy.exceptions.ServerError) and isinstance( if isinstance(e, reppy.exceptions.ServerError) and isinstance(
e.args[0], brozzler.ReachedLimit): e.args[0], brozzler.ReachedLimit):
raise e.args[0] raise e.args[0]