Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

handle exceptions in robots.txt fetching/parsing

Browse Source
This commit is contained in:
Noah Levitt 2015-07-22 00:54:49 +00:00
parent f00571f7bd
commit 6a09f2095c
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
brozzler/site.py
View File

@ -46,7 +46,11 @@ class Site:
self.scope_surt = new_scope_surt
def is_permitted_by_robots(self, url):
return self.ignore_robots or self._robots_cache.allowed(url, "brozzler")
try:
return self.ignore_robots or self._robots_cache.allowed(url, "brozzler")
except BaseException as e:
self.logger.error("problem with robots.txt for {}: {}".format(url, e))
return False
def is_in_scope(self, url):
try: