Revert "initial commit"

This reverts commit 5368a84066.
This commit is contained in:
Barbara Miller 2019-08-08 14:03:30 -07:00
parent d0c46db746
commit c6308fe754

View file

@ -90,7 +90,7 @@ def is_permitted_by_robots(site, url, proxy=None):
brozzler.ReachedLimit: if warcprox responded with 420 Reached Limit brozzler.ReachedLimit: if warcprox responded with 420 Reached Limit
requests.exceptions.ProxyError: if the proxy is down requests.exceptions.ProxyError: if the proxy is down
''' '''
if site.ignore_robots and not site.user_agent: if site.ignore_robots:
return True return True
try: try: