mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-09-21 05:04:40 -04:00
parent
d0c46db746
commit
c6308fe754
1 changed files with 1 additions and 1 deletions
|
@ -90,7 +90,7 @@ def is_permitted_by_robots(site, url, proxy=None):
|
||||||
brozzler.ReachedLimit: if warcprox responded with 420 Reached Limit
|
brozzler.ReachedLimit: if warcprox responded with 420 Reached Limit
|
||||||
requests.exceptions.ProxyError: if the proxy is down
|
requests.exceptions.ProxyError: if the proxy is down
|
||||||
'''
|
'''
|
||||||
if site.ignore_robots and not site.user_agent:
|
if site.ignore_robots:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue