From 5368a840665dcf9770ede6006d685ff113c84a3f Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Thu, 11 Jul 2019 12:52:52 -0700 Subject: [PATCH] initial commit --- brozzler/robots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brozzler/robots.py b/brozzler/robots.py index 4122093..9acdd10 100644 --- a/brozzler/robots.py +++ b/brozzler/robots.py @@ -90,7 +90,7 @@ def is_permitted_by_robots(site, url, proxy=None): brozzler.ReachedLimit: if warcprox responded with 420 Reached Limit requests.exceptions.ProxyError: if the proxy is down ''' - if site.ignore_robots: + if site.ignore_robots and not site.user_agent: return True try: