debugging travis-ci test failure

2025-02-23 16:19:49 -05:00 · 2017-04-18 10:23:45 -07:00 · 2017-04-18 10:23:45 -07:00 · 0880ab615b
commit 0880ab615b
parent dc43794363
1 changed files with 3 additions and 0 deletions
--- a/brozzler/robots.py
+++ b/brozzler/robots.py
@ -102,6 +102,9 @@ def is_permitted_by_robots(site, url, proxy=None):
                    url, site.user_agent or "brozzler")
            return result
        except Exception as e:
+            logging.warn(
+                    "caught exception fetching robots.txt: e=%s vars(e)=%s",
+                    repr(e), vars(e))
            if isinstance(e, reppy.exceptions.ServerError) and isinstance(
                    e.args[0], brozzler.ReachedLimit):
                raise e.args[0]