From 0880ab615b700faf3282d02bbac69b97f34c4578 Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Tue, 18 Apr 2017 10:23:45 -0700 Subject: [PATCH] debugging travis-ci test failure --- brozzler/robots.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/brozzler/robots.py b/brozzler/robots.py index 046ef22..97c7708 100644 --- a/brozzler/robots.py +++ b/brozzler/robots.py @@ -102,6 +102,9 @@ def is_permitted_by_robots(site, url, proxy=None): url, site.user_agent or "brozzler") return result except Exception as e: + logging.warn( + "caught exception fetching robots.txt: e=%s vars(e)=%s", + repr(e), vars(e)) if isinstance(e, reppy.exceptions.ServerError) and isinstance( e.args[0], brozzler.ReachedLimit): raise e.args[0]