Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-08 06:22:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

handle 420 Reached limit when fetching robots in brozzler-hq

Browse source
This commit is contained in:
Noah Levitt 2015-08-01 17:54:29 +00:00
parent 511e19ff4d
commit e6eeca6ae2
3 changed files with 35 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

2
brozzler/browser.py
View file

@ -238,7 +238,7 @@ class Browser:
if (not self._reached_limit
and message["params"]["response"]["status"] == 420
and "Warcprox-Meta" in CaseInsensitiveDict(message["params"]["response"]["headers"])):
warcprox_meta = json.loads(message["params"]["response"]["headers"]["Warcprox-Meta"])
warcprox_meta = json.loads(CaseInsensitiveDict(message["params"]["response"]["headers"])["Warcprox-Meta"])
self._reached_limit = brozzler.ReachedLimit(warcprox_meta=warcprox_meta)
self.logger.info("reached limit %s", self._reached_limit)