From f10b4c71e637f32b175e814c51b612cb19c3a7ea Mon Sep 17 00:00:00 2001
From: Noah Levitt <nlevitt@archive.org>
Date: Tue, 8 Nov 2016 13:39:32 -0800
Subject: [PATCH] update for reppy api change and pin to current version of
 reppy

---
 brozzler/robots.py | 37 +++++++++++++++++++------------------
 setup.py           |  4 ++--
 2 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/brozzler/robots.py b/brozzler/robots.py
index b4732ae..7c464ab 100644
--- a/brozzler/robots.py
+++ b/brozzler/robots.py
@@ -1,20 +1,20 @@
-#
-# brozzler/robots.py - robots.txt support
-#
-# Copyright (C) 2014-2016 Internet Archive
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
+'''
+brozzler/robots.py - robots.txt support
+
+Copyright (C) 2014-2016 Internet Archive
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
 
 import json
 import logging
@@ -44,7 +44,8 @@ def _robots_cache(site):
             req_sesh.headers.update(site.extra_headers())
         if site.user_agent:
             req_sesh.headers['User-Agent'] = site.user_agent
-        _robots_caches[site.id] = reppy.cache.RobotsCache(session=req_sesh)
+        _robots_caches[site.id] = reppy.cache.RobotsCache(
+                session=req_sesh, capacity=5)
 
     return _robots_caches[site.id]
 
diff --git a/setup.py b/setup.py
index 187b6b6..c13a531 100644
--- a/setup.py
+++ b/setup.py
@@ -32,7 +32,7 @@ def find_package_data(package):
 
 setuptools.setup(
         name='brozzler',
-        version='1.1b7.dev114',
+        version='1.1b7.dev115',
         description='Distributed web crawling with browsers',
         url='https://github.com/internetarchive/brozzler',
         author='Noah Levitt',
@@ -59,7 +59,7 @@ setuptools.setup(
         install_requires=[
             'PyYAML',
             'youtube-dl',
-            'reppy',
+            'reppy==0.4.1',
             'requests',
             'websocket-client',
             'pillow==3.3.0',