Merge branch 'master' into qa

* master:
  bump dev version number after some PR merges
  bugfix for BrozzlerWorker._needs_browsing
  Remove redundant method parameter.
  bugfix
  Make youtube-dl optional in BrozzlerWorker.brozzle_page

brozzler/browser.py

@@ -376,7 +376,7 @@ class Browser:
         return self.websock_url is not None
 
     def browse_page(
-            self, page_url, ignore_cert_errors=False, extra_headers=None,
+            self, page_url, extra_headers=None,
             user_agent=None, behavior_parameters=None,
             on_request=None, on_response=None, on_screenshot=None,
             username=None, password=None, hashtags=None,

brozzler/worker.py

@@ -316,8 +316,10 @@ class BrozzlerWorker:
 
         return full_jpeg, thumb_jpeg
 
-    def brozzle_page(self, browser, site, page, on_screenshot=None):
+    def brozzle_page(self, browser, site, page, on_screenshot=None,
+                     enable_youtube_dl=True):
         self.logger.info("brozzling {}".format(page))
+        if enable_youtube_dl:
             try:
                 with tempfile.TemporaryDirectory(prefix='brzl-ydl-') as tempdir:
                     ydl = self._youtube_dl(tempdir, site)
@@ -340,6 +342,8 @@ class BrozzlerWorker:
                     self.logger.error(
                             'youtube_dl raised exception on %s', page,
                             exc_info=True)
+        else:
+            ydl_spy = False
 
         if self._needs_browsing(page, ydl_spy):
             self.logger.info('needs browsing: %s', page)
@@ -435,6 +439,7 @@ class BrozzlerWorker:
                     'proxy error fetching %s' % page.url) from e
 
     def _needs_browsing(self, page, brozzler_spy):
+        if brozzler_spy:
             final_bounces = brozzler_spy.final_bounces(page.url)
             if not final_bounces:
                 return True
@@ -443,8 +448,11 @@ class BrozzlerWorker:
                         'text/html', 'application/xhtml+xml']:
                     return True
             return False
+        else:
+            return True
 
     def _already_fetched(self, page, brozzler_spy):
+        if brozzler_spy:
             for txn in brozzler_spy.final_bounces(page.url):
                 if (txn['method'] == 'GET' and txn['status_code'] == 200):
                     return True

setup.py

@@ -32,7 +32,7 @@ def find_package_data(package):
 
 setuptools.setup(
         name='brozzler',
-        version='1.1b12.dev265',
+        version='1.1b12.dev266',
         description='Distributed web crawling with browsers',
         url='https://github.com/internetarchive/brozzler',
         author='Noah Levitt',