configurable behavior timeout

brozzler/__init__.py

@@ -106,6 +106,7 @@ def behavior_script(url, template_parameters=None, behaviors_dir=None):
     Returns the javascript behavior string populated with template_parameters.
     '''
     import re, logging, json
+    behavior_timeout_custom = None
     for behavior in behaviors(behaviors_dir=behaviors_dir):
         if re.match(behavior['url_regex'], url):
             parameters = dict()
@@ -119,8 +120,10 @@ def behavior_script(url, template_parameters=None, behaviors_dir=None):
             logging.info(
                     'using template=%r populated with parameters=%r for %r',
                     behavior['behavior_js_template'], json.dumps(parameters), url)
-            return script
-    return None
+            if 'behavior_timeout_sec' in parameters:
+                behavior_timeout_custom = int(parameters['behavior_timeout_sec'])
+            return script, behavior_timeout_custom
+    return None, behavior_timeout_custom
 
 class ThreadExceptionGate:
     logger = logging.getLogger(__module__ + "." + __qualname__)

brozzler/behaviors.yaml

@@ -59,6 +59,7 @@
   default_parameters:
     actions:
       - selector: div.teaser, li.pager__item a
+    behavior_timeout_sec: 1800
 - # https://webarchive.jira.com/browse/ARI-5389
   url_regex: '^https?://pitchfork\.com/.*$'
   behavior_js_template: umbraBehavior.js.j2

brozzler/browser.py

@@ -462,9 +462,11 @@ class Browser:
                         self.navigate_to_page(page_url, timeout=page_timeout)
                 if on_screenshot:
                     self._try_screenshot(on_screenshot)
-                behavior_script = brozzler.behavior_script(
+                behavior_script, behavior_timeout_custom = brozzler.behavior_script(
                         page_url, behavior_parameters,
                         behaviors_dir=behaviors_dir)
+                if behavior_timeout_custom:
+                    behavior_timeout = behavior_timeout_custom
                 self.run_behavior(behavior_script, timeout=behavior_timeout)
                 if skip_extract_outlinks:
                     outlinks = []