diff --git a/.travis.yml b/.travis.yml index cef3d80..80b82d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ before_install: - sudo pip install ansible==2.1.3.0 install: - ansible-playbook --extra-vars="brozzler_pip_name=file://$TRAVIS_BUILD_DIR#egg=brozzler user=travis" --inventory-file=ansible/hosts-localhost ansible/playbook.yml -- pip install $TRAVIS_BUILD_DIR 'warcprox==2.3' pytest +- pip install $TRAVIS_BUILD_DIR 'warcprox>=2.4b1' pytest script: - DISPLAY=:1 py.test -v tests after_failure: diff --git a/ansible/roles/pywb/tasks/main.yml b/ansible/roles/pywb/tasks/main.yml index 975359f..16b9ea7 100644 --- a/ansible/roles/pywb/tasks/main.yml +++ b/ansible/roles/pywb/tasks/main.yml @@ -5,6 +5,7 @@ become: true - name: install pywb in virtualenv pip: name=pywb + version=0.33.2 virtualenv={{venv_root}}/pywb-ve34 virtualenv_python=python3.4 extra_args='--no-input --upgrade --pre --cache-dir=/tmp/pip-cache' diff --git a/brozzler/behaviors.yaml b/brozzler/behaviors.yaml index 7ccf698..4d88637 100644 --- a/brozzler/behaviors.yaml +++ b/brozzler/behaviors.yaml @@ -68,6 +68,12 @@ click_css_selector: a[onclick] click_until_hard_timeout: False request_idle_timeout_sec: 10 +- # https://webarchive.jira.com/browse/ARI-5294 + url_regex: '^https?://citymedfordwi\.civicweb\.net/.*$' + behavior_js_template: umbraBehavior.js.j2 + default_parameters: + actions: + - selector: div.meeting-document-type-buttons button.button-small - # https://webarchive.jira.com/browse/ARI-5409 url_regex: '^https?://(?:www\.)?tuebingen.de/.*$' behavior_js_template: simpleclicks.js.j2 diff --git a/brozzler/easy.py b/brozzler/easy.py index e41f013..c9480a1 100644 --- a/brozzler/easy.py +++ b/brozzler/easy.py @@ -3,7 +3,7 @@ brozzler-easy - brozzler-worker, warcprox, pywb, and brozzler-dashboard all working together in a single process -Copyright (C) 2016 Internet Archive +Copyright (C) 2016-2018 Internet Archive Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -122,8 +122,8 @@ class BrozzlerEasyController: def __init__(self, args): self.stop = threading.Event() self.args = args - self.warcprox_controller = warcprox.main.init_controller( - self._warcprox_args(args)) + self.warcprox_controller = warcprox.controller.WarcproxController( + self._warcprox_opts(args)) self.brozzler_worker = self._init_brozzler_worker(args) self.pywb_httpd = self._init_pywb(args) self.dashboard_httpd = self._init_brozzler_dashboard(args) @@ -221,40 +221,38 @@ class BrozzlerEasyController: finally: self.shutdown() - def _warcprox_args(self, args): + def _warcprox_opts(self, args): ''' Takes args as produced by the argument parser built by _build_arg_parser and builds warcprox arguments object suitable to pass to warcprox.main.init_controller. Copies some arguments, renames some, populates some with defaults appropriate for brozzler-easy, etc. ''' - warcprox_args = argparse.Namespace() - warcprox_args.address = 'localhost' + warcprox_opts = warcprox.Options() + warcprox_opts.address = 'localhost' # let the OS choose an available port; discover it later using # sock.getsockname()[1] - warcprox_args.port = 0 - warcprox_args.cacert = args.cacert - warcprox_args.certs_dir = args.certs_dir - warcprox_args.directory = args.warcs_dir - warcprox_args.gzip = True - warcprox_args.prefix = 'brozzler' - warcprox_args.size = 1000 * 1000* 1000 - warcprox_args.rollover_idle_time = 3 * 60 - warcprox_args.digest_algorithm = 'sha1' - warcprox_args.base32 = True - warcprox_args.stats_db_file = None - warcprox_args.playback_port = None - warcprox_args.playback_index_db_file = None - warcprox_args.rethinkdb_servers = args.rethinkdb_servers - warcprox_args.rethinkdb_db = args.rethinkdb_db - warcprox_args.rethinkdb_big_table = True - warcprox_args.kafka_broker_list = None - warcprox_args.kafka_capture_feed_topic = None - warcprox_args.queue_size = 500 - warcprox_args.max_threads = None - warcprox_args.profile = False - warcprox_args.onion_tor_socks_proxy = args.onion_tor_socks_proxy - return warcprox_args + warcprox_opts.port = 0 + warcprox_opts.cacert = args.cacert + warcprox_opts.certs_dir = args.certs_dir + warcprox_opts.directory = args.warcs_dir + warcprox_opts.gzip = True + warcprox_opts.prefix = 'brozzler' + warcprox_opts.size = 1000 * 1000* 1000 + warcprox_opts.rollover_idle_time = 3 * 60 + warcprox_opts.digest_algorithm = 'sha1' + warcprox_opts.base32 = True + warcprox_opts.stats_db_file = None + warcprox_opts.playback_port = None + warcprox_opts.playback_index_db_file = None + warcprox_opts.rethinkdb_big_table_url = ( + 'rethinkdb://%s/%s/captures' % ( + args.rethinkdb_servers, args.rethinkdb_db)) + warcprox_opts.queue_size = 500 + warcprox_opts.max_threads = None + warcprox_opts.profile = False + warcprox_opts.onion_tor_socks_proxy = args.onion_tor_socks_proxy + return warcprox_opts def dump_state(self, signum=None, frame=None): state_strs = [] diff --git a/setup.py b/setup.py index 6c3a66d..5957cd2 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b12.dev281', + version='1.1b13.dev283', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt', @@ -79,8 +79,8 @@ setuptools.setup( extras_require={ 'dashboard': ['flask>=0.11', 'gunicorn'], 'easy': [ - 'warcprox>=2.1b1.dev87', - 'pywb', + 'warcprox>=2.4b1.dev145', + 'pywb<2', 'flask>=0.11', 'gunicorn' ], diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 48a9384..32e9734 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -661,11 +661,10 @@ def test_warcprox_outage_resiliency(httpd): opts = warcprox.Options() opts.address = '0.0.0.0' opts.port = 0 + opts.rethinkdb_services_url = 'rethinkdb://localhost/brozzler/services' - warcprox1 = warcprox.controller.WarcproxController( - service_registry=svcreg, options=opts) - warcprox2 = warcprox.controller.WarcproxController( - service_registry=svcreg, options=opts) + warcprox1 = warcprox.controller.WarcproxController(opts) + warcprox2 = warcprox.controller.WarcproxController(opts) warcprox1_thread = threading.Thread( target=warcprox1.run_until_shutdown, name='warcprox1') warcprox2_thread = threading.Thread(