diff --git a/brozzler/webconsole/__init__.py b/brozzler/webconsole/__init__.py index 5bd5705..cbb2677 100644 --- a/brozzler/webconsole/__init__.py +++ b/brozzler/webconsole/__init__.py @@ -59,7 +59,12 @@ SETTINGS = { } r = rethinkstuff.Rethinker( SETTINGS['RETHINKDB_SERVERS'], db=SETTINGS['RETHINKDB_DB']) -service_registry = rethinkstuff.ServiceRegistry(r) +_svc_reg = None +def service_registry(): + global _svc_reg + if not _svc_reg: + _svc_reg = rethinkstuff.ServiceRegistry(r) + return _svc_reg @app.route("/api/sites//queued_count") @app.route("/api/site//queued_count") @@ -174,12 +179,12 @@ def job_yaml(job_id): @app.route("/api/workers") def workers(): - workers_ = service_registry.available_services("brozzler-worker") + workers_ = service_registry().available_services("brozzler-worker") return flask.jsonify(workers=list(workers_)) @app.route("/api/services") def services(): - services_ = service_registry.available_services() + services_ = service_registry().available_services() return flask.jsonify(services=list(services_)) @app.route("/api/jobs") diff --git a/setup.py b/setup.py index 52b60ea..357be5f 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b6.dev75', + version='1.1b6.dev76', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt', diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 820c7fa..16220f7 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -1,16 +1,13 @@ Vagrant.configure(2) do |config| config.vm.box = "ubuntu/trusty64" - config.vm.hostname = "brozzler-easy" + config.vm.define "10.9.9.9" + config.vm.hostname = "brzl" + config.vm.network :private_network, ip: "10.9.9.9" config.vm.synced_folder "..", "/brozzler" config.vm.provision "ansible" do |ansible| + ansible.inventory_path = "ansible/hosts" ansible.playbook = "ansible/playbook.yml" - ansible.groups = { - "rethinkdb" => ["default"], - "warcprox" => ["default"], - "brozzler-worker" => ["default"], - # "brozzler-webconsole" => ["default"], - } end end diff --git a/vagrant/ansible/hosts b/vagrant/ansible/hosts new file mode 100644 index 0000000..708a07a --- /dev/null +++ b/vagrant/ansible/hosts @@ -0,0 +1,16 @@ +ansible_ssh_private_key_file=.vagrant/machines/10.9.9.9/virtualbox/private_key + +[rethinkdb] +10.9.9.9 + +[warcprox] +10.9.9.9 + +[brozzler-worker] +10.9.9.9 + +[brozzler-webconsole] +10.9.9.9 + +[pywb] +10.9.9.9 \ No newline at end of file diff --git a/vagrant/ansible/playbook.yml b/vagrant/ansible/playbook.yml index f22f327..fda30b8 100644 --- a/vagrant/ansible/playbook.yml +++ b/vagrant/ansible/playbook.yml @@ -2,27 +2,27 @@ - name: apply common configuration to all nodes hosts: all roles: - - common + - common - name: deploy rethinkdb hosts: rethinkdb roles: - - rethinkdb + - rethinkdb - name: deploy warcprox hosts: warcprox roles: - - warcprox + - warcprox - name: deploy brozzler-worker hosts: brozzler-worker roles: - - brozzler-worker + - brozzler-worker -# - name: deploy brozzler-webconsole -# hosts: brozzler-webconsole -# roles: -# - brozzler-webconsole +- name: deploy brozzler-webconsole + hosts: brozzler-webconsole + roles: + - brozzler-webconsole # - name: deploy pywb # hosts: pywb diff --git a/vagrant/ansible/roles/brozzler-webconsole/handlers/main.yml b/vagrant/ansible/roles/brozzler-webconsole/handlers/main.yml new file mode 100644 index 0000000..732d565 --- /dev/null +++ b/vagrant/ansible/roles/brozzler-webconsole/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: restart brozzler-webconsole + service: name=brozzler-webconsole state=restarted + become: true diff --git a/vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml b/vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml index 6b54696..f0f70d8 100644 --- a/vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml +++ b/vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml @@ -1,19 +1,15 @@ --- -- name: git clone https://github.com/internetarchive/brozzler.git - git: repo=https://github.com/internetarchive/brozzler.git - dest=/home/vagrant/brozzler -- name: pip install -r requirements.txt in virtualenv - pip: requirements=/home/vagrant/brozzler/webconsole/requirements.txt +- name: install brozzler[webconsole] in virtualenv + become: true + pip: name='-e /brozzler[webconsole]' virtualenv=/home/vagrant/brozzler-webconsole-ve34 virtualenv_python=python3.4 extra_args='--no-input --upgrade --pre' notify: - - restart brozzler-webconsole + - restart brozzler-webconsole - name: install upstart config /etc/init/brozzler-webconsole.conf become: true template: src=templates/brozzler-webconsole.conf.j2 dest=/etc/init/brozzler-webconsole.conf notify: - - restart brozzler-webconsole - - + - restart brozzler-webconsole \ No newline at end of file diff --git a/vagrant/ansible/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2 b/vagrant/ansible/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2 index efe2d03..c8d0656 100644 --- a/vagrant/ansible/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2 +++ b/vagrant/ansible/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2 @@ -3,19 +3,16 @@ description "brozzler-webconsole" start on runlevel [2345] stop on runlevel [!2345] -env PYTHONPATH=/home/vagrant/brozzler-webconsole-ve34/lib/python3.4/site-packages:/home/vagrant/brozzler/webconsole +env PYTHONPATH=/home/vagrant/brozzler-webconsole-ve34/lib/python3.4/site-packages env PATH=/home/vagrant/brozzler-webconsole-ve34/bin:/usr/bin:/bin env LC_ALL=C.UTF-8 -env WAYBACK_BASEURL={{base_wayback_url}}/all -# env RETHINKDB_SERVERS={{groups['rethinkdb'] | join(',')}} -env RETHINKDB_SERVERS=localhost -env RETHINKDB_DB={{rethinkdb_db}} +env WAYBACK_BASEURL=http://{{groups['pywb'][0]}}:8880/brozzler +env RETHINKDB_SERVERS={{groups['rethinkdb'] | join(',')}} +env RETHINKDB_DB=brozzler setuid vagrant # console log -exec gunicorn --bind=0.0.0.0:8081 brozzler-webconsole:app >&/vagrant/logs/brozzler-webconsole.log - - +exec gunicorn --bind=0.0.0.0:8881 brozzler.webconsole:app >>/vagrant/logs/brozzler-webconsole.log 2>&1 diff --git a/vagrant/ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2 b/vagrant/ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2 index 4ec328a..b36403e 100644 --- a/vagrant/ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2 +++ b/vagrant/ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2 @@ -19,7 +19,5 @@ stop on stopping Xvnc kill timeout 60 exec nice brozzler-worker \ - --rethinkdb-servers=localhost \ - --max-browsers=4 >>/vagrant/logs/brozzler-worker.log 2>&1 - # --rethinkdb-servers={{groups['rethinkdb'] | join(',')}} \ - + --rethinkdb-servers={{groups['rethinkdb'] | join(',')}} \ + --max-browsers=4 >>/vagrant/logs/brozzler-worker.log 2>&1 diff --git a/vagrant/ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2 b/vagrant/ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2 index 86b4012..32b959f 100644 --- a/vagrant/ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2 +++ b/vagrant/ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2 @@ -10,5 +10,6 @@ console log env PYTHONPATH=/home/vagrant/websockify-ve34/lib/python3.4/site-packages env PATH=/home/vagrant/websockify-ve34/bin:/usr/bin:/bin +# port 8901 is hard-coded in brozzler/webconsole/static/partials/workers.html exec nice websockify 0.0.0.0:8901 localhost:5901 diff --git a/vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-easy.conf.j2 b/vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-easy.conf.j2 index bbb1099..62b3ac5 100644 --- a/vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-easy.conf.j2 +++ b/vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-easy.conf.j2 @@ -1,5 +1,5 @@ runuser=vagrant -# bind=0.0.0.0 +bind=0.0.0.0 # directory=/var/lib/rethinkdb # log-file=/var/log/rethinkdb.log log-file=/vagrant/logs/rethinkdb.log # synced dir