for vagrant, static ansible inventory file, add brozzler-webconsole

brozzler/webconsole/__init__.py

@@ -59,7 +59,12 @@ SETTINGS = {
 }
 r = rethinkstuff.Rethinker(
         SETTINGS['RETHINKDB_SERVERS'], db=SETTINGS['RETHINKDB_DB'])
-service_registry = rethinkstuff.ServiceRegistry(r)
+_svc_reg = None
+def service_registry():
+    global _svc_reg
+    if not _svc_reg:
+        _svc_reg = rethinkstuff.ServiceRegistry(r)
+    return _svc_reg
 
 @app.route("/api/sites/<site_id>/queued_count")
 @app.route("/api/site/<site_id>/queued_count")
@@ -174,12 +179,12 @@ def job_yaml(job_id):
 
 @app.route("/api/workers")
 def workers():
-    workers_ = service_registry.available_services("brozzler-worker")
+    workers_ = service_registry().available_services("brozzler-worker")
     return flask.jsonify(workers=list(workers_))
 
 @app.route("/api/services")
 def services():
-    services_ = service_registry.available_services()
+    services_ = service_registry().available_services()
     return flask.jsonify(services=list(services_))
 
 @app.route("/api/jobs")

setup.py

@@ -32,7 +32,7 @@ def find_package_data(package):
 
 setuptools.setup(
         name='brozzler',
-        version='1.1b6.dev75',
+        version='1.1b6.dev76',
         description='Distributed web crawling with browsers',
         url='https://github.com/internetarchive/brozzler',
         author='Noah Levitt',

vagrant/Vagrantfile

@@ -1,16 +1,13 @@
 Vagrant.configure(2) do |config|
   config.vm.box = "ubuntu/trusty64"
-  config.vm.hostname = "brozzler-easy"
+  config.vm.define "10.9.9.9"
+  config.vm.hostname = "brzl"
+  config.vm.network :private_network, ip: "10.9.9.9"
 
   config.vm.synced_folder "..", "/brozzler"
 
   config.vm.provision "ansible" do |ansible|
+    ansible.inventory_path = "ansible/hosts"
     ansible.playbook = "ansible/playbook.yml"
-    ansible.groups = {
-      "rethinkdb" => ["default"],
-      "warcprox" => ["default"],
-      "brozzler-worker" => ["default"],
-      # "brozzler-webconsole" => ["default"],
-    }
   end
 end

vagrant/ansible/hosts

@@ -0,0 +1,16 @@
+ansible_ssh_private_key_file=.vagrant/machines/10.9.9.9/virtualbox/private_key
+
+[rethinkdb]
+10.9.9.9
+
+[warcprox]
+10.9.9.9
+
+[brozzler-worker]
+10.9.9.9
+
+[brozzler-webconsole]
+10.9.9.9
+
+[pywb]
+10.9.9.9

vagrant/ansible/playbook.yml

@@ -2,27 +2,27 @@
 - name: apply common configuration to all nodes
   hosts: all
   roles:
-    - common
+  - common
 
 - name: deploy rethinkdb
   hosts: rethinkdb
   roles:
-    - rethinkdb
+  - rethinkdb
 
 - name: deploy warcprox
   hosts: warcprox
   roles:
-    - warcprox
+  - warcprox
 
 - name: deploy brozzler-worker
   hosts: brozzler-worker
   roles:
-    - brozzler-worker
+  - brozzler-worker
 
-# - name: deploy brozzler-webconsole
+- name: deploy brozzler-webconsole
-#   hosts: brozzler-webconsole
+  hosts: brozzler-webconsole
-#   roles:
+  roles:
-#     - brozzler-webconsole
+  - brozzler-webconsole
 
 # - name: deploy pywb
 #   hosts: pywb

vagrant/ansible/roles/brozzler-webconsole/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+- name: restart brozzler-webconsole
+  service: name=brozzler-webconsole state=restarted
+  become: true

vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml

@@ -1,19 +1,15 @@
 ---
-- name: git clone https://github.com/internetarchive/brozzler.git
+- name: install brozzler[webconsole] in virtualenv
-  git: repo=https://github.com/internetarchive/brozzler.git
+  become: true
-       dest=/home/vagrant/brozzler
+  pip: name='-e /brozzler[webconsole]'
-- name: pip install -r requirements.txt in virtualenv
-  pip: requirements=/home/vagrant/brozzler/webconsole/requirements.txt
        virtualenv=/home/vagrant/brozzler-webconsole-ve34
        virtualenv_python=python3.4
        extra_args='--no-input --upgrade --pre'
   notify:
-    - restart brozzler-webconsole
+  - restart brozzler-webconsole
 - name: install upstart config /etc/init/brozzler-webconsole.conf
   become: true
   template: src=templates/brozzler-webconsole.conf.j2
             dest=/etc/init/brozzler-webconsole.conf
   notify:
-    - restart brozzler-webconsole
+  - restart brozzler-webconsole
-
-

vagrant/ansible/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2

@@ -3,19 +3,16 @@ description "brozzler-webconsole"
 start on runlevel [2345]
 stop on runlevel [!2345]
 
-env PYTHONPATH=/home/vagrant/brozzler-webconsole-ve34/lib/python3.4/site-packages:/home/vagrant/brozzler/webconsole
+env PYTHONPATH=/home/vagrant/brozzler-webconsole-ve34/lib/python3.4/site-packages
 env PATH=/home/vagrant/brozzler-webconsole-ve34/bin:/usr/bin:/bin
 env LC_ALL=C.UTF-8
 
-env WAYBACK_BASEURL={{base_wayback_url}}/all
+env WAYBACK_BASEURL=http://{{groups['pywb'][0]}}:8880/brozzler
-# env RETHINKDB_SERVERS={{groups['rethinkdb'] | join(',')}}
+env RETHINKDB_SERVERS={{groups['rethinkdb'] | join(',')}}
-env RETHINKDB_SERVERS=localhost
+env RETHINKDB_DB=brozzler
-env RETHINKDB_DB={{rethinkdb_db}}
 
 setuid vagrant
 
 # console log
 
-exec gunicorn --bind=0.0.0.0:8081 brozzler-webconsole:app >&/vagrant/logs/brozzler-webconsole.log
+exec gunicorn --bind=0.0.0.0:8881 brozzler.webconsole:app >>/vagrant/logs/brozzler-webconsole.log 2>&1
-
-

vagrant/ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2

@@ -19,7 +19,5 @@ stop on stopping Xvnc
 kill timeout 60
 
 exec nice brozzler-worker \
-     --rethinkdb-servers=localhost \
+    --rethinkdb-servers={{groups['rethinkdb'] | join(',')}} \
-     --max-browsers=4 >>/vagrant/logs/brozzler-worker.log 2>&1
+    --max-browsers=4 >>/vagrant/logs/brozzler-worker.log 2>&1
-     # --rethinkdb-servers={{groups['rethinkdb'] | join(',')}} \
-

vagrant/ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2

@@ -10,5 +10,6 @@ console log
 env PYTHONPATH=/home/vagrant/websockify-ve34/lib/python3.4/site-packages
 env PATH=/home/vagrant/websockify-ve34/bin:/usr/bin:/bin
 
+# port 8901 is hard-coded in brozzler/webconsole/static/partials/workers.html
 exec nice websockify 0.0.0.0:8901 localhost:5901
 

vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-easy.conf.j2

@@ -1,5 +1,5 @@
 runuser=vagrant
-# bind=0.0.0.0
+bind=0.0.0.0
 # directory=/var/lib/rethinkdb
 # log-file=/var/log/rethinkdb.log
 log-file=/vagrant/logs/rethinkdb.log  # synced dir