working on including pywb in vagrant environment (not finished)

setup.py

@@ -32,7 +32,7 @@ def find_package_data(package):
 
 setuptools.setup(
         name='brozzler',
-        version='1.1b6.dev81',
+        version='1.1b6.dev82',
         description='Distributed web crawling with browsers',
         url='https://github.com/internetarchive/brozzler',
         author='Noah Levitt',

vagrant/ansible/hosts

@@ -13,4 +13,4 @@ ansible_ssh_private_key_file=.vagrant/machines/10.9.9.9/virtualbox/private_key
 10.9.9.9
 
 [pywb]
-10.9.9.9
+10.9.9.9

vagrant/ansible/playbook.yml

@@ -24,7 +24,7 @@
   roles:
   - brozzler-webconsole
 
-# - name: deploy pywb
-#   hosts: pywb
-#   roles:
-#     - pywb
+- name: deploy pywb
+  hosts: pywb
+  roles:
+    - pywb

vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml

@@ -1,6 +1,5 @@
 ---
 - name: install brozzler[webconsole] in virtualenv
-  become: true
   pip: name='-e /brozzler[webconsole]'
        virtualenv=/home/vagrant/brozzler-webconsole-ve34
        virtualenv_python=python3.4
@@ -12,4 +11,4 @@
   template: src=templates/brozzler-webconsole.conf.j2
             dest=/etc/init/brozzler-webconsole.conf
   notify:
-  - restart brozzler-webconsole
+  - restart brozzler-webconsole

vagrant/ansible/roles/brozzler-worker/tasks/main.yml

@@ -26,7 +26,6 @@
     - ttf-indic-fonts
     - fonts-thai-tlwg
     - fonts-lklug-sinhala
-    - python3-pip
     - git
     - libjpeg-turbo8-dev
     - zlib1g-dev
@@ -49,7 +48,6 @@
   notify:
     - restart vnc-websock
 - name: install brozzler in virtualenv
-  become: true
   pip: # name=git+https://github.com/internetarchive/brozzler.git#egg=brozzler
        name='-e /brozzler'
        virtualenv=/home/vagrant/brozzler-ve34

vagrant/ansible/roles/common/tasks/main.yml

@@ -1,4 +1,24 @@
 ---
-- name: ensure logs directory exists
+## # get latest pip (had problems with version from apt-get, specifically
+## # "pip install pyopenssl" did not install the dependency "cryptography")
+## # http://stackoverflow.com/questions/34587473/what-is-get-pip-py-checksum-where-can-i-get-it-for-sure
+## - name: install setuptools for python 2 and 3
+##   become: true
+##   apt: name={{item}} state=present
+##   with_items:
+##   - python-setuptools
+##   - python3-setuptools
+## - name: download pip-8.1.2.tar.gz
+##   get_url:
+##     url: https://pypi.python.org/packages/e7/a8/7556133689add8d1a54c0b14aeff0acb03c64707ce100ecd53934da1aa13/pip-8.1.2.tar.gz
+##     dest: /tmp
+##     checksum: sha1:1c13c247967ec5bee6de5fd104c5d78ba30951c7
+## - name: extract pip-8.1.2.tar.gz
+##   unarchive: src=/tmp/pip-8.1.2.tar.gz dest=/tmp copy=no
+## - name: run "python3 setup.py install" in /tmp/pip-8.1.2
+##   command: python3 setup.py install chdir=/tmp/pip-8.1.2
+##            creates=/usr/local/lib/python2.7/dist-packages/pip-8.1.2-py2.7.egg/pip/__init__.py
+##   become: true
+- name: mkdir /vagrant/logs
   file: path=/vagrant/logs state=directory
   become: true

vagrant/ansible/roles/pywb/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: restart pywb
+  service: name=pywb state=restarted
+  become: true
+

vagrant/ansible/roles/pywb/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: install pywb in virtualenv
+  pip: name=pywb
+       virtualenv=/home/vagrant/pywb-ve34
+       virtualenv_python=python3.4
+       extra_args='--no-input --upgrade --pre'
+  notify:
+  - restart pywb
+- name: install brozzler in pywb virtualenv
+  pip: name='-e /brozzler'
+       virtualenv=/home/vagrant/pywb-ve34
+       virtualenv_python=python3.4
+       extra_args='--no-input --upgrade --pre'
+  notify:
+  - restart pywb
+- name: pywb config file /etc/pywb.yml
+  template: src=templates/pywb.yml.j2
+            dest=/etc/pywb.yml
+  become: true
+  notify:
+  - restart pywb
+- name: upstart config file /etc/init/pywb.conf
+  template: src=templates/pywb.conf.j2
+            dest=/etc/init/pywb.conf
+  become: true
+  notify:
+  - restart pywb

vagrant/ansible/roles/pywb/templates/pywb.conf.j2

@@ -0,0 +1,14 @@
+description "pywb"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+env PYTHONPATH=/home/vagrant/pywb-ve34/lib/python3.4/site-packages
+env PATH=/home/vagrant/pywb-ve34/bin:/usr/bin:/bin
+env PYWB_CONFIG_FILE=/etc/pywb.yml
+
+setuid vagrant
+
+# console log
+
+exec nice brozzler-wayback >>/vagrant/logs/pywb.log 2>&1

vagrant/ansible/roles/pywb/templates/pywb.yml.j2

@@ -0,0 +1,12 @@
+archive_paths: /vagrant/warcs/
+collections:
+  brozzler:
+    index_paths: !!python/object:brozzler.pywb.RethinkCDXSource
+      db: brozzler
+      servers: [localhost]
+      table: captures
+enable_auto_colls: false
+enable_cdx_api: true
+framed_replay: true
+port: 8880
+

vagrant/ansible/roles/rethinkdb/tasks/main.yml

@@ -10,12 +10,14 @@
   apt: name=rethinkdb state=present
   become: true
   notify:
-    - restart rethinkdb
+  - restart rethinkdb
+# XXX rethinkdb fails to start in spite of this, I think because /vagrant
+# gets mounted too late, and it tries to log there
 - name: ensure rethinkdb starts on reboot
   service: name=rethinkdb enabled=yes
 - name: ensure rethinkdb instance config file is installed
-  template: src=templates/rethinkdb-brozzler-easy.conf.j2
-            dest=/etc/rethinkdb/instances.d/rethinkdb-brozzler-easy.conf
+  template: src=templates/rethinkdb-brozzler-vagrant-1.conf.j2
+            dest=/etc/rethinkdb/instances.d/rethinkdb-brozzler-vagrant-1.conf
   become: true
   notify:
-    - restart rethinkdb
+  - restart rethinkdb

vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-easy.conf.j2

@@ -1,5 +0,0 @@
-runuser=vagrant
-bind=0.0.0.0
-# directory=/var/lib/rethinkdb
-# log-file=/var/log/rethinkdb.log
-log-file=/vagrant/logs/rethinkdb.log  # synced dir

vagrant/ansible/roles/warcprox/tasks/main.yml

@@ -3,23 +3,23 @@
   become: true
   apt: name={{item}} state=present
   with_items:
-    - gcc
-    - python-virtualenv
-    - python3.4
-    - libpython3.4-dev
-    - libffi-dev
-    - libssl-dev
-    - tor
-    - git
+  - gcc
+  - python-virtualenv
+  - python3.4
+  - libpython3.4-dev
+  - libffi-dev
+  - libssl-dev
+  - tor
+  - git
 - name: install warcprox in virtualenv
   pip: name=git+https://github.com/internetarchive/warcprox.git@2.x#egg=warcprox
        virtualenv=/home/vagrant/warcprox-ve34
        virtualenv_python=python3.4
        extra_args='--no-input --upgrade --pre'
   notify:
-    - restart warcprox
+  - restart warcprox
 - name: install upstart config /etc/init/warcprox.conf
   become: true
   template: src=templates/warcprox.conf.j2 dest=/etc/init/warcprox.conf
   notify:
-    - restart warcprox
+  - restart warcprox