porting ansible config to xenial

no more upstart, switch to daemontools, among other things

.travis.yml

@@ -21,9 +21,9 @@ script:
 - DISPLAY=:1 py.test --tb=native -v tests
 after_failure:
 - chromium-browser --version
-- sudo cat /var/log/upstart/warcprox.log
-- sudo cat /var/log/upstart/brozzler-worker.log
-- sudo cat /var/log/upstart/pywb.log
+- sudo cat /var/log/warcprox.log
+- sudo cat /var/log/brozzler-worker.log
+- sudo cat /var/log/pywb.log
 notifications:
   slack:
     secure: KPPXSscXnmSEQ2NXBZFKrzDEYHg067Kv1WR7RTRUH8EIlSS9MHTyErRa7HkaRPmqOllj4vvPbplNU2ALnCfhP4cqW+MvF0xv3GuEGXQ7Om2sBvVUQ3w0JJ5rLq9ferAfGdSnQFeViqfDix5LA3fMNZGouUHQdUHq7iO8E9n9jntvkKO9Jff7Dyo0K5KvOZOJfM9KsqFZLlFO5zoNB6Y9jubIT7+Ulk3EDto/Kny34VPIyJIm7y0cHHlYLEq780AweY0EIwMyMg/VPSRrVAsbLSrilO0YRgsQpjPC9Ci/rAWNWooaOk0eA+bwv1uHQnGtH0z446XUMXr3UZ2QlD4DE/uoP2okkl8EtqvlmEyjV8eO86TqYFDRgKfYpvlK6hHtb7SAHX28QeXQjbKNc5f7KpKO5PtZqaoBRL7acLlKyS8xQGiRtonTPFSBTFR2A+s6dZmKO9dDboglptiHk4dvL1ZD4S8qLJn1JjTJqvIU6tpCY3BpNErn4n1MkDjN5nqdXf7Q9Vmui8vRetwnMf1oXcsKj9FEt2utNfDqFNXcFsN+Mnr9rhXQ1++gt/7Zo844OowiARcxqZTNy5LqSD01WgGCvNMy3Odf+FTQ8PcDOF+001+g8La1R99U0o9/hT/gy+WYk2prYneWru4pQHF/a6goZgkLTwkskcaPVpDJtDs=

ansible/roles/brozzler-dashboard/handlers/main.yml

@@ -1,4 +1,8 @@
 ---
 - name: restart brozzler-dashboard
-  service: name=brozzler-dashboard state=restarted
+  svc:
+    name: brozzler-dashboard
+    state: restarted
+    service_dir: /etc/service
   become: true
+

ansible/roles/brozzler-dashboard/tasks/main.yml

@@ -3,18 +3,31 @@
   file: path={{venv_root}}/brozzler-dashboard-ve3 state=directory
         owner={{user}}
   become: true
+
 - name: install brozzler[dashboard] in virtualenv
-  pip: name='{{brozzler_pip_name}}[dashboard]'
-       virtualenv={{venv_root}}/brozzler-dashboard-ve3
-       virtualenv_python=python3
-       extra_args='--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
+  pip:
+    name: '{{brozzler_pip_name}}[dashboard]'
+    virtualenv: '{{venv_root}}/brozzler-dashboard-ve3'
+    virtualenv_python: python3
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
+    extra_args: '--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
   become: true
   become_user: '{{user}}'
   notify:
   - restart brozzler-dashboard
-- name: install upstart config /etc/init/brozzler-dashboard.conf
+
+- name: mkdir /etc/service/brozzler-dashboard
+  file:
+    path: /etc/service/brozzler-dashboard
+    state: directory
   become: true
-  template: src=templates/brozzler-dashboard.conf.j2
-            dest=/etc/init/brozzler-dashboard.conf
+
+- name: install /etc/service/brozzler-dashboard/run
+  template:
+    src: templates/brozzler-dashboard-run.j2
+    dest: /etc/service/brozzler-dashboard/run
+    mode: 0755
   notify:
   - restart brozzler-dashboard
+  become: true
+

ansible/roles/brozzler-dashboard/templates/brozzler-dashboard-run.j2

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+logfile=/var/log/brozzler-dashboard.log
+touch $logfile
+chown {{user}} $logfile
+
+exec nice setuidgid {{user}} \
+    env WAYBACK_BASEURL=http://{{groups['pywb'][0]}}:8880/brozzler \
+        RETHINKDB_SERVERS={{groups['rethinkdb'] | join(',')}} \
+        RETHINKDB_DB=brozzler LANG=en_US.UTF-8 LC_COLLATE=C \
+    gunicorn --bind=0.0.0.0:8881 brozzler.dashboard:app \
+    >> $logfile 2>&1
+

ansible/roles/brozzler-dashboard/templates/brozzler-dashboard.conf.j2

@@ -1,17 +0,0 @@
-description "brozzler-dashboard"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-env PATH={{venv_root}}/brozzler-dashboard-ve3/bin:/usr/bin:/bin
-env LC_ALL=C.UTF-8
-
-env WAYBACK_BASEURL=http://{{groups['pywb'][0]}}:8880/brozzler
-env RETHINKDB_SERVERS={{groups['rethinkdb'] | join(',')}}
-env RETHINKDB_DB=brozzler
-
-setuid {{user}}
-
-console log
-
-exec gunicorn --bind=0.0.0.0:8881 brozzler.dashboard:app

ansible/roles/brozzler-worker/handlers/main.yml

@@ -1,13 +1,22 @@
 ---
 - name: restart Xvnc
-  service: name=Xvnc state=restarted
-  become: true
-- name: restart websockify
-  service: name=websockify state=restarted
+  svc:
+    name: Xvnc
+    state: restarted
+    service_dir: /etc/service
   become: true
+
 - name: restart vnc-websock
-  service: name=vnc-websock state=restarted
+  svc:
+    name: vnc-websock
+    state: restarted
+    service_dir: /etc/service
   become: true
+
 - name: restart brozzler-worker
-  service: name=brozzler-worker state=restarted
+  svc:
+    name: brozzler-worker
+    state: restarted
+    service_dir: /etc/service
   become: true
+

ansible/roles/brozzler-worker/tasks/main.yml

@@ -3,8 +3,10 @@
   apt_repository: repo='deb http://archive.canonical.com/ubuntu trusty partner'
                   state=present
   become: true
+
 - apt: update_cache=yes
   become: true
+
 - name: ensure required packages are installed
   become: true
   apt: name={{item}} state=present
@@ -33,41 +35,72 @@
   - fonts-indic
   - fonts-thai-tlwg
   - fonts-lklug-sinhala
-- name: install Xvnc upstart config /etc/init/Xvnc.conf
-  template: src=templates/Xvnc.conf.j2 dest=/etc/init/Xvnc.conf
+
+- name: mkdir /etc/service/warcprox
+  file:
+    path: '/etc/service/{{item}}'
+    state: directory
+  with_items:
+  - Xvnc
+  - websockify
+  - vnc-websock
+  - brozzler-worker
   become: true
+
+- name: install /etc/service/Xvnc/run
+  template:
+    src: templates/Xvnc-run.j2
+    dest: /etc/service/Xvnc/run
+    mode: 0755
   notify:
   - restart Xvnc
+  become: true
+
 - name: mkdir {{venv_root}}/websockify-ve3
   become: true
   file: path={{venv_root}}/websockify-ve3 state=directory owner={{user}}
+
 - name: install websockify in virtualenv
-  pip: name=git+https://github.com/kanaka/websockify.git#egg=websockify
-       virtualenv={{venv_root}}/websockify-ve3
-       virtualenv_python=python3
-       extra_args='--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
+  pip:
+    name: git+https://github.com/kanaka/websockify.git#egg=websockify
+    virtualenv: '{{venv_root}}/websockify-ve3'
+    virtualenv_python: python3
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
+    extra_args: '--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
   become: true
   become_user: '{{user}}'
-- name: install vnc-websock upstart config /etc/init/vnc-websock.conf
-  template: src=templates/vnc-websock.conf.j2 dest=/etc/init/vnc-websock.conf
-  become: true
+
+- name: install /etc/service/vnc-websock/run
+  template:
+    src: templates/vnc-websock-run.j2
+    dest: /etc/service/vnc-websock/run
+    mode: 0755
   notify:
   - restart vnc-websock
+  become: true
+
 - name: mkdir {{venv_root}}/brozzler-ve3
   become: true
   file: path={{venv_root}}/brozzler-ve3 state=directory owner={{user}}
+
 - name: install brozzler in virtualenv
   pip:
     name: '{{brozzler_pip_name}}'
     virtualenv: '{{venv_root}}/brozzler-ve3'
     virtualenv_python: python3
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
     extra_args: '--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
   become: true
   become_user: '{{user}}'
   notify:
   - restart brozzler-worker
-- name: install brozzler-worker upstart config /etc/init/brozzler-worker.conf
-  template: src=templates/brozzler-worker.conf.j2 dest=/etc/init/brozzler-worker.conf
-  become: true
+
+- name: install /etc/service/brozzler-worker/run
+  template:
+    src: templates/brozzler-worker-run.j2
+    dest: /etc/service/brozzler-worker/run
+    mode: 0755
   notify:
   - restart brozzler-worker
+  become: true
+

ansible/roles/brozzler-worker/templates/Xvnc-run.j2

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+cd /tmp
+
+logfile=/var/log/Xvnc.log
+touch $logfile
+chown {{user}} $logfile
+
+exec nice setuidgid {{user}} Xvnc4 :1 -auth /tmp/Xauthority.{{user}} \
+    -geometry 1600x1000 -depth 24 -rfbwait 0 -nolisten tcp -rfbport 5901 \
+    -SecurityTypes None -pn -fp /usr/share/fonts/X11/misc/ -co /etc/X11/rgb \
+    AcceptCutText=0 AcceptPointerEvents=0 AcceptKeyEvents=0 \
+    >> $logfile 2>&1
+

ansible/roles/brozzler-worker/templates/Xvnc.conf.j2

@@ -1,14 +0,0 @@
-description "Xvnc"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-setuid {{user}}
-
-console log
-
-exec nice Xvnc4 :1 -auth /tmp/Xauthority.{{user}} \
-    -geometry 1600x1000 -depth 24 -rfbwait 0 -nolisten tcp -rfbport 5901 \
-    -SecurityTypes None -pn -fp /usr/share/fonts/X11/misc/ -co /etc/X11/rgb \
-    AcceptCutText=0 AcceptPointerEvents=0 AcceptKeyEvents=0
-

ansible/roles/brozzler-worker/templates/brozzler-worker-run.j2

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+logfile=/var/log/brozzler-worker.log
+touch $logfile
+chown {{user}} $logfile
+
+source {{venv_root}}/brozzler-ve3/bin/activate
+
+exec nice setuidgid {{user}} \
+    env LANG=en_US.UTF-8 LC_COLLATE=C \
+    brozzler-worker \
+     --rethinkdb-servers={{groups['rethinkdb'] | join(',')}} \
+     --max-browsers=4 \
+     --verbose \
+     --warcprox-auto \
+     >> $logfile 2>&1
+

ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2

@@ -1,24 +0,0 @@
-description "brozzler-worker"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-env DISPLAY=:1
-env PATH={{venv_root}}/brozzler-ve3/bin:/usr/bin:/bin
-env LANG=C.UTF-8
-
-setuid {{user}}
-
-console log
-
-# depends on vnc server
-start on started Xvnc
-stop on stopping Xvnc
-
-kill timeout 60
-
-exec nice brozzler-worker \
-    --rethinkdb-servers={{groups['rethinkdb'] | join(',')}} \
-    --max-browsers=4 \
-    --verbose \
-    --warcprox-auto

ansible/roles/brozzler-worker/templates/vnc-websock-run.j2

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+logfile=/var/log/vnc-websock.log
+touch $logfile
+chown {{user}} $logfile
+
+source /opt/websockify-ve3/bin/activate
+
+exec nice setuidgid {{user}} websockify 0.0.0.0:8901 localhost:5901 >> $logfile 2>&1
+

ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2

@@ -1,14 +0,0 @@
-description "vnc-websock"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-setuid {{user}}
-
-console log
-
-env PATH={{venv_root}}/websockify-ve3/bin:/usr/bin:/bin
-
-# port 8901 is hard-coded in brozzler/dashboard/static/partials/workers.html
-exec nice websockify 0.0.0.0:8901 localhost:5901
-

ansible/roles/common/tasks/main.yml

@@ -1,44 +1,74 @@
 ---
-# get latest pip (had problems with version from apt-get, specifically
-# "pip install pyopenssl" did not install the dependency "cryptography")
-# http://stackoverflow.com/questions/34587473/what-is-get-pip-py-checksum-where-can-i-get-it-for-sure
-- name: install setuptools for python 2 and 3
+- apt:
+    name:
+    - python3-setuptools
+    - python3-pip
+    - python3-virtualenv
+    - daemontools
+    - daemontools-run
+    state: present
+    update_cache: yes
+    cache_valid_time: 86400 # one day
   become: true
-  apt: name={{item}} state=present
-  with_items:
-  - python-setuptools
-  - python3-setuptools
-- name: download pip-9.0.1.tar.gz
-  get_url:
-    url: https://pypi.python.org/packages/11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/pip-9.0.1.tar.gz
-    dest: /tmp
-    checksum: sha1:57ff41e99cb01b6a1c2b0999161589b726f0ec8b
-- name: extract pip-9.0.1.tar.gz
-  unarchive: src=/tmp/pip-9.0.1.tar.gz dest=/tmp copy=no
+
+# # get recent virtualenv, which bundles a recent pip
+# - find:
+#     paths:
+#     - /usr/local/lib/python3.4/dist-packages
+#     - /usr/local/lib/python3.5/dist-packages
+#     recurse: true
+#     patterns: virtualenv.py
+#     contains: '__version__ = "16.4.3"'
+#   register: virtualenv_py_16_4_3
+# 
+# - command: mktemp -d
+#   register: mktempd_out
+#   when: virtualenv_py_16_4_3.matched == 0
+# 
+# - name: download virtualenv-16.4.3
+#   get_url:
+#     url: https://files.pythonhosted.org/packages/37/db/89d6b043b22052109da35416abc3c397655e4bd3cff031446ba02b9654fa/virtualenv-16.4.3.tar.gz
+#     dest: '{{mktempd_out.stdout}}'
+#     checksum: sha256:984d7e607b0a5d1329425dd8845bd971b957424b5ba664729fab51ab8c11bc39
+#   when: virtualenv_py_16_4_3.matched == 0
+# 
+# - name: extract virtualenv-16.4.3.tar.gz
+#   unarchive:
+#     src: '{{mktempd_out.stdout}}/virtualenv-16.4.3.tar.gz'
+#     dest: '{{mktempd_out.stdout}}'
+#     copy: no
+#   when: virtualenv_py_16_4_3.matched == 0
+# 
+# - name: run "python3 setup.py install" in {{mktempd_out.stdout}}/virtualenv-16.4.3
+#   become: true
+#   command: python3 setup.py install
+#   args:
+#     chdir: '{{mktempd_out.stdout}}/virtualenv-16.4.3'
+#   when: virtualenv_py_16_4_3.matched == 0
+# 
+# - file:
+#     path: '{{mktempd_out.stdout}}'
+#     state: absent
+#   become: true
+#   when: virtualenv_py_16_4_3.matched == 0
 
 # this clause is a workaround for travis-ci, which only wants to install in /usr
 # see https://travis-ci.org/internetarchive/brozzler/builds/174338601
 # but it complains that /usr/lib/python3.5/site-packages doesn't exist
 # see https://travis-ci.org/internetarchive/brozzler/builds/174094831
-- file: path={{item}} state=directory
+- file:
+    path: '{{item}}'
+    state: directory
   with_items:
   - /usr/lib/python3.5/site-packages
   - /usr/lib/python3.5/dist-packages
   become: true
 
-- name: run "python3 setup.py install" in /tmp/pip-9.0.1
-  command: python3 setup.py install
-           chdir=/tmp/pip-9.0.1
-           creates=/usr/local/lib/python3.5/dist-packages/pip-9.0.1-py3.5.egg/pip/__init__.py
-  become: true
-- name: run "pip install virtualenv"
-  command: pip install virtualenv
-           creates=/usr/local/lib/python3.5/dist-packages/virtualenv.py
-  become: true
 - command: id {{user}}
   register: id_user
   ignore_errors: true
   changed_when: false
+
 - name: ensure service user {{user}} exists
   user: name={{user}} system=yes createhome=no home=/nonexistent
         shell=/usr/sbin/nologin

ansible/roles/pywb/handlers/main.yml

@@ -1,5 +1,9 @@
 ---
 - name: restart pywb
-  service: name=pywb state=restarted
+  svc:
+    name: pywb
+    state: restarted
+    service_dir: /etc/service
   become: true
 
+

ansible/roles/pywb/tasks/main.yml

@@ -3,34 +3,50 @@
   file: path={{venv_root}}/pywb-ve3 state=directory
         owner={{user}}
   become: true
+
 - name: install pywb in virtualenv
-  pip: name=pywb
-       version=0.33.2
-       virtualenv={{venv_root}}/pywb-ve3
-       virtualenv_python=python3
-       extra_args='--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
+  pip:
+    name: pywb
+    version: 0.33.2
+    virtualenv: '{{venv_root}}/pywb-ve3'
+    virtualenv_python: python3
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
+    extra_args: '--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
   become: true
   become_user: '{{user}}'
   notify:
   - restart pywb
+
 - name: install brozzler in pywb virtualenv
-  pip: name='{{brozzler_pip_name}}'
-       virtualenv={{venv_root}}/pywb-ve3
-       virtualenv_python=python3
-       extra_args='--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
+  pip:
+    name: '{{brozzler_pip_name}}'
+    virtualenv: '{{venv_root}}/pywb-ve3'
+    virtualenv_python: python3
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
+    extra_args: '--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
   become: true
   become_user: '{{user}}'
   notify:
   - restart pywb
+
 - name: pywb config file /etc/pywb.yml
   template: src=templates/pywb.yml.j2
             dest=/etc/pywb.yml
   become: true
   notify:
   - restart pywb
-- name: upstart config file /etc/init/pywb.conf
-  template: src=templates/pywb.conf.j2
-            dest=/etc/init/pywb.conf
+
+- name: mkdir /etc/service/pywb
+  file:
+    path: /etc/service/pywb
+    state: directory
   become: true
+
+- name: install /etc/service/pywb/run
+  template:
+    src: templates/pywb-run.j2
+    dest: /etc/service/pywb/run
+    mode: 0755
   notify:
   - restart pywb
+  become: true

ansible/roles/pywb/templates/pywb-run.j2

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+logfile=/var/log/pywb.log
+touch $logfile
+chown {{user}} $logfile
+
+exec nice setuidgid {{user}} env PYWB_CONFIG_FILE=/etc/pywb.yml \
+    {{venv_root}}/pywb-ve3/bin/python {{venv_root}}/pywb-ve3/bin/brozzler-wayback \
+    >> $logfile 2>&1
+

ansible/roles/pywb/templates/pywb.conf.j2

@@ -1,12 +0,0 @@
-description "pywb"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-env PYWB_CONFIG_FILE=/etc/pywb.yml
-
-setuid {{user}}
-
-console log
-
-exec nice {{venv_root}}/pywb-ve3/bin/python {{venv_root}}/pywb-ve3/bin/brozzler-wayback

ansible/roles/warcprox/handlers/main.yml

@@ -1,4 +1,7 @@
 ---
 - name: restart warcprox
-  service: name=warcprox state=restarted
+  svc:
+    name: warcprox
+    state: restarted
+    service_dir: /etc/service
   become: true

ansible/roles/warcprox/tasks/main.yml

@@ -13,16 +13,28 @@
   become: true
   file: path={{venv_root}}/warcprox-ve3 state=directory owner={{user}}
 - name: install warcprox in virtualenv
-  pip: name=git+https://github.com/internetarchive/warcprox.git#egg=warcprox
-       virtualenv={{venv_root}}/warcprox-ve3
-       virtualenv_python=python3
-       extra_args='--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
+  pip:
+    name: git+https://github.com/internetarchive/warcprox.git#egg=warcprox
+    virtualenv: '{{venv_root}}/warcprox-ve3'
+    virtualenv_python: python3
+    extra_args: --no-input --upgrade --pre --cache-dir=/tmp/pip-cache
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
   become: true
   become_user: '{{user}}'
   notify:
   - restart warcprox
-- name: install upstart config /etc/init/warcprox.conf
+
+- name: mkdir /etc/service/warcprox
+  file:
+    path: /etc/service/warcprox
+    state: directory
   become: true
-  template: src=templates/warcprox.conf.j2 dest=/etc/init/warcprox.conf
+
+- name: install /etc/service/warcprox/run
+  template:
+    src: templates/run.j2
+    dest: /etc/service/warcprox/run
+    mode: 0755
   notify:
   - restart warcprox
+  become: true

ansible/roles/warcprox/templates/run.j2

@@ -1,16 +1,15 @@
-description "warcprox"
+#!/bin/bash
 
-start on runlevel [2345]
-stop on runlevel [!2345]
 
-# by default warcprox creates some files/dirs relative to cwd
-chdir {{work_dir}}
-setuid {{user}}
+logfile=/var/log/warcprox.log
+touch $logfile
+chown {{user}} $logfile
 
-console log
+ulimit -n 4096
 
-# --profile
-exec nice {{venv_root}}/warcprox-ve3/bin/python {{venv_root}}/warcprox-ve3/bin/warcprox \
+source {{venv_root}}/warcprox-ve3/bin/activate
+
+exec nice -n5 setuidgid {{user}} env LANG=en_US.UTF-8 LC_COLLATE=C warcprox \
          --address=0.0.0.0 \
          --dir={{warcs_dir}} \
          --base32 \
@@ -19,4 +18,6 @@ exec nice {{venv_root}}/warcprox-ve3/bin/python {{venv_root}}/warcprox-ve3/bin/w
          --onion-tor-socks-proxy=localhost:9050 \
          --rethinkdb-services-url=rethinkdb://{{groups['rethinkdb']|join(',')}}/brozzler/services \
          --rethinkdb-stats-url=rethinkdb://{{groups['rethinkdb']|join(',')}}/brozzler/stats \
-         --rethinkdb-big-table-url=rethinkdb://{{groups['rethinkdb']|join(',')}}/brozzler/captures
+         --rethinkdb-big-table-url=rethinkdb://{{groups['rethinkdb']|join(',')}}/brozzler/captures \
+         >> $logfile 2>&1
+

vagrant/README.rst

@@ -24,27 +24,27 @@ the brozzler virtualenv.
 ::
 
     my-laptop$ vagrant ssh
-    vagrant@brzl:~$ source /opt/brozzler-ve34/bin/activate
-    (brozzler-ve34)vagrant@brzl:~$
+    vagrant@brzl:~$ source /opt/brozzler-ve3/bin/activate
+    (brozzler-ve3)vagrant@brzl:~$
 
 Then you can run brozzler-new-site:
 
 ::
 
-    (brozzler-ve34)vagrant@brzl:~$ brozzler-new-site --proxy=localhost:8000 http://example.com/
+    (brozzler-ve3)vagrant@brzl:~$ brozzler-new-site --proxy=localhost:8000 http://example.com/
 
 
 Or brozzler-new-job (make sure to set the proxy to localhost:8000):
 
 ::
 
-    (brozzler-ve34)vagrant@brzl:~$ cat >job1.yml <<EOF
+    (brozzler-ve3)vagrant@brzl:~$ cat >job1.yml <<EOF
     id: job1
     proxy: localhost:8000 # point at warcprox for archiving
     seeds:
     - url: https://example.org/
     EOF
-    (brozzler-ve34)vagrant@brzl:~$ brozzler-new-job job1.yml
+    (brozzler-ve3)vagrant@brzl:~$ brozzler-new-job job1.yml
 
 WARC files will appear in ./warcs and brozzler, warcprox and rethinkdb logs in
 ./logs (via vagrant folders syncing).

vagrant/Vagrantfile

@@ -1,8 +1,9 @@
 Vagrant.configure(2) do |config|
-  config.vm.box = "ubuntu/trusty64"
+  config.vm.box = "ubuntu/xenial64"
   config.vm.define "10.9.9.9"
   config.vm.hostname = "brzl"
   config.vm.network :private_network, ip: "10.9.9.9"
+  config.disksize.size = '50GB'
 
   config.vm.synced_folder "..", "/brozzler"
 
@@ -14,6 +15,7 @@ Vagrant.configure(2) do |config|
   config.vm.provision "ansible" do |ansible|
     ansible.inventory_path = "../ansible/hosts-vagrant"
     ansible.playbook = "../ansible/playbook.yml"
+    # ansible.verbose = "-vvv"
   end
 
   config.vm.provider 'virtualbox' do |v|

vagrant/run-tests.sh

@@ -17,5 +17,5 @@ vagrant ssh -- 'status warcprox ;
                 status vnc-websock'
 echo
 
-vagrant ssh -- 'set -x ; source /opt/brozzler-ve34/bin/activate && pip install pytest && pip install --upgrade --pre "warcprox>=2.1b1.dev86"'
-vagrant ssh -- "source /opt/brozzler-ve34/bin/activate && DISPLAY=:1 py.test -v /brozzler/tests $@"
+vagrant ssh -- 'set -x ; source /opt/brozzler-ve3/bin/activate && pip install pytest && pip install --upgrade --pre "warcprox>=2.1b1.dev86"'
+vagrant ssh -- "source /opt/brozzler-ve3/bin/activate && DISPLAY=:1 py.test -v /brozzler/tests $@"

vagrant/vagrant-brozzler-new-job.py

@@ -7,7 +7,7 @@ This is a standalone script with no dependencies other than python, and should
 work with python 2.7 or python 3.2+. The only reason it's not a bash script is
 so we can use the argparse library.
 
-Copyright (C) 2016 Internet Archive
+Copyright (C) 2016-2019 Internet Archive
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -41,9 +41,8 @@ def main(argv=[]):
         subprocess.call([
             'vagrant', 'ssh', '--',
             'f=`mktemp` && cat > $f && '
-            'PYTHONPATH=/home/vagrant/brozzler-ve34/lib/python3.4/site-packages '
-            '/home/vagrant/brozzler-ve34/bin/python '
-            '/home/vagrant/brozzler-ve34/bin/brozzler-new-job $f'],
+            '/home/vagrant/brozzler-ve3/bin/python '
+            '/home/vagrant/brozzler-ve3/bin/brozzler-new-job $f'],
             stdin=f)
 
 if __name__ == '__main__':

vagrant/vagrant-brozzler-new-site.py

@@ -74,11 +74,9 @@ def main(argv=[]):
     os.chdir(os.path.dirname(__file__))
 
     cmd = (
-        'PYTHONPATH=/home/vagrant/brozzler-ve34/lib/python3.4/site-packages '
-        '/home/vagrant/brozzler-ve34/bin/python '
-        '/home/vagrant/brozzler-ve34/bin/brozzler-new-site '
-        '--proxy=localhost:8000 %s %s') % (
-                ' '.join(options), args.seed)
+        '/home/vagrant/brozzler-ve3/bin/python '
+        '/home/vagrant/brozzler-ve3/bin/brozzler-new-site '
+        '--proxy=localhost:8000 %s %s') % (' '.join(options), args.seed)
     subprocess.call(['vagrant', 'ssh', '--', cmd])
 
 if __name__ == '__main__':