porting ansible config to xenial

no more upstart, switch to daemontools, among other things

ansible/roles/brozzler-dashboard/handlers/main.yml

@@ -1,4 +1,8 @@
 ---
 - name: restart brozzler-dashboard
-  service: name=brozzler-dashboard state=restarted
+  svc:
+    name: brozzler-dashboard
+    state: restarted
+    service_dir: /etc/service
   become: true
+

ansible/roles/brozzler-dashboard/tasks/main.yml

@@ -3,18 +3,31 @@
   file: path={{venv_root}}/brozzler-dashboard-ve3 state=directory
         owner={{user}}
   become: true
+
 - name: install brozzler[dashboard] in virtualenv
-  pip: name='{{brozzler_pip_name}}[dashboard]'
-       virtualenv={{venv_root}}/brozzler-dashboard-ve3
-       virtualenv_python=python3
-       extra_args='--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
+  pip:
+    name: '{{brozzler_pip_name}}[dashboard]'
+    virtualenv: '{{venv_root}}/brozzler-dashboard-ve3'
+    virtualenv_python: python3
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
+    extra_args: '--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
   become: true
   become_user: '{{user}}'
   notify:
   - restart brozzler-dashboard
-- name: install upstart config /etc/init/brozzler-dashboard.conf
+
+- name: mkdir /etc/service/brozzler-dashboard
+  file:
+    path: /etc/service/brozzler-dashboard
+    state: directory
   become: true
-  template: src=templates/brozzler-dashboard.conf.j2
-            dest=/etc/init/brozzler-dashboard.conf
+
+- name: install /etc/service/brozzler-dashboard/run
+  template:
+    src: templates/brozzler-dashboard-run.j2
+    dest: /etc/service/brozzler-dashboard/run
+    mode: 0755
   notify:
   - restart brozzler-dashboard
+  become: true
+

ansible/roles/brozzler-dashboard/templates/brozzler-dashboard-run.j2

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+logfile=/var/log/brozzler-dashboard.log
+touch $logfile
+chown {{user}} $logfile
+
+exec nice setuidgid {{user}} \
+    env WAYBACK_BASEURL=http://{{groups['pywb'][0]}}:8880/brozzler \
+        RETHINKDB_SERVERS={{groups['rethinkdb'] | join(',')}} \
+        RETHINKDB_DB=brozzler LANG=en_US.UTF-8 LC_COLLATE=C \
+    gunicorn --bind=0.0.0.0:8881 brozzler.dashboard:app \
+    >> $logfile 2>&1
+

ansible/roles/brozzler-dashboard/templates/brozzler-dashboard.conf.j2

@@ -1,17 +0,0 @@
-description "brozzler-dashboard"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-env PATH={{venv_root}}/brozzler-dashboard-ve3/bin:/usr/bin:/bin
-env LC_ALL=C.UTF-8
-
-env WAYBACK_BASEURL=http://{{groups['pywb'][0]}}:8880/brozzler
-env RETHINKDB_SERVERS={{groups['rethinkdb'] | join(',')}}
-env RETHINKDB_DB=brozzler
-
-setuid {{user}}
-
-console log
-
-exec gunicorn --bind=0.0.0.0:8881 brozzler.dashboard:app

ansible/roles/brozzler-worker/handlers/main.yml

@@ -1,13 +1,22 @@
 ---
 - name: restart Xvnc
-  service: name=Xvnc state=restarted
-  become: true
-- name: restart websockify
-  service: name=websockify state=restarted
+  svc:
+    name: Xvnc
+    state: restarted
+    service_dir: /etc/service
   become: true
+
 - name: restart vnc-websock
-  service: name=vnc-websock state=restarted
+  svc:
+    name: vnc-websock
+    state: restarted
+    service_dir: /etc/service
   become: true
+
 - name: restart brozzler-worker
-  service: name=brozzler-worker state=restarted
+  svc:
+    name: brozzler-worker
+    state: restarted
+    service_dir: /etc/service
   become: true
+

ansible/roles/brozzler-worker/tasks/main.yml

@@ -3,8 +3,10 @@
   apt_repository: repo='deb http://archive.canonical.com/ubuntu trusty partner'
                   state=present
   become: true
+
 - apt: update_cache=yes
   become: true
+
 - name: ensure required packages are installed
   become: true
   apt: name={{item}} state=present
@@ -33,41 +35,72 @@
   - fonts-indic
   - fonts-thai-tlwg
   - fonts-lklug-sinhala
-- name: install Xvnc upstart config /etc/init/Xvnc.conf
-  template: src=templates/Xvnc.conf.j2 dest=/etc/init/Xvnc.conf
+
+- name: mkdir /etc/service/warcprox
+  file:
+    path: '/etc/service/{{item}}'
+    state: directory
+  with_items:
+  - Xvnc
+  - websockify
+  - vnc-websock
+  - brozzler-worker
   become: true
+
+- name: install /etc/service/Xvnc/run
+  template:
+    src: templates/Xvnc-run.j2
+    dest: /etc/service/Xvnc/run
+    mode: 0755
   notify:
   - restart Xvnc
+  become: true
+
 - name: mkdir {{venv_root}}/websockify-ve3
   become: true
   file: path={{venv_root}}/websockify-ve3 state=directory owner={{user}}
+
 - name: install websockify in virtualenv
-  pip: name=git+https://github.com/kanaka/websockify.git#egg=websockify
-       virtualenv={{venv_root}}/websockify-ve3
-       virtualenv_python=python3
-       extra_args='--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
+  pip:
+    name: git+https://github.com/kanaka/websockify.git#egg=websockify
+    virtualenv: '{{venv_root}}/websockify-ve3'
+    virtualenv_python: python3
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
+    extra_args: '--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
   become: true
   become_user: '{{user}}'
-- name: install vnc-websock upstart config /etc/init/vnc-websock.conf
-  template: src=templates/vnc-websock.conf.j2 dest=/etc/init/vnc-websock.conf
-  become: true
+
+- name: install /etc/service/vnc-websock/run
+  template:
+    src: templates/vnc-websock-run.j2
+    dest: /etc/service/vnc-websock/run
+    mode: 0755
   notify:
   - restart vnc-websock
+  become: true
+
 - name: mkdir {{venv_root}}/brozzler-ve3
   become: true
   file: path={{venv_root}}/brozzler-ve3 state=directory owner={{user}}
+
 - name: install brozzler in virtualenv
   pip:
     name: '{{brozzler_pip_name}}'
     virtualenv: '{{venv_root}}/brozzler-ve3'
     virtualenv_python: python3
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
     extra_args: '--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
   become: true
   become_user: '{{user}}'
   notify:
   - restart brozzler-worker
-- name: install brozzler-worker upstart config /etc/init/brozzler-worker.conf
-  template: src=templates/brozzler-worker.conf.j2 dest=/etc/init/brozzler-worker.conf
-  become: true
+
+- name: install /etc/service/brozzler-worker/run
+  template:
+    src: templates/brozzler-worker-run.j2
+    dest: /etc/service/brozzler-worker/run
+    mode: 0755
   notify:
   - restart brozzler-worker
+  become: true
+

ansible/roles/brozzler-worker/templates/Xvnc-run.j2

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+cd /tmp
+
+logfile=/var/log/Xvnc.log
+touch $logfile
+chown {{user}} $logfile
+
+exec nice setuidgid {{user}} Xvnc4 :1 -auth /tmp/Xauthority.{{user}} \
+    -geometry 1600x1000 -depth 24 -rfbwait 0 -nolisten tcp -rfbport 5901 \
+    -SecurityTypes None -pn -fp /usr/share/fonts/X11/misc/ -co /etc/X11/rgb \
+    AcceptCutText=0 AcceptPointerEvents=0 AcceptKeyEvents=0 \
+    >> $logfile 2>&1
+

ansible/roles/brozzler-worker/templates/Xvnc.conf.j2

@@ -1,14 +0,0 @@
-description "Xvnc"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-setuid {{user}}
-
-console log
-
-exec nice Xvnc4 :1 -auth /tmp/Xauthority.{{user}} \
-    -geometry 1600x1000 -depth 24 -rfbwait 0 -nolisten tcp -rfbport 5901 \
-    -SecurityTypes None -pn -fp /usr/share/fonts/X11/misc/ -co /etc/X11/rgb \
-    AcceptCutText=0 AcceptPointerEvents=0 AcceptKeyEvents=0
-

ansible/roles/brozzler-worker/templates/brozzler-worker-run.j2

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+logfile=/var/log/brozzler-worker.log
+touch $logfile
+chown {{user}} $logfile
+
+source {{venv_root}}/brozzler-ve3/bin/activate
+
+exec nice setuidgid {{user}} \
+    env LANG=en_US.UTF-8 LC_COLLATE=C \
+    brozzler-worker \
+     --rethinkdb-servers={{groups['rethinkdb'] | join(',')}} \
+     --max-browsers=4 \
+     --verbose \
+     --warcprox-auto \
+     >> $logfile 2>&1
+

ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2

@@ -1,24 +0,0 @@
-description "brozzler-worker"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-env DISPLAY=:1
-env PATH={{venv_root}}/brozzler-ve3/bin:/usr/bin:/bin
-env LANG=C.UTF-8
-
-setuid {{user}}
-
-console log
-
-# depends on vnc server
-start on started Xvnc
-stop on stopping Xvnc
-
-kill timeout 60
-
-exec nice brozzler-worker \
-    --rethinkdb-servers={{groups['rethinkdb'] | join(',')}} \
-    --max-browsers=4 \
-    --verbose \
-    --warcprox-auto

ansible/roles/brozzler-worker/templates/vnc-websock-run.j2

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+logfile=/var/log/vnc-websock.log
+touch $logfile
+chown {{user}} $logfile
+
+source /opt/websockify-ve3/bin/activate
+
+exec nice setuidgid {{user}} websockify 0.0.0.0:8901 localhost:5901 >> $logfile 2>&1
+

ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2

@@ -1,14 +0,0 @@
-description "vnc-websock"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-setuid {{user}}
-
-console log
-
-env PATH={{venv_root}}/websockify-ve3/bin:/usr/bin:/bin
-
-# port 8901 is hard-coded in brozzler/dashboard/static/partials/workers.html
-exec nice websockify 0.0.0.0:8901 localhost:5901
-

ansible/roles/common/tasks/main.yml

@@ -1,44 +1,74 @@
 ---
-# get latest pip (had problems with version from apt-get, specifically
-# "pip install pyopenssl" did not install the dependency "cryptography")
-# http://stackoverflow.com/questions/34587473/what-is-get-pip-py-checksum-where-can-i-get-it-for-sure
-- name: install setuptools for python 2 and 3
+- apt:
+    name:
+    - python3-setuptools
+    - python3-pip
+    - python3-virtualenv
+    - daemontools
+    - daemontools-run
+    state: present
+    update_cache: yes
+    cache_valid_time: 86400 # one day
   become: true
-  apt: name={{item}} state=present
-  with_items:
-  - python-setuptools
-  - python3-setuptools
-- name: download pip-9.0.1.tar.gz
-  get_url:
-    url: https://pypi.python.org/packages/11/b6/abcb525026a4be042b486df43905d6893fb04f05aac21c32c638e939e447/pip-9.0.1.tar.gz
-    dest: /tmp
-    checksum: sha1:57ff41e99cb01b6a1c2b0999161589b726f0ec8b
-- name: extract pip-9.0.1.tar.gz
-  unarchive: src=/tmp/pip-9.0.1.tar.gz dest=/tmp copy=no
+
+# # get recent virtualenv, which bundles a recent pip
+# - find:
+#     paths:
+#     - /usr/local/lib/python3.4/dist-packages
+#     - /usr/local/lib/python3.5/dist-packages
+#     recurse: true
+#     patterns: virtualenv.py
+#     contains: '__version__ = "16.4.3"'
+#   register: virtualenv_py_16_4_3
+# 
+# - command: mktemp -d
+#   register: mktempd_out
+#   when: virtualenv_py_16_4_3.matched == 0
+# 
+# - name: download virtualenv-16.4.3
+#   get_url:
+#     url: https://files.pythonhosted.org/packages/37/db/89d6b043b22052109da35416abc3c397655e4bd3cff031446ba02b9654fa/virtualenv-16.4.3.tar.gz
+#     dest: '{{mktempd_out.stdout}}'
+#     checksum: sha256:984d7e607b0a5d1329425dd8845bd971b957424b5ba664729fab51ab8c11bc39
+#   when: virtualenv_py_16_4_3.matched == 0
+# 
+# - name: extract virtualenv-16.4.3.tar.gz
+#   unarchive:
+#     src: '{{mktempd_out.stdout}}/virtualenv-16.4.3.tar.gz'
+#     dest: '{{mktempd_out.stdout}}'
+#     copy: no
+#   when: virtualenv_py_16_4_3.matched == 0
+# 
+# - name: run "python3 setup.py install" in {{mktempd_out.stdout}}/virtualenv-16.4.3
+#   become: true
+#   command: python3 setup.py install
+#   args:
+#     chdir: '{{mktempd_out.stdout}}/virtualenv-16.4.3'
+#   when: virtualenv_py_16_4_3.matched == 0
+# 
+# - file:
+#     path: '{{mktempd_out.stdout}}'
+#     state: absent
+#   become: true
+#   when: virtualenv_py_16_4_3.matched == 0
 
 # this clause is a workaround for travis-ci, which only wants to install in /usr
 # see https://travis-ci.org/internetarchive/brozzler/builds/174338601
 # but it complains that /usr/lib/python3.5/site-packages doesn't exist
 # see https://travis-ci.org/internetarchive/brozzler/builds/174094831
-- file: path={{item}} state=directory
+- file:
+    path: '{{item}}'
+    state: directory
   with_items:
   - /usr/lib/python3.5/site-packages
   - /usr/lib/python3.5/dist-packages
   become: true
 
-- name: run "python3 setup.py install" in /tmp/pip-9.0.1
-  command: python3 setup.py install
-           chdir=/tmp/pip-9.0.1
-           creates=/usr/local/lib/python3.5/dist-packages/pip-9.0.1-py3.5.egg/pip/__init__.py
-  become: true
-- name: run "pip install virtualenv"
-  command: pip install virtualenv
-           creates=/usr/local/lib/python3.5/dist-packages/virtualenv.py
-  become: true
 - command: id {{user}}
   register: id_user
   ignore_errors: true
   changed_when: false
+
 - name: ensure service user {{user}} exists
   user: name={{user}} system=yes createhome=no home=/nonexistent
         shell=/usr/sbin/nologin

ansible/roles/pywb/handlers/main.yml

@@ -1,5 +1,9 @@
 ---
 - name: restart pywb
-  service: name=pywb state=restarted
+  svc:
+    name: pywb
+    state: restarted
+    service_dir: /etc/service
   become: true
 
+

ansible/roles/pywb/tasks/main.yml

@@ -3,34 +3,50 @@
   file: path={{venv_root}}/pywb-ve3 state=directory
         owner={{user}}
   become: true
+
 - name: install pywb in virtualenv
-  pip: name=pywb
-       version=0.33.2
-       virtualenv={{venv_root}}/pywb-ve3
-       virtualenv_python=python3
-       extra_args='--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
+  pip:
+    name: pywb
+    version: 0.33.2
+    virtualenv: '{{venv_root}}/pywb-ve3'
+    virtualenv_python: python3
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
+    extra_args: '--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
   become: true
   become_user: '{{user}}'
   notify:
   - restart pywb
+
 - name: install brozzler in pywb virtualenv
-  pip: name='{{brozzler_pip_name}}'
-       virtualenv={{venv_root}}/pywb-ve3
-       virtualenv_python=python3
-       extra_args='--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
+  pip:
+    name: '{{brozzler_pip_name}}'
+    virtualenv: '{{venv_root}}/pywb-ve3'
+    virtualenv_python: python3
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
+    extra_args: '--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
   become: true
   become_user: '{{user}}'
   notify:
   - restart pywb
+
 - name: pywb config file /etc/pywb.yml
   template: src=templates/pywb.yml.j2
             dest=/etc/pywb.yml
   become: true
   notify:
   - restart pywb
-- name: upstart config file /etc/init/pywb.conf
-  template: src=templates/pywb.conf.j2
-            dest=/etc/init/pywb.conf
+
+- name: mkdir /etc/service/pywb
+  file:
+    path: /etc/service/pywb
+    state: directory
   become: true
+
+- name: install /etc/service/pywb/run
+  template:
+    src: templates/pywb-run.j2
+    dest: /etc/service/pywb/run
+    mode: 0755
   notify:
   - restart pywb
+  become: true

ansible/roles/pywb/templates/pywb-run.j2

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+logfile=/var/log/pywb.log
+touch $logfile
+chown {{user}} $logfile
+
+exec nice setuidgid {{user}} env PYWB_CONFIG_FILE=/etc/pywb.yml \
+    {{venv_root}}/pywb-ve3/bin/python {{venv_root}}/pywb-ve3/bin/brozzler-wayback \
+    >> $logfile 2>&1
+

ansible/roles/pywb/templates/pywb.conf.j2

@@ -1,12 +0,0 @@
-description "pywb"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-env PYWB_CONFIG_FILE=/etc/pywb.yml
-
-setuid {{user}}
-
-console log
-
-exec nice {{venv_root}}/pywb-ve3/bin/python {{venv_root}}/pywb-ve3/bin/brozzler-wayback

ansible/roles/warcprox/handlers/main.yml

@@ -1,4 +1,7 @@
 ---
 - name: restart warcprox
-  service: name=warcprox state=restarted
+  svc:
+    name: warcprox
+    state: restarted
+    service_dir: /etc/service
   become: true

ansible/roles/warcprox/tasks/main.yml

@@ -13,16 +13,28 @@
   become: true
   file: path={{venv_root}}/warcprox-ve3 state=directory owner={{user}}
 - name: install warcprox in virtualenv
-  pip: name=git+https://github.com/internetarchive/warcprox.git#egg=warcprox
-       virtualenv={{venv_root}}/warcprox-ve3
-       virtualenv_python=python3
-       extra_args='--no-input --upgrade --pre --cache-dir=/tmp/pip-cache'
+  pip:
+    name: git+https://github.com/internetarchive/warcprox.git#egg=warcprox
+    virtualenv: '{{venv_root}}/warcprox-ve3'
+    virtualenv_python: python3
+    extra_args: --no-input --upgrade --pre --cache-dir=/tmp/pip-cache
+    virtualenv_command: python3 /usr/lib/python3/dist-packages/virtualenv.py
   become: true
   become_user: '{{user}}'
   notify:
   - restart warcprox
-- name: install upstart config /etc/init/warcprox.conf
+
+- name: mkdir /etc/service/warcprox
+  file:
+    path: /etc/service/warcprox
+    state: directory
   become: true
-  template: src=templates/warcprox.conf.j2 dest=/etc/init/warcprox.conf
+
+- name: install /etc/service/warcprox/run
+  template:
+    src: templates/run.j2
+    dest: /etc/service/warcprox/run
+    mode: 0755
   notify:
   - restart warcprox
+  become: true

ansible/roles/warcprox/templates/run.j2

@@ -1,16 +1,15 @@
-description "warcprox"
+#!/bin/bash
 
-start on runlevel [2345]
-stop on runlevel [!2345]
 
-# by default warcprox creates some files/dirs relative to cwd
-chdir {{work_dir}}
-setuid {{user}}
+logfile=/var/log/warcprox.log
+touch $logfile
+chown {{user}} $logfile
 
-console log
+ulimit -n 4096
 
-# --profile
-exec nice {{venv_root}}/warcprox-ve3/bin/python {{venv_root}}/warcprox-ve3/bin/warcprox \
+source {{venv_root}}/warcprox-ve3/bin/activate
+
+exec nice -n5 setuidgid {{user}} env LANG=en_US.UTF-8 LC_COLLATE=C warcprox \
          --address=0.0.0.0 \
          --dir={{warcs_dir}} \
          --base32 \
@@ -19,4 +18,6 @@ exec nice {{venv_root}}/warcprox-ve3/bin/python {{venv_root}}/warcprox-ve3/bin/w
          --onion-tor-socks-proxy=localhost:9050 \
          --rethinkdb-services-url=rethinkdb://{{groups['rethinkdb']|join(',')}}/brozzler/services \
          --rethinkdb-stats-url=rethinkdb://{{groups['rethinkdb']|join(',')}}/brozzler/stats \
-         --rethinkdb-big-table-url=rethinkdb://{{groups['rethinkdb']|join(',')}}/brozzler/captures
+         --rethinkdb-big-table-url=rethinkdb://{{groups['rethinkdb']|join(',')}}/brozzler/captures \
+         >> $logfile 2>&1
+