From 3627209be167562f5ee9ff98b6ddc911cbd765fa Mon Sep 17 00:00:00 2001 From: Noah Levitt Date: Thu, 13 Oct 2016 17:21:55 -0700 Subject: [PATCH] move ansible directory to top level; generalize formerly vagrant-specific ansible configuration; let upstart manage logging with "console log" --- ansible/hosts-vagrant | 23 ++++++ {vagrant/ansible => ansible}/playbook.yml | 7 +- .../brozzler-webconsole/handlers/main.yml | 0 .../roles/brozzler-webconsole/meta/main.yml | 3 + .../roles/brozzler-webconsole/tasks/main.yml | 10 ++- .../templates/brozzler-webconsole.conf.j2 | 10 +-- .../roles/brozzler-worker/handlers/main.yml | 0 ansible/roles/brozzler-worker/meta/main.yml | 3 + .../roles/brozzler-worker/tasks/main.yml | 74 +++++++++++-------- .../brozzler-worker/templates/Xvnc.conf.j2 | 4 +- .../templates/brozzler-worker.conf.j2 | 10 +-- .../templates/vnc-websock.conf.j2 | 6 +- ansible/roles/common/defaults/main.yml | 7 ++ .../roles/common/tasks/main.yml | 5 +- .../roles/pywb/handlers/main.yml | 0 ansible/roles/pywb/meta/main.yml | 3 + .../roles/pywb/tasks/main.yml | 14 +++- ansible/roles/pywb/templates/pywb.conf.j2 | 14 ++++ .../roles/pywb/templates/pywb.yml.j2 | 7 +- .../roles/rethinkdb/handlers/main.yml | 0 .../roles/rethinkdb/tasks/main.yml | 12 ++- .../templates/rethinkdb-brozzler.conf.j2 | 6 ++ ansible/roles/warcprox/defaults/main.yml | 2 + ansible/roles/warcprox/handlers/main.yml | 4 + ansible/roles/warcprox/meta/main.yml | 3 + .../roles/warcprox/tasks/main.yml | 7 +- .../roles/warcprox/templates/warcprox.conf.j2 | 24 ++++++ setup.py | 2 +- vagrant/Vagrantfile | 4 +- vagrant/ansible/hosts | 16 ---- .../ansible/roles/pywb/templates/pywb.conf.j2 | 14 ---- .../rethinkdb-brozzler-vagrant-1.conf.j2 | 5 -- .../ansible/roles/warcprox/handlers/main.yml | 14 ---- .../roles/warcprox/templates/warcprox.conf.j2 | 26 ------- 34 files changed, 194 insertions(+), 145 deletions(-) create mode 100644 ansible/hosts-vagrant rename {vagrant/ansible => ansible}/playbook.yml (79%) rename {vagrant/ansible => ansible}/roles/brozzler-webconsole/handlers/main.yml (100%) create mode 100644 ansible/roles/brozzler-webconsole/meta/main.yml rename {vagrant/ansible => ansible}/roles/brozzler-webconsole/tasks/main.yml (57%) rename {vagrant/ansible => ansible}/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2 (51%) rename {vagrant/ansible => ansible}/roles/brozzler-worker/handlers/main.yml (100%) create mode 100644 ansible/roles/brozzler-worker/meta/main.yml rename {vagrant/ansible => ansible}/roles/brozzler-worker/tasks/main.yml (52%) rename {vagrant/ansible => ansible}/roles/brozzler-worker/templates/Xvnc.conf.j2 (81%) rename {vagrant/ansible => ansible}/roles/brozzler-worker/templates/brozzler-worker.conf.j2 (58%) rename {vagrant/ansible => ansible}/roles/brozzler-worker/templates/vnc-websock.conf.j2 (62%) create mode 100644 ansible/roles/common/defaults/main.yml rename {vagrant/ansible => ansible}/roles/common/tasks/main.yml (88%) rename {vagrant/ansible => ansible}/roles/pywb/handlers/main.yml (100%) create mode 100644 ansible/roles/pywb/meta/main.yml rename {vagrant/ansible => ansible}/roles/pywb/tasks/main.yml (65%) create mode 100644 ansible/roles/pywb/templates/pywb.conf.j2 rename {vagrant/ansible => ansible}/roles/pywb/templates/pywb.yml.j2 (62%) rename {vagrant/ansible => ansible}/roles/rethinkdb/handlers/main.yml (100%) rename {vagrant/ansible => ansible}/roles/rethinkdb/tasks/main.yml (64%) create mode 100644 ansible/roles/rethinkdb/templates/rethinkdb-brozzler.conf.j2 create mode 100644 ansible/roles/warcprox/defaults/main.yml create mode 100644 ansible/roles/warcprox/handlers/main.yml create mode 100644 ansible/roles/warcprox/meta/main.yml rename {vagrant/ansible => ansible}/roles/warcprox/tasks/main.yml (74%) create mode 100644 ansible/roles/warcprox/templates/warcprox.conf.j2 delete mode 100644 vagrant/ansible/hosts delete mode 100644 vagrant/ansible/roles/pywb/templates/pywb.conf.j2 delete mode 100644 vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-vagrant-1.conf.j2 delete mode 100644 vagrant/ansible/roles/warcprox/handlers/main.yml delete mode 100644 vagrant/ansible/roles/warcprox/templates/warcprox.conf.j2 diff --git a/ansible/hosts-vagrant b/ansible/hosts-vagrant new file mode 100644 index 0000000..b395fc6 --- /dev/null +++ b/ansible/hosts-vagrant @@ -0,0 +1,23 @@ +ansible_ssh_private_key_file=.vagrant/machines/10.9.9.9/virtualbox/private_key +warcs_dir=/vagrant/warcs +brozzler_pip_name='-e git+file:///brozzler' + +### possible values for a prod deployment +# brozzler_pip_name=brozzler # get it from pypi +# brozzler_pip_name=git+https://github.com/internetarchive/brozzler.git#egg=brozzler + +[rethinkdb] +10.9.9.9 + +[warcprox] +work_dir=/vagrant +10.9.9.9 + +[brozzler-worker] +10.9.9.9 + +[brozzler-webconsole] +10.9.9.9 + +[pywb] +10.9.9.9 diff --git a/vagrant/ansible/playbook.yml b/ansible/playbook.yml similarity index 79% rename from vagrant/ansible/playbook.yml rename to ansible/playbook.yml index f55a4b1..6dcfc56 100644 --- a/vagrant/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -1,9 +1,4 @@ --- -- name: apply common configuration to all nodes - hosts: all - roles: - - common - - name: deploy rethinkdb hosts: rethinkdb roles: @@ -27,4 +22,4 @@ - name: deploy pywb hosts: pywb roles: - - pywb + - pywb diff --git a/vagrant/ansible/roles/brozzler-webconsole/handlers/main.yml b/ansible/roles/brozzler-webconsole/handlers/main.yml similarity index 100% rename from vagrant/ansible/roles/brozzler-webconsole/handlers/main.yml rename to ansible/roles/brozzler-webconsole/handlers/main.yml diff --git a/ansible/roles/brozzler-webconsole/meta/main.yml b/ansible/roles/brozzler-webconsole/meta/main.yml new file mode 100644 index 0000000..5a155ce --- /dev/null +++ b/ansible/roles/brozzler-webconsole/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: +- role: common diff --git a/vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml b/ansible/roles/brozzler-webconsole/tasks/main.yml similarity index 57% rename from vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml rename to ansible/roles/brozzler-webconsole/tasks/main.yml index 1d7194a..213ded7 100644 --- a/vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml +++ b/ansible/roles/brozzler-webconsole/tasks/main.yml @@ -1,9 +1,15 @@ --- +- name: mkdir {{venv_root}}/brozzler-webconsole-ve34 + file: path={{venv_root}}/brozzler-webconsole-ve34 state=directory + owner={{user}} + become: true - name: install brozzler[webconsole] in virtualenv - pip: name='-e /brozzler[webconsole]' - virtualenv=/home/vagrant/brozzler-webconsole-ve34 + pip: name='{{brozzler_pip_name}}[webconsole]' + virtualenv={{venv_root}}/brozzler-webconsole-ve34 virtualenv_python=python3.4 extra_args='--no-input --upgrade --pre' + become: true + become_user: '{{user}}' notify: - restart brozzler-webconsole - name: install upstart config /etc/init/brozzler-webconsole.conf diff --git a/vagrant/ansible/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2 b/ansible/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2 similarity index 51% rename from vagrant/ansible/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2 rename to ansible/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2 index c8d0656..727709d 100644 --- a/vagrant/ansible/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2 +++ b/ansible/roles/brozzler-webconsole/templates/brozzler-webconsole.conf.j2 @@ -3,16 +3,16 @@ description "brozzler-webconsole" start on runlevel [2345] stop on runlevel [!2345] -env PYTHONPATH=/home/vagrant/brozzler-webconsole-ve34/lib/python3.4/site-packages -env PATH=/home/vagrant/brozzler-webconsole-ve34/bin:/usr/bin:/bin +env PYTHONPATH={{venv_root}}/brozzler-webconsole-ve34/lib/python3.4/site-packages +env PATH={{venv_root}}/brozzler-webconsole-ve34/bin:/usr/bin:/bin env LC_ALL=C.UTF-8 env WAYBACK_BASEURL=http://{{groups['pywb'][0]}}:8880/brozzler env RETHINKDB_SERVERS={{groups['rethinkdb'] | join(',')}} env RETHINKDB_DB=brozzler -setuid vagrant +setuid {{user}} -# console log +console log -exec gunicorn --bind=0.0.0.0:8881 brozzler.webconsole:app >>/vagrant/logs/brozzler-webconsole.log 2>&1 +exec gunicorn --bind=0.0.0.0:8881 brozzler.webconsole:app diff --git a/vagrant/ansible/roles/brozzler-worker/handlers/main.yml b/ansible/roles/brozzler-worker/handlers/main.yml similarity index 100% rename from vagrant/ansible/roles/brozzler-worker/handlers/main.yml rename to ansible/roles/brozzler-worker/handlers/main.yml diff --git a/ansible/roles/brozzler-worker/meta/main.yml b/ansible/roles/brozzler-worker/meta/main.yml new file mode 100644 index 0000000..5a155ce --- /dev/null +++ b/ansible/roles/brozzler-worker/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: +- role: common diff --git a/vagrant/ansible/roles/brozzler-worker/tasks/main.yml b/ansible/roles/brozzler-worker/tasks/main.yml similarity index 52% rename from vagrant/ansible/roles/brozzler-worker/tasks/main.yml rename to ansible/roles/brozzler-worker/tasks/main.yml index 7dad56a..a06ad82 100644 --- a/vagrant/ansible/roles/brozzler-worker/tasks/main.yml +++ b/ansible/roles/brozzler-worker/tasks/main.yml @@ -7,56 +7,66 @@ become: true apt: name={{item}} state=present with_items: - - python-virtualenv - - vnc4server - - chromium-browser - - xfonts-base - - fonts-arphic-bkai00mp - - fonts-arphic-bsmi00lp - - fonts-arphic-gbsn00lp - - fonts-arphic-gkai00mp - - fonts-arphic-ukai - - fonts-farsiweb - - fonts-nafees - - fonts-sil-abyssinica - - fonts-sil-ezra - - fonts-sil-padauk - - fonts-unfonts-extra - - fonts-unfonts-core - - ttf-indic-fonts - - fonts-thai-tlwg - - fonts-lklug-sinhala - - git - - libjpeg-turbo8-dev - - zlib1g-dev - - gcc - - libpython3.4-dev - - adobe-flashplugin + - python-virtualenv + - vnc4server + - chromium-browser + - xfonts-base + - fonts-arphic-bkai00mp + - fonts-arphic-bsmi00lp + - fonts-arphic-gbsn00lp + - fonts-arphic-gkai00mp + - fonts-arphic-ukai + - fonts-farsiweb + - fonts-nafees + - fonts-sil-abyssinica + - fonts-sil-ezra + - fonts-sil-padauk + - fonts-unfonts-extra + - fonts-unfonts-core + - ttf-indic-fonts + - fonts-thai-tlwg + - fonts-lklug-sinhala + - git + - libjpeg-turbo8-dev + - zlib1g-dev + - gcc + - libpython3.4-dev + - adobe-flashplugin - name: install Xvnc upstart config /etc/init/Xvnc.conf template: src=templates/Xvnc.conf.j2 dest=/etc/init/Xvnc.conf become: true notify: - - restart Xvnc + - restart Xvnc +- name: mkdir {{venv_root}}/websockify-ve34 + become: true + file: path={{venv_root}}/websockify-ve34 state=directory owner={{user}} - name: install websockify in virtualenv pip: name=git+https://github.com/kanaka/websockify.git#egg=websockify - virtualenv=/home/vagrant/websockify-ve34 + virtualenv={{venv_root}}/websockify-ve34 virtualenv_python=python3.4 extra_args='--no-input --upgrade --pre' + become: true + become_user: '{{user}}' - name: install vnc-websock upstart config /etc/init/vnc-websock.conf template: src=templates/vnc-websock.conf.j2 dest=/etc/init/vnc-websock.conf become: true notify: - - restart vnc-websock + - restart vnc-websock +- name: mkdir {{venv_root}}/brozzler-ve34 + become: true + file: path={{venv_root}}/brozzler-ve34 state=directory owner={{user}} - name: install brozzler in virtualenv pip: # name=git+https://github.com/internetarchive/brozzler.git#egg=brozzler - name='-e /brozzler' - virtualenv=/home/vagrant/brozzler-ve34 + name='{{brozzler_pip_name}}' + virtualenv={{venv_root}}/brozzler-ve34 virtualenv_python=python3.4 extra_args='--no-input --upgrade --pre' + become: true + become_user: '{{user}}' notify: - - restart brozzler-worker + - restart brozzler-worker - name: install brozzler-worker upstart config /etc/init/brozzler-worker.conf template: src=templates/brozzler-worker.conf.j2 dest=/etc/init/brozzler-worker.conf become: true notify: - - restart brozzler-worker + - restart brozzler-worker diff --git a/vagrant/ansible/roles/brozzler-worker/templates/Xvnc.conf.j2 b/ansible/roles/brozzler-worker/templates/Xvnc.conf.j2 similarity index 81% rename from vagrant/ansible/roles/brozzler-worker/templates/Xvnc.conf.j2 rename to ansible/roles/brozzler-worker/templates/Xvnc.conf.j2 index 6381143..57ece99 100644 --- a/vagrant/ansible/roles/brozzler-worker/templates/Xvnc.conf.j2 +++ b/ansible/roles/brozzler-worker/templates/Xvnc.conf.j2 @@ -3,11 +3,11 @@ description "Xvnc" start on runlevel [2345] stop on runlevel [!2345] -setuid vagrant +setuid {{user}} console log -exec nice Xvnc4 :1 -auth /tmp/Xauthority.vagrant \ +exec nice Xvnc4 :1 -auth /tmp/Xauthority.{{user}} \ -geometry 1600x1000 -depth 24 -rfbwait 0 -nolisten tcp -rfbport 5901 \ -SecurityTypes None -pn -fp /usr/share/fonts/X11/misc/ -co /etc/X11/rgb \ AcceptCutText=0 AcceptPointerEvents=0 AcceptKeyEvents=0 diff --git a/vagrant/ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2 b/ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2 similarity index 58% rename from vagrant/ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2 rename to ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2 index b36403e..3d8906e 100644 --- a/vagrant/ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2 +++ b/ansible/roles/brozzler-worker/templates/brozzler-worker.conf.j2 @@ -4,13 +4,13 @@ start on runlevel [2345] stop on runlevel [!2345] env DISPLAY=:1 -env PATH=/home/vagrant/brozzler-ve34/bin:/usr/bin:/bin -env PYTHONPATH=/home/vagrant/brozzler-ve34/lib/python3.4/site-packages +env PATH={{venv_root}}/brozzler-ve34/bin:/usr/bin:/bin +env PYTHONPATH={{venv_root}}/brozzler-ve34/lib/python3.4/site-packages env LANG=C.UTF-8 -setuid vagrant +setuid {{user}} -# console log +console log # depends on vnc server start on started Xvnc @@ -20,4 +20,4 @@ kill timeout 60 exec nice brozzler-worker \ --rethinkdb-servers={{groups['rethinkdb'] | join(',')}} \ - --max-browsers=4 >>/vagrant/logs/brozzler-worker.log 2>&1 + --max-browsers=4 diff --git a/vagrant/ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2 b/ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2 similarity index 62% rename from vagrant/ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2 rename to ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2 index 32b959f..7258706 100644 --- a/vagrant/ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2 +++ b/ansible/roles/brozzler-worker/templates/vnc-websock.conf.j2 @@ -3,12 +3,12 @@ description "vnc-websock" start on runlevel [2345] stop on runlevel [!2345] -setuid vagrant +setuid {{user}} console log -env PYTHONPATH=/home/vagrant/websockify-ve34/lib/python3.4/site-packages -env PATH=/home/vagrant/websockify-ve34/bin:/usr/bin:/bin +env PYTHONPATH={{venv_root}}/websockify-ve34/lib/python3.4/site-packages +env PATH={{venv_root}}/websockify-ve34/bin:/usr/bin:/bin # port 8901 is hard-coded in brozzler/webconsole/static/partials/workers.html exec nice websockify 0.0.0.0:8901 localhost:5901 diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml new file mode 100644 index 0000000..6fc3326 --- /dev/null +++ b/ansible/roles/common/defaults/main.yml @@ -0,0 +1,7 @@ +# variables default values, these can be overridden in the ansible inventory +# or various other places +--- +user: brozzler +venv_root: /opt +warcs_dir: /var/tmp/warcs +brozzler_pip_name: brozzler # get it from pypi by default diff --git a/vagrant/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml similarity index 88% rename from vagrant/ansible/roles/common/tasks/main.yml rename to ansible/roles/common/tasks/main.yml index 9a14357..d3462a5 100644 --- a/vagrant/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -19,6 +19,7 @@ ## command: python3 setup.py install chdir=/tmp/pip-8.1.2 ## creates=/usr/local/lib/python2.7/dist-packages/pip-8.1.2-py2.7.egg/pip/__init__.py ## become: true -- name: mkdir /vagrant/logs - file: path=/vagrant/logs state=directory +- name: ensure service user {{user}} exists + user: name={{user}} system=yes createhome=no home=/nonexistent + shell=/usr/sbin/nologin become: true diff --git a/vagrant/ansible/roles/pywb/handlers/main.yml b/ansible/roles/pywb/handlers/main.yml similarity index 100% rename from vagrant/ansible/roles/pywb/handlers/main.yml rename to ansible/roles/pywb/handlers/main.yml diff --git a/ansible/roles/pywb/meta/main.yml b/ansible/roles/pywb/meta/main.yml new file mode 100644 index 0000000..5a155ce --- /dev/null +++ b/ansible/roles/pywb/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: +- role: common diff --git a/vagrant/ansible/roles/pywb/tasks/main.yml b/ansible/roles/pywb/tasks/main.yml similarity index 65% rename from vagrant/ansible/roles/pywb/tasks/main.yml rename to ansible/roles/pywb/tasks/main.yml index 08fac19..2494096 100644 --- a/vagrant/ansible/roles/pywb/tasks/main.yml +++ b/ansible/roles/pywb/tasks/main.yml @@ -1,16 +1,24 @@ --- +- name: mkdir {{venv_root}}/pywb-ve34 + file: path={{venv_root}}/pywb-ve34 state=directory + owner={{user}} + become: true - name: install pywb in virtualenv pip: name=pywb - virtualenv=/home/vagrant/pywb-ve34 + virtualenv={{venv_root}}/pywb-ve34 virtualenv_python=python3.4 extra_args='--no-input --upgrade --pre' + become: true + become_user: '{{user}}' notify: - restart pywb - name: install brozzler in pywb virtualenv - pip: name='-e /brozzler' - virtualenv=/home/vagrant/pywb-ve34 + pip: name='{{brozzler_pip_name}}' + virtualenv={{venv_root}}/pywb-ve34 virtualenv_python=python3.4 extra_args='--no-input --upgrade --pre' + become: true + become_user: '{{user}}' notify: - restart pywb - name: pywb config file /etc/pywb.yml diff --git a/ansible/roles/pywb/templates/pywb.conf.j2 b/ansible/roles/pywb/templates/pywb.conf.j2 new file mode 100644 index 0000000..c2cc89e --- /dev/null +++ b/ansible/roles/pywb/templates/pywb.conf.j2 @@ -0,0 +1,14 @@ +description "pywb" + +start on runlevel [2345] +stop on runlevel [!2345] + +env PYTHONPATH={{venv_root}}/pywb-ve34/lib/python3.4/site-packages +env PATH={{venv_root}}/pywb-ve34/bin:/usr/bin:/bin +env PYWB_CONFIG_FILE=/etc/pywb.yml + +setuid {{user}} + +console log + +exec nice brozzler-wayback diff --git a/vagrant/ansible/roles/pywb/templates/pywb.yml.j2 b/ansible/roles/pywb/templates/pywb.yml.j2 similarity index 62% rename from vagrant/ansible/roles/pywb/templates/pywb.yml.j2 rename to ansible/roles/pywb/templates/pywb.yml.j2 index c17281c..9a3cd6a 100644 --- a/vagrant/ansible/roles/pywb/templates/pywb.yml.j2 +++ b/ansible/roles/pywb/templates/pywb.yml.j2 @@ -1,9 +1,12 @@ -archive_paths: /vagrant/warcs/ +archive_paths: {{warcs_dir}} collections: brozzler: index_paths: !!python/object:brozzler.pywb.RethinkCDXSource db: brozzler - servers: [localhost] + servers: + {% for node in groups['rethinkdb'] %} + - {{node}} + {% endfor %} table: captures enable_auto_colls: false enable_cdx_api: true diff --git a/vagrant/ansible/roles/rethinkdb/handlers/main.yml b/ansible/roles/rethinkdb/handlers/main.yml similarity index 100% rename from vagrant/ansible/roles/rethinkdb/handlers/main.yml rename to ansible/roles/rethinkdb/handlers/main.yml diff --git a/vagrant/ansible/roles/rethinkdb/tasks/main.yml b/ansible/roles/rethinkdb/tasks/main.yml similarity index 64% rename from vagrant/ansible/roles/rethinkdb/tasks/main.yml rename to ansible/roles/rethinkdb/tasks/main.yml index 7f40ff4..3136828 100644 --- a/vagrant/ansible/roles/rethinkdb/tasks/main.yml +++ b/ansible/roles/rethinkdb/tasks/main.yml @@ -11,13 +11,17 @@ become: true notify: - restart rethinkdb -# XXX rethinkdb fails to start in spite of this, I think because /vagrant -# gets mounted too late, and it tries to log there - name: ensure rethinkdb starts on reboot service: name=rethinkdb enabled=yes +- stat: path=/var/log/rethinkdb.log + register: p +- name: ensure user rethinkdb owns /var/log/rethinkdb.log + file: path=/var/log/rethinkdb.log owner=rethinkdb state=touch mode=0644 + when: not p.stat.exists + become: true - name: ensure rethinkdb instance config file is installed - template: src=templates/rethinkdb-brozzler-vagrant-1.conf.j2 - dest=/etc/rethinkdb/instances.d/rethinkdb-brozzler-vagrant-1.conf + template: src=templates/rethinkdb-brozzler.conf.j2 + dest=/etc/rethinkdb/instances.d/rethinkdb-brozzler.conf become: true notify: - restart rethinkdb diff --git a/ansible/roles/rethinkdb/templates/rethinkdb-brozzler.conf.j2 b/ansible/roles/rethinkdb/templates/rethinkdb-brozzler.conf.j2 new file mode 100644 index 0000000..f790d88 --- /dev/null +++ b/ansible/roles/rethinkdb/templates/rethinkdb-brozzler.conf.j2 @@ -0,0 +1,6 @@ +bind=0.0.0.0 +# directory=/var/lib/rethinkdb +log-file=/var/log/rethinkdb.log +{% for node in groups['rethinkdb'] %} +join={{node}}:29015 +{% endfor %} diff --git a/ansible/roles/warcprox/defaults/main.yml b/ansible/roles/warcprox/defaults/main.yml new file mode 100644 index 0000000..492d43d --- /dev/null +++ b/ansible/roles/warcprox/defaults/main.yml @@ -0,0 +1,2 @@ +--- +work_dir: /var/tmp diff --git a/ansible/roles/warcprox/handlers/main.yml b/ansible/roles/warcprox/handlers/main.yml new file mode 100644 index 0000000..0b7edcd --- /dev/null +++ b/ansible/roles/warcprox/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: restart warcprox + service: name=warcprox state=restarted + become: true diff --git a/ansible/roles/warcprox/meta/main.yml b/ansible/roles/warcprox/meta/main.yml new file mode 100644 index 0000000..5a155ce --- /dev/null +++ b/ansible/roles/warcprox/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: +- role: common diff --git a/vagrant/ansible/roles/warcprox/tasks/main.yml b/ansible/roles/warcprox/tasks/main.yml similarity index 74% rename from vagrant/ansible/roles/warcprox/tasks/main.yml rename to ansible/roles/warcprox/tasks/main.yml index 7a9c7d0..6487646 100644 --- a/vagrant/ansible/roles/warcprox/tasks/main.yml +++ b/ansible/roles/warcprox/tasks/main.yml @@ -11,11 +11,16 @@ - libssl-dev - tor - git +- name: mkdir {{venv_root}}/warcprox-ve34 + become: true + file: path={{venv_root}}/warcprox-ve34 state=directory owner={{user}} - name: install warcprox in virtualenv pip: name=git+https://github.com/internetarchive/warcprox.git@2.x#egg=warcprox - virtualenv=/home/vagrant/warcprox-ve34 + virtualenv={{venv_root}}/warcprox-ve34 virtualenv_python=python3.4 extra_args='--no-input --upgrade --pre' + become: true + become_user: '{{user}}' notify: - restart warcprox - name: install upstart config /etc/init/warcprox.conf diff --git a/ansible/roles/warcprox/templates/warcprox.conf.j2 b/ansible/roles/warcprox/templates/warcprox.conf.j2 new file mode 100644 index 0000000..34101d6 --- /dev/null +++ b/ansible/roles/warcprox/templates/warcprox.conf.j2 @@ -0,0 +1,24 @@ +description "warcprox" + +start on runlevel [2345] +stop on runlevel [!2345] + +env PYTHONPATH={{venv_root}}/warcprox-ve34/lib/python3.4/site-packages +env PATH={{venv_root}}/warcprox-ve34/bin:/usr/bin:/bin + +# by default warcprox creates some files/dirs relative to cwd +chdir {{work_dir}} +setuid {{user}} + +console log + +# --profile +exec nice warcprox \ + --dir={{warcs_dir}} \ + --base32 \ + --gzip \ + --rollover-idle-time=180 \ + --onion-tor-socks-proxy=localhost:9050 \ + --rethinkdb-servers={{groups['rethinkdb']|join(',')}} \ + --rethinkdb-db=brozzler \ + --rethinkdb-big-table diff --git a/setup.py b/setup.py index 8bdb616..9b81a94 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b7.dev96', + version='1.1b7.dev97', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt', diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 16220f7..5a1c8ae 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -7,7 +7,7 @@ Vagrant.configure(2) do |config| config.vm.synced_folder "..", "/brozzler" config.vm.provision "ansible" do |ansible| - ansible.inventory_path = "ansible/hosts" - ansible.playbook = "ansible/playbook.yml" + ansible.inventory_path = "../ansible/hosts-vagrant" + ansible.playbook = "../ansible/playbook.yml" end end diff --git a/vagrant/ansible/hosts b/vagrant/ansible/hosts deleted file mode 100644 index 0257aeb..0000000 --- a/vagrant/ansible/hosts +++ /dev/null @@ -1,16 +0,0 @@ -ansible_ssh_private_key_file=.vagrant/machines/10.9.9.9/virtualbox/private_key - -[rethinkdb] -10.9.9.9 - -[warcprox] -10.9.9.9 - -[brozzler-worker] -10.9.9.9 - -[brozzler-webconsole] -10.9.9.9 - -[pywb] -10.9.9.9 diff --git a/vagrant/ansible/roles/pywb/templates/pywb.conf.j2 b/vagrant/ansible/roles/pywb/templates/pywb.conf.j2 deleted file mode 100644 index 5b2887e..0000000 --- a/vagrant/ansible/roles/pywb/templates/pywb.conf.j2 +++ /dev/null @@ -1,14 +0,0 @@ -description "pywb" - -start on runlevel [2345] -stop on runlevel [!2345] - -env PYTHONPATH=/home/vagrant/pywb-ve34/lib/python3.4/site-packages -env PATH=/home/vagrant/pywb-ve34/bin:/usr/bin:/bin -env PYWB_CONFIG_FILE=/etc/pywb.yml - -setuid vagrant - -# console log - -exec nice brozzler-wayback >>/vagrant/logs/pywb.log 2>&1 diff --git a/vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-vagrant-1.conf.j2 b/vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-vagrant-1.conf.j2 deleted file mode 100644 index 62b3ac5..0000000 --- a/vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-vagrant-1.conf.j2 +++ /dev/null @@ -1,5 +0,0 @@ -runuser=vagrant -bind=0.0.0.0 -# directory=/var/lib/rethinkdb -# log-file=/var/log/rethinkdb.log -log-file=/vagrant/logs/rethinkdb.log # synced dir diff --git a/vagrant/ansible/roles/warcprox/handlers/main.yml b/vagrant/ansible/roles/warcprox/handlers/main.yml deleted file mode 100644 index 698d871..0000000 --- a/vagrant/ansible/roles/warcprox/handlers/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# - name: start warcprox -# environment: -# PYTHONPATH: /home/vagrant/warcprox-ve34/lib/python3.4/site-packages -# PATH: /home/vagrant/warcprox-ve34/bin:/usr/bin:/bin -# args: -# executable: /bin/bash -# shell: nice warcprox --dir=/vagrant/warcs --base32 --gzip -# --rollover-idle-time=180 --cacert=/vagrant/warcprox-ca.pem -# --onion-tor-socks-proxy=localhost:9050 --rethinkdb-servers=localhost -# --rethinkdb-big-table &> /vagrant/logs/warcprox.out & -- name: restart warcprox - service: name=warcprox state=restarted - become: true diff --git a/vagrant/ansible/roles/warcprox/templates/warcprox.conf.j2 b/vagrant/ansible/roles/warcprox/templates/warcprox.conf.j2 deleted file mode 100644 index 1afccce..0000000 --- a/vagrant/ansible/roles/warcprox/templates/warcprox.conf.j2 +++ /dev/null @@ -1,26 +0,0 @@ -description "warcprox" - -start on runlevel [2345] -stop on runlevel [!2345] - -env PYTHONPATH=/home/vagrant/warcprox-ve34/lib/python3.4/site-packages -env PATH=/home/vagrant/warcprox-ve34/bin:/usr/bin:/bin - -# by default warcprox creates some files/dirs relative to cwd -chdir /home/vagrant -setuid vagrant - -# console log - -# --profile -exec nice warcprox \ - --dir=/vagrant/warcs \ - --base32 \ - --gzip \ - --rollover-idle-time=180 \ - --cacert=/vagrant/warcprox-ca.pem \ - --onion-tor-socks-proxy=localhost:9050 \ - --rethinkdb-servers=localhost \ - --rethinkdb-db=brozzler \ - --rethinkdb-big-table >>/vagrant/logs/warcprox.log 2>&1 - # --rethinkdb-servers={{groups['rethinkdb'] | join(',')}} \