diff --git a/setup.py b/setup.py index 083a52b..d37f14a 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b6.dev81', + version='1.1b6.dev82', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt', diff --git a/vagrant/ansible/hosts b/vagrant/ansible/hosts index 708a07a..0257aeb 100644 --- a/vagrant/ansible/hosts +++ b/vagrant/ansible/hosts @@ -13,4 +13,4 @@ ansible_ssh_private_key_file=.vagrant/machines/10.9.9.9/virtualbox/private_key 10.9.9.9 [pywb] -10.9.9.9 \ No newline at end of file +10.9.9.9 diff --git a/vagrant/ansible/playbook.yml b/vagrant/ansible/playbook.yml index fda30b8..f55a4b1 100644 --- a/vagrant/ansible/playbook.yml +++ b/vagrant/ansible/playbook.yml @@ -24,7 +24,7 @@ roles: - brozzler-webconsole -# - name: deploy pywb -# hosts: pywb -# roles: -# - pywb +- name: deploy pywb + hosts: pywb + roles: + - pywb diff --git a/vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml b/vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml index f0f70d8..1d7194a 100644 --- a/vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml +++ b/vagrant/ansible/roles/brozzler-webconsole/tasks/main.yml @@ -1,6 +1,5 @@ --- - name: install brozzler[webconsole] in virtualenv - become: true pip: name='-e /brozzler[webconsole]' virtualenv=/home/vagrant/brozzler-webconsole-ve34 virtualenv_python=python3.4 @@ -12,4 +11,4 @@ template: src=templates/brozzler-webconsole.conf.j2 dest=/etc/init/brozzler-webconsole.conf notify: - - restart brozzler-webconsole \ No newline at end of file + - restart brozzler-webconsole diff --git a/vagrant/ansible/roles/brozzler-worker/tasks/main.yml b/vagrant/ansible/roles/brozzler-worker/tasks/main.yml index a4ec194..7dad56a 100644 --- a/vagrant/ansible/roles/brozzler-worker/tasks/main.yml +++ b/vagrant/ansible/roles/brozzler-worker/tasks/main.yml @@ -26,7 +26,6 @@ - ttf-indic-fonts - fonts-thai-tlwg - fonts-lklug-sinhala - - python3-pip - git - libjpeg-turbo8-dev - zlib1g-dev @@ -49,7 +48,6 @@ notify: - restart vnc-websock - name: install brozzler in virtualenv - become: true pip: # name=git+https://github.com/internetarchive/brozzler.git#egg=brozzler name='-e /brozzler' virtualenv=/home/vagrant/brozzler-ve34 diff --git a/vagrant/ansible/roles/common/tasks/main.yml b/vagrant/ansible/roles/common/tasks/main.yml index f9012ca..9a14357 100644 --- a/vagrant/ansible/roles/common/tasks/main.yml +++ b/vagrant/ansible/roles/common/tasks/main.yml @@ -1,4 +1,24 @@ --- -- name: ensure logs directory exists +## # get latest pip (had problems with version from apt-get, specifically +## # "pip install pyopenssl" did not install the dependency "cryptography") +## # http://stackoverflow.com/questions/34587473/what-is-get-pip-py-checksum-where-can-i-get-it-for-sure +## - name: install setuptools for python 2 and 3 +## become: true +## apt: name={{item}} state=present +## with_items: +## - python-setuptools +## - python3-setuptools +## - name: download pip-8.1.2.tar.gz +## get_url: +## url: https://pypi.python.org/packages/e7/a8/7556133689add8d1a54c0b14aeff0acb03c64707ce100ecd53934da1aa13/pip-8.1.2.tar.gz +## dest: /tmp +## checksum: sha1:1c13c247967ec5bee6de5fd104c5d78ba30951c7 +## - name: extract pip-8.1.2.tar.gz +## unarchive: src=/tmp/pip-8.1.2.tar.gz dest=/tmp copy=no +## - name: run "python3 setup.py install" in /tmp/pip-8.1.2 +## command: python3 setup.py install chdir=/tmp/pip-8.1.2 +## creates=/usr/local/lib/python2.7/dist-packages/pip-8.1.2-py2.7.egg/pip/__init__.py +## become: true +- name: mkdir /vagrant/logs file: path=/vagrant/logs state=directory become: true diff --git a/vagrant/ansible/roles/pywb/handlers/main.yml b/vagrant/ansible/roles/pywb/handlers/main.yml new file mode 100644 index 0000000..4424b3e --- /dev/null +++ b/vagrant/ansible/roles/pywb/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart pywb + service: name=pywb state=restarted + become: true + diff --git a/vagrant/ansible/roles/pywb/tasks/main.yml b/vagrant/ansible/roles/pywb/tasks/main.yml new file mode 100644 index 0000000..08fac19 --- /dev/null +++ b/vagrant/ansible/roles/pywb/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: install pywb in virtualenv + pip: name=pywb + virtualenv=/home/vagrant/pywb-ve34 + virtualenv_python=python3.4 + extra_args='--no-input --upgrade --pre' + notify: + - restart pywb +- name: install brozzler in pywb virtualenv + pip: name='-e /brozzler' + virtualenv=/home/vagrant/pywb-ve34 + virtualenv_python=python3.4 + extra_args='--no-input --upgrade --pre' + notify: + - restart pywb +- name: pywb config file /etc/pywb.yml + template: src=templates/pywb.yml.j2 + dest=/etc/pywb.yml + become: true + notify: + - restart pywb +- name: upstart config file /etc/init/pywb.conf + template: src=templates/pywb.conf.j2 + dest=/etc/init/pywb.conf + become: true + notify: + - restart pywb diff --git a/vagrant/ansible/roles/pywb/templates/pywb.conf.j2 b/vagrant/ansible/roles/pywb/templates/pywb.conf.j2 new file mode 100644 index 0000000..5b2887e --- /dev/null +++ b/vagrant/ansible/roles/pywb/templates/pywb.conf.j2 @@ -0,0 +1,14 @@ +description "pywb" + +start on runlevel [2345] +stop on runlevel [!2345] + +env PYTHONPATH=/home/vagrant/pywb-ve34/lib/python3.4/site-packages +env PATH=/home/vagrant/pywb-ve34/bin:/usr/bin:/bin +env PYWB_CONFIG_FILE=/etc/pywb.yml + +setuid vagrant + +# console log + +exec nice brozzler-wayback >>/vagrant/logs/pywb.log 2>&1 diff --git a/vagrant/ansible/roles/pywb/templates/pywb.yml.j2 b/vagrant/ansible/roles/pywb/templates/pywb.yml.j2 new file mode 100644 index 0000000..c17281c --- /dev/null +++ b/vagrant/ansible/roles/pywb/templates/pywb.yml.j2 @@ -0,0 +1,12 @@ +archive_paths: /vagrant/warcs/ +collections: + brozzler: + index_paths: !!python/object:brozzler.pywb.RethinkCDXSource + db: brozzler + servers: [localhost] + table: captures +enable_auto_colls: false +enable_cdx_api: true +framed_replay: true +port: 8880 + diff --git a/vagrant/ansible/roles/rethinkdb/tasks/main.yml b/vagrant/ansible/roles/rethinkdb/tasks/main.yml index 77bbb89..7f40ff4 100644 --- a/vagrant/ansible/roles/rethinkdb/tasks/main.yml +++ b/vagrant/ansible/roles/rethinkdb/tasks/main.yml @@ -10,12 +10,14 @@ apt: name=rethinkdb state=present become: true notify: - - restart rethinkdb + - restart rethinkdb +# XXX rethinkdb fails to start in spite of this, I think because /vagrant +# gets mounted too late, and it tries to log there - name: ensure rethinkdb starts on reboot service: name=rethinkdb enabled=yes - name: ensure rethinkdb instance config file is installed - template: src=templates/rethinkdb-brozzler-easy.conf.j2 - dest=/etc/rethinkdb/instances.d/rethinkdb-brozzler-easy.conf + template: src=templates/rethinkdb-brozzler-vagrant-1.conf.j2 + dest=/etc/rethinkdb/instances.d/rethinkdb-brozzler-vagrant-1.conf become: true notify: - - restart rethinkdb + - restart rethinkdb diff --git a/vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-easy.conf.j2 b/vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-easy.conf.j2 deleted file mode 100644 index 62b3ac5..0000000 --- a/vagrant/ansible/roles/rethinkdb/templates/rethinkdb-brozzler-easy.conf.j2 +++ /dev/null @@ -1,5 +0,0 @@ -runuser=vagrant -bind=0.0.0.0 -# directory=/var/lib/rethinkdb -# log-file=/var/log/rethinkdb.log -log-file=/vagrant/logs/rethinkdb.log # synced dir diff --git a/vagrant/ansible/roles/warcprox/tasks/main.yml b/vagrant/ansible/roles/warcprox/tasks/main.yml index c9f611d..7a9c7d0 100644 --- a/vagrant/ansible/roles/warcprox/tasks/main.yml +++ b/vagrant/ansible/roles/warcprox/tasks/main.yml @@ -3,23 +3,23 @@ become: true apt: name={{item}} state=present with_items: - - gcc - - python-virtualenv - - python3.4 - - libpython3.4-dev - - libffi-dev - - libssl-dev - - tor - - git + - gcc + - python-virtualenv + - python3.4 + - libpython3.4-dev + - libffi-dev + - libssl-dev + - tor + - git - name: install warcprox in virtualenv pip: name=git+https://github.com/internetarchive/warcprox.git@2.x#egg=warcprox virtualenv=/home/vagrant/warcprox-ve34 virtualenv_python=python3.4 extra_args='--no-input --upgrade --pre' notify: - - restart warcprox + - restart warcprox - name: install upstart config /etc/init/warcprox.conf become: true template: src=templates/warcprox.conf.j2 dest=/etc/init/warcprox.conf notify: - - restart warcprox + - restart warcprox