Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

2 instances of warcprox; no docker for brozzler worker

Browse Source
This commit is contained in:
Noah Levitt 2015-11-12 02:59:21 +00:00
parent 4c2ecab856
commit 3bcd2400f7
1 changed files with 11 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
cluster-control.sh
View File

@ -9,7 +9,7 @@ _status() {
something_running=1 something_running=1
warcprox_pids=( $(pgrep -f /home/nlevitt/workspace/warcprox/warcprox-ve34/bin/warcprox) ) warcprox_pids=( $(pgrep -f /home/nlevitt/workspace/warcprox/warcprox-ve34/bin/warcprox) )
worker_pids=( $(pgrep -f 'ssh .* docker run .* internetarchive/brozzler-worker .* brozzler-worker') ) worker_pids=( $(pgrep -f 'ssh .*brozzler-worker') )
pywayback_pids=( $(pgrep -f /home/nlevitt/workspace/pygwb/pygwb-ve27/bin/gunicorn) ) pywayback_pids=( $(pgrep -f /home/nlevitt/workspace/pygwb/pygwb-ve27/bin/gunicorn) )
ait_brozzler_boss=( $(pgrep -f /home/nlevitt/workspace/ait5/scripts/ait-brozzler-boss.py) ) ait_brozzler_boss=( $(pgrep -f /home/nlevitt/workspace/ait5/scripts/ait-brozzler-boss.py) )
ait5_pids=( $(pgrep -f 0.0.0.0:8888) ) ait5_pids=( $(pgrep -f 0.0.0.0:8888) )
@ -39,19 +39,15 @@ _stop() {
pkill -f /home/nlevitt/workspace/ait5/scripts/ait-brozzler-boss.py pkill -f /home/nlevitt/workspace/ait5/scripts/ait-brozzler-boss.py
pkill -f 0.0.0.0:8888 pkill -f 0.0.0.0:8888
pkill -f app=.*brozzler-webconsole.py pkill -f app=.*brozzler-webconsole.py
set +x
for node in aidata{400,401}{,-bu} ; do for node in aidata{400,401}{,-bu} ; do
container_id=$(ssh $node docker ps --filter=image=internetarchive/brozzler-worker --filter=status=running --format='{{.ID}}') ssh $node pkill -f brozzler-worker
[ -n "$container_id" ] && (set -x ; ssh $node docker stop --time=60 $container_id )
done done
for node in wbgrp-svc{110,111} ; do
ssh $node pkill -f /home/nlevitt/workspace/warcprox/warcprox-ve34/bin/warcprox
done
set +x
fi fi
set -x
for node in wbgrp-svc{110,111} ; do
ssh $node pkill -f /home/nlevitt/workspace/warcprox/warcprox-ve34/bin/warcprox
done
set +x
if _status > /dev/null ; then if _status > /dev/null ; then
while _status > /dev/null ; do sleep 0.5 ; done while _status > /dev/null ; do sleep 0.5 ; done
fi fi
@ -122,7 +118,7 @@ EOF
start_warcprox() { start_warcprox() {
echo $0: starting warcprox echo $0: starting warcprox
for node in wbgrp-svc{110,111} ; do for node in wbgrp-svc{110,111} ; do
ssh -fn $node 'PYTHONPATH=/home/nlevitt/workspace/warcprox/warcprox-ve34/lib/python3.4/site-packages nice /home/nlevitt/workspace/warcprox/warcprox-ve34/bin/warcprox --dir=/1/brzl/warcs --rethinkdb-servers=wbgrp-svc020,wbgrp-svc035,wbgrp-svc036 --rethinkdb-db=archiveit_brozzler --rethinkdb-big-table --cacert=/1/brzl/warcprox-ca.pem --certs-dir=/1/brzl/certs --address=0.0.0.0 --base32 --gzip --rollover-idle-time=180 --kafka-broker-list=qa-archive-it.org:6092 --kafka-capture-feed-topic=ait-brozzler-captures' &>>/1/brzl/logs/warcprox-$node.out & ssh -fn $node 'PYTHONPATH=/home/nlevitt/workspace/warcprox/warcprox-ve34/lib/python3.4/site-packages nice /home/nlevitt/workspace/warcprox/warcprox-ve34/bin/warcprox --profile --dir=/1/brzl/warcs --rethinkdb-servers=wbgrp-svc020,wbgrp-svc035,wbgrp-svc036 --rethinkdb-db=archiveit_brozzler --rethinkdb-big-table --cacert=/1/brzl/warcprox-ca.pem --certs-dir=/1/brzl/certs --address=0.0.0.0 --base32 --gzip --rollover-idle-time=180 --kafka-broker-list=qa-archive-it.org:6092 --kafka-capture-feed-topic=ait-brozzler-captures' &>>/1/brzl/logs/warcprox-$node.out &
while ! rethinkdb_tables_ready stats:0 captures:2 services:0 ; do while ! rethinkdb_tables_ready stats:0 captures:2 services:0 ; do
sleep 1 ; sleep 1 ;
done done
@ -139,13 +135,8 @@ start_brozzler_boss() {
start_brozzler_workers() { start_brozzler_workers() {
echo $0: starting brozzler-workers echo $0: starting brozzler-workers
for node in aidata{400,401}{,-bu} ; do for node in aidata{400,401}{,-bu} ; do
( ssh -fn $node DISPLAY=:1 XAUTHORITY=/tmp/Xauthority.nlevitt PYTHONPATH=/home/nlevitt/workspace/brozzler/brozzler-ve34/lib/python3.4/site-packages:/home/nlevitt/workspace/brozzler /home/nlevitt/workspace/brozzler/brozzler-ve34/bin/brozzler-worker --rethinkdb-servers=wbgrp-svc036,wbgrp-svc020,wbgrp-svc035 --rethinkdb-db=archiveit_brozzler --max-browsers=10 &>> /1/brzl/logs/brozzler-worker-$node.out
set -x while ! rethinkdb_tables_ready jobs:0 pages:1 sites:2 ; do sleep 1 ; done
ssh $node "docker --version || curl -sSL https://get.docker.com/ | sh && sudo usermod -aG docker $USER"
ssh $node 'docker build -t internetarchive/brozzler-worker /home/nlevitt/workspace/brozzler/docker'
ssh -fn $node 'docker run -t --rm -p 8901:8901 -p 5901:5901 internetarchive/brozzler-worker /sbin/my_init -- setuser brozzler bash -c "DISPLAY=:1 brozzler-worker --rethinkdb-servers=wbgrp-svc036,wbgrp-svc020,wbgrp-svc035 --rethinkdb-db=archiveit_brozzler --max-browsers=10"' &>> /1/brzl/logs/brozzler-worker-$node.out
)
while ! rethinkdb_tables_ready jobs:0 pages:1 sites:2 ; do sleep 1 ; done
done done
} }
@ -157,7 +148,7 @@ start_pywayback() {
start_ait5() { start_ait5() {
echo $0: starting ait5 partner webapp echo $0: starting ait5 partner webapp
PYTHONPATH=/home/nlevitt/workspace/ait5/ait5-ve34/lib/python3.4/site-packages python3.4 /home/nlevitt/workspace/ait5/manage.py runserver_plus 0.0.0.0:8888 &>> /1/brzl/logs/ait5.out & PYTHONPATH=/home/nlevitt/workspace/ait5/ait5-ve34/lib/python3.4/site-packages /home/nlevitt/workspace/ait5/ait5-ve34/bin/python /home/nlevitt/workspace/ait5/manage.py runserver_plus 0.0.0.0:8888 &>> /1/brzl/logs/ait5.out &
} }
start_brozzler_console() { start_brozzler_console() {
@ -167,7 +158,7 @@ start_brozzler_console() {
start_dead() { start_dead() {
warcprox_pids=( $(pgrep -f /home/nlevitt/workspace/warcprox/warcprox-ve34/bin/warcprox) ) warcprox_pids=( $(pgrep -f /home/nlevitt/workspace/warcprox/warcprox-ve34/bin/warcprox) )
worker_pids=( $(pgrep -f 'ssh .* docker run .* internetarchive/brozzler-worker .* brozzler-worker') ) worker_pids=( $(pgrep -f 'ssh .*brozzler-worker') )
pywayback_pids=( $(pgrep -f /home/nlevitt/workspace/pygwb/pygwb-ve27/bin/gunicorn) ) pywayback_pids=( $(pgrep -f /home/nlevitt/workspace/pygwb/pygwb-ve27/bin/gunicorn) )
ait_brozzler_boss=( $(pgrep -f /home/nlevitt/workspace/ait5/scripts/ait-brozzler-boss.py) ) ait_brozzler_boss=( $(pgrep -f /home/nlevitt/workspace/ait5/scripts/ait-brozzler-boss.py) )
ait5_pids=( $(pgrep -f 0.0.0.0:8888) ) ait5_pids=( $(pgrep -f 0.0.0.0:8888) )