Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-19 12:24:20 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

tweaks that have been sitting here

Browse source
This commit is contained in:
Noah Levitt 2016-02-10 00:38:48 +00:00
parent 887eadb99a
commit 664bb33add
1 changed files with 9 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

13
cluster-control.sh
View file

@ -52,6 +52,10 @@ _stop() {
while _status > /dev/null ; do sleep 0.5 ; done while _status > /dev/null ; do sleep 0.5 ; done
fi fi
for node in aidata{400,401}{,-bu} ; do
ssh $node killall chromium-browser
done
echo "$0: all services stopped" echo "$0: all services stopped"
} }
@ -118,17 +122,14 @@ EOF
start_warcprox() { start_warcprox() {
echo $0: starting warcprox echo $0: starting warcprox
for node in wbgrp-svc{110,111} ; do for node in wbgrp-svc{110,111} ; do
ssh -fn $node 'PYTHONPATH=/home/nlevitt/workspace/warcprox/warcprox-ve34/lib/python3.4/site-packages nice /home/nlevitt/workspace/warcprox/warcprox-ve34/bin/warcprox --profile --dir=/1/brzl/warcs --rethinkdb-servers=wbgrp-svc020,wbgrp-svc035,wbgrp-svc036 --rethinkdb-db=archiveit_brozzler --rethinkdb-big-table --cacert=/1/brzl/warcprox-ca.pem --certs-dir=/1/brzl/certs --address=0.0.0.0 --base32 --gzip --rollover-idle-time=180 --kafka-broker-list=qa-archive-it.org:6092 --kafka-capture-feed-topic=ait-brozzler-captures' &>>/1/brzl/logs/warcprox-$node.out & ssh -fn $node 'PYTHONPATH=/home/nlevitt/workspace/warcprox/warcprox-ve34/lib/python3.4/site-packages nice /home/nlevitt/workspace/warcprox/warcprox-ve34/bin/warcprox --profile --dir=/1/brzl/warcs --rethinkdb-servers=wbgrp-svc020,wbgrp-svc035,wbgrp-svc036 --rethinkdb-db=archiveit_brozzler --rethinkdb-big-table --cacert=/1/brzl/warcprox-ca.pem --certs-dir=/1/brzl/certs --address=0.0.0.0 --base32 --gzip --rollover-idle-time=180 --kafka-broker-list=qa-archive-it.org:6092 --kafka-capture-feed-topic=ait-brozzler-captures --onion-tor-socks-proxy=localhost:9050' &>>/1/brzl/logs/warcprox-$node.out &
while ! rethinkdb_tables_ready stats:0 captures:2 services:0 ; do while ! rethinkdb_tables_ready stats:0 captures:2 services:0 ; do sleep 1 ; done
sleep 1 ;
done
done done
} }
start_brozzler_boss() { start_brozzler_boss() {
echo $0: starting ait-brozzler-boss.py echo $0: starting ait-brozzler-boss.py
venv=/home/nlevitt/workspace/ait5/ait5-ve34 PYTHONPATH=/home/nlevitt/workspace/ait5/ait5-ve34/lib/python3.4/site-packages /home/nlevitt/workspace/ait5/ait5-ve34/bin/python /home/nlevitt/workspace/ait5/scripts/ait-brozzler-boss.py &>> /1/brzl/logs/ait-brozzler-boss.out &
PYTHONPATH=$venv/lib/python3.4/site-packages $venv/bin/python /home/nlevitt/workspace/ait5/scripts/ait-brozzler-boss.py &>> /1/brzl/logs/ait-brozzler-boss.out &
while ! rethinkdb_tables_ready stats:0 services:0 jobs:0 pages:1 sites:2 ; do sleep 1 ; done while ! rethinkdb_tables_ready stats:0 services:0 jobs:0 pages:1 sites:2 ; do sleep 1 ; done
} }