mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-04-21 08:06:27 -04:00
Merge branch 'master' into qa
* master: support for BROZZLER_RETHINKDB_SERVERS and BROZZLER_RETHINKDB_DB environment variables, honored by all the brozzler-* commands
This commit is contained in:
Noah Levitt
commit
049c2688bf
@ -58,13 +58,21 @@ def _add_common_options(arg_parser):
|
||||
|
||||
def _add_rethinkdb_options(arg_parser):
|
||||
arg_parser.add_argument(
|
||||
'--rethinkdb-servers', dest='rethinkdb_servers',
|
||||
default='localhost', help=(
|
||||
'--rethinkdb-servers', dest='rethinkdb_servers', help=(
|
||||
'rethinkdb servers, e.g. '
|
||||
'db0.foo.org,db0.foo.org:38015,db1.foo.org'))
|
||||
'db0.foo.org,db0.foo.org:38015,db1.foo.org (takes precedence '
|
||||
'over environment variable BROZZLER_RETHINKDB_SERVERS)'))
|
||||
arg_parser.add_argument(
|
||||
'--rethinkdb-db', dest='rethinkdb_db', default='brozzler',
|
||||
help='rethinkdb database name')
|
||||
'--rethinkdb-db', dest='rethinkdb_db', help=(
|
||||
'rethinkdb database name (takes precedence over '
|
||||
'environment variable BROZZLER_RETHINKDB_DB)'))
|
||||
|
||||
def rethinker(args):
|
||||
servers = args.rethinkdb_servers or os.environ.get(
|
||||
'BROZZLER_RETHINKDB_SERVERS') or 'localhost'
|
||||
db = args.rethinkdb_db or os.environ.get(
|
||||
'BROZZLER_RETHINKDB_DB') or 'brozzler'
|
||||
return rethinkstuff.Rethinker(servers.split(','), db)
|
||||
|
||||
def _add_proxy_options(arg_parser):
|
||||
arg_parser.add_argument(
|
||||
@ -197,8 +205,7 @@ def brozzler_new_job():
|
||||
args = arg_parser.parse_args(args=sys.argv[1:])
|
||||
_configure_logging(args)
|
||||
|
||||
r = rethinkstuff.Rethinker(
|
||||
args.rethinkdb_servers.split(','), args.rethinkdb_db)
|
||||
r = rethinker(args)
|
||||
frontier = brozzler.RethinkDbFrontier(r)
|
||||
try:
|
||||
brozzler.job.new_job_file(frontier, args.job_conf_file)
|
||||
@ -260,8 +267,7 @@ def brozzler_new_site():
|
||||
args.behavior_parameters) if args.behavior_parameters else None,
|
||||
username=args.username, password=args.password)
|
||||
|
||||
r = rethinkstuff.Rethinker(
|
||||
args.rethinkdb_servers.split(","), args.rethinkdb_db)
|
||||
r = rethinker()
|
||||
frontier = brozzler.RethinkDbFrontier(r)
|
||||
brozzler.new_site(frontier, site)
|
||||
|
||||
@ -316,8 +322,7 @@ def brozzler_worker():
|
||||
signal.signal(signal.SIGTERM, sigterm)
|
||||
signal.signal(signal.SIGINT, sigint)
|
||||
|
||||
r = rethinkstuff.Rethinker(
|
||||
args.rethinkdb_servers.split(','), args.rethinkdb_db)
|
||||
r = rethinker(args)
|
||||
frontier = brozzler.RethinkDbFrontier(r)
|
||||
service_registry = rethinkstuff.ServiceRegistry(r)
|
||||
worker = brozzler.worker.BrozzlerWorker(
|
||||
@ -345,8 +350,7 @@ def brozzler_ensure_tables():
|
||||
args = arg_parser.parse_args(args=sys.argv[1:])
|
||||
_configure_logging(args)
|
||||
|
||||
r = rethinkstuff.Rethinker(
|
||||
args.rethinkdb_servers.split(','), args.rethinkdb_db)
|
||||
r = rethinker(args)
|
||||
|
||||
# services table
|
||||
rethinkstuff.ServiceRegistry(r)
|
||||
@ -376,8 +380,7 @@ def brozzler_list_jobs():
|
||||
args = arg_parser.parse_args(args=sys.argv[1:])
|
||||
_configure_logging(args)
|
||||
|
||||
r = rethinkstuff.Rethinker(
|
||||
args.rethinkdb_servers.split(','), args.rethinkdb_db)
|
||||
r = rethinker(args)
|
||||
reql = r.table('jobs').order_by('id')
|
||||
if not args.all:
|
||||
reql = reql.filter({'status': 'ACTIVE'})
|
||||
@ -406,8 +409,7 @@ def brozzler_list_sites():
|
||||
args = arg_parser.parse_args(args=sys.argv[1:])
|
||||
_configure_logging(args)
|
||||
|
||||
r = rethinkstuff.Rethinker(
|
||||
args.rethinkdb_servers.split(','), args.rethinkdb_db)
|
||||
r = rethinker(args)
|
||||
|
||||
reql = r.table('sites')
|
||||
if args.job:
|
||||
@ -453,8 +455,7 @@ def brozzler_list_pages():
|
||||
args = arg_parser.parse_args(args=sys.argv[1:])
|
||||
_configure_logging(args)
|
||||
|
||||
r = rethinkstuff.Rethinker(
|
||||
args.rethinkdb_servers.split(','), args.rethinkdb_db)
|
||||
r = rethinker(args)
|
||||
if args.job:
|
||||
try:
|
||||
job_id = int(args.job)
|
||||
@ -516,8 +517,7 @@ def brozzler_list_captures():
|
||||
args = arg_parser.parse_args(args=sys.argv[1:])
|
||||
_configure_logging(args)
|
||||
|
||||
r = rethinkstuff.Rethinker(
|
||||
args.rethinkdb_servers.split(','), args.rethinkdb_db)
|
||||
r = rethinker(args)
|
||||
|
||||
if args.url_or_sha1[:5] == 'sha1:':
|
||||
if args.prefix:
|
||||
|
||||
@ -52,8 +52,8 @@ app.logger.setLevel(logging.INFO)
|
||||
# configure with environment variables
|
||||
SETTINGS = {
|
||||
'RETHINKDB_SERVERS': os.environ.get(
|
||||
'RETHINKDB_SERVERS', 'localhost').split(','),
|
||||
'RETHINKDB_DB': os.environ.get('RETHINKDB_DB', 'brozzler'),
|
||||
'BROZZLER_RETHINKDB_SERVERS', 'localhost').split(','),
|
||||
'RETHINKDB_DB': os.environ.get('BROZZLER_RETHINKDB_DB', 'brozzler'),
|
||||
'WAYBACK_BASEURL': os.environ.get(
|
||||
'WAYBACK_BASEURL', 'http://localhost:8091/brozzler'),
|
||||
}
|
||||
@ -246,10 +246,11 @@ def main():
|
||||
epilog=(
|
||||
'brozzler-dashboard has no command line options, but can be '
|
||||
'configured using the following environment variables:\n\n'
|
||||
' RETHINKDB_SERVERS rethinkdb servers, e.g. db0.foo.org,'
|
||||
'db0.foo.org:38015,db1.foo.org (default: localhost)\n'
|
||||
' RETHINKDB_DB rethinkdb database name (default: '
|
||||
'brozzler)\n'
|
||||
' BROZZLER_RETHINKDB_SERVERS rethinkdb servers, e.g. '
|
||||
'db0.foo.org,db0.foo.org:38015,db1.foo.org (default: '
|
||||
'localhost)\n'
|
||||
' BROZZLER_RETHINKDB_DB rethinkdb database name '
|
||||
'(default: brozzler)\n'
|
||||
' WAYBACK_BASEURL base url for constructing wayback '
|
||||
'links (default http://localhost:8091/brozzler)'))
|
||||
args = arg_parser.parse_args(args=sys.argv[1:])
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user