diff --git a/brozzler/cli.py b/brozzler/cli.py index 26efaa9..8724e0f 100644 --- a/brozzler/cli.py +++ b/brozzler/cli.py @@ -58,13 +58,21 @@ def _add_common_options(arg_parser): def _add_rethinkdb_options(arg_parser): arg_parser.add_argument( - '--rethinkdb-servers', dest='rethinkdb_servers', - default='localhost', help=( + '--rethinkdb-servers', dest='rethinkdb_servers', help=( 'rethinkdb servers, e.g. ' - 'db0.foo.org,db0.foo.org:38015,db1.foo.org')) + 'db0.foo.org,db0.foo.org:38015,db1.foo.org (takes precedence ' + 'over environment variable BROZZLER_RETHINKDB_SERVERS)')) arg_parser.add_argument( - '--rethinkdb-db', dest='rethinkdb_db', default='brozzler', - help='rethinkdb database name') + '--rethinkdb-db', dest='rethinkdb_db', help=( + 'rethinkdb database name (takes precedence over ' + 'environment variable BROZZLER_RETHINKDB_DB)')) + +def rethinker(args): + servers = args.rethinkdb_servers or os.environ.get( + 'BROZZLER_RETHINKDB_SERVERS') or 'localhost' + db = args.rethinkdb_db or os.environ.get( + 'BROZZLER_RETHINKDB_DB') or 'brozzler' + return rethinkstuff.Rethinker(servers.split(','), db) def _add_proxy_options(arg_parser): arg_parser.add_argument( @@ -197,8 +205,7 @@ def brozzler_new_job(): args = arg_parser.parse_args(args=sys.argv[1:]) _configure_logging(args) - r = rethinkstuff.Rethinker( - args.rethinkdb_servers.split(','), args.rethinkdb_db) + r = rethinker(args) frontier = brozzler.RethinkDbFrontier(r) try: brozzler.job.new_job_file(frontier, args.job_conf_file) @@ -260,8 +267,7 @@ def brozzler_new_site(): args.behavior_parameters) if args.behavior_parameters else None, username=args.username, password=args.password) - r = rethinkstuff.Rethinker( - args.rethinkdb_servers.split(","), args.rethinkdb_db) + r = rethinker() frontier = brozzler.RethinkDbFrontier(r) brozzler.new_site(frontier, site) @@ -316,8 +322,7 @@ def brozzler_worker(): signal.signal(signal.SIGTERM, sigterm) signal.signal(signal.SIGINT, sigint) - r = rethinkstuff.Rethinker( - args.rethinkdb_servers.split(','), args.rethinkdb_db) + r = rethinker(args) frontier = brozzler.RethinkDbFrontier(r) service_registry = rethinkstuff.ServiceRegistry(r) worker = brozzler.worker.BrozzlerWorker( @@ -345,8 +350,7 @@ def brozzler_ensure_tables(): args = arg_parser.parse_args(args=sys.argv[1:]) _configure_logging(args) - r = rethinkstuff.Rethinker( - args.rethinkdb_servers.split(','), args.rethinkdb_db) + r = rethinker(args) # services table rethinkstuff.ServiceRegistry(r) @@ -376,8 +380,7 @@ def brozzler_list_jobs(): args = arg_parser.parse_args(args=sys.argv[1:]) _configure_logging(args) - r = rethinkstuff.Rethinker( - args.rethinkdb_servers.split(','), args.rethinkdb_db) + r = rethinker(args) reql = r.table('jobs').order_by('id') if not args.all: reql = reql.filter({'status': 'ACTIVE'}) @@ -406,8 +409,7 @@ def brozzler_list_sites(): args = arg_parser.parse_args(args=sys.argv[1:]) _configure_logging(args) - r = rethinkstuff.Rethinker( - args.rethinkdb_servers.split(','), args.rethinkdb_db) + r = rethinker(args) reql = r.table('sites') if args.job: @@ -453,8 +455,7 @@ def brozzler_list_pages(): args = arg_parser.parse_args(args=sys.argv[1:]) _configure_logging(args) - r = rethinkstuff.Rethinker( - args.rethinkdb_servers.split(','), args.rethinkdb_db) + r = rethinker(args) if args.job: try: job_id = int(args.job) @@ -516,8 +517,7 @@ def brozzler_list_captures(): args = arg_parser.parse_args(args=sys.argv[1:]) _configure_logging(args) - r = rethinkstuff.Rethinker( - args.rethinkdb_servers.split(','), args.rethinkdb_db) + r = rethinker(args) if args.url_or_sha1[:5] == 'sha1:': if args.prefix: diff --git a/brozzler/dashboard/__init__.py b/brozzler/dashboard/__init__.py index 3abe8f8..a1adc9b 100644 --- a/brozzler/dashboard/__init__.py +++ b/brozzler/dashboard/__init__.py @@ -52,8 +52,8 @@ app.logger.setLevel(logging.INFO) # configure with environment variables SETTINGS = { 'RETHINKDB_SERVERS': os.environ.get( - 'RETHINKDB_SERVERS', 'localhost').split(','), - 'RETHINKDB_DB': os.environ.get('RETHINKDB_DB', 'brozzler'), + 'BROZZLER_RETHINKDB_SERVERS', 'localhost').split(','), + 'RETHINKDB_DB': os.environ.get('BROZZLER_RETHINKDB_DB', 'brozzler'), 'WAYBACK_BASEURL': os.environ.get( 'WAYBACK_BASEURL', 'http://localhost:8091/brozzler'), } @@ -246,10 +246,11 @@ def main(): epilog=( 'brozzler-dashboard has no command line options, but can be ' 'configured using the following environment variables:\n\n' - ' RETHINKDB_SERVERS rethinkdb servers, e.g. db0.foo.org,' - 'db0.foo.org:38015,db1.foo.org (default: localhost)\n' - ' RETHINKDB_DB rethinkdb database name (default: ' - 'brozzler)\n' + ' BROZZLER_RETHINKDB_SERVERS rethinkdb servers, e.g. ' + 'db0.foo.org,db0.foo.org:38015,db1.foo.org (default: ' + 'localhost)\n' + ' BROZZLER_RETHINKDB_DB rethinkdb database name ' + '(default: brozzler)\n' ' WAYBACK_BASEURL base url for constructing wayback ' 'links (default http://localhost:8091/brozzler)')) args = arg_parser.parse_args(args=sys.argv[1:]) diff --git a/setup.py b/setup.py index 7c6720d..9b57433 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.1b9.dev168', + version='1.1b9.dev169', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt',