mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-24 08:39:59 -05:00
33 lines
1.4 KiB
Python
Executable File
33 lines
1.4 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import logging
|
|
import brozzler
|
|
import yaml
|
|
import json
|
|
import rethinkstuff
|
|
|
|
arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
|
|
description="brozzler-new-job - queue new job with brozzler",
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
arg_parser.add_argument('job_conf_file', metavar='JOB_CONF_FILE', help='brozzler job configuration file in yaml')
|
|
arg_parser.add_argument('--rethinkdb-servers', dest='rethinkdb_servers', default="localhost",
|
|
help='rethinkdb servers, e.g. db0.foo.org,db0.foo.org:38015,db1.foo.org')
|
|
arg_parser.add_argument('--rethinkdb-db', dest='rethinkdb_db', default="brozzler",
|
|
help='rethinkdb database name')
|
|
arg_parser.add_argument("-v", "--verbose", dest="log_level",
|
|
action="store_const", default=logging.INFO, const=logging.DEBUG)
|
|
arg_parser.add_argument("--version", action="version",
|
|
version="brozzler {} - {}".format(brozzler.__version__, os.path.basename(__file__)))
|
|
args = arg_parser.parse_args(args=sys.argv[1:])
|
|
|
|
logging.basicConfig(stream=sys.stdout, level=args.log_level,
|
|
format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s")
|
|
|
|
r = rethinkstuff.Rethinker(args.rethinkdb_servers.split(","), args.rethinkdb_db)
|
|
frontier = brozzler.RethinkDbFrontier(r)
|
|
brozzler.job.new_job_file(frontier, args.job_conf_file)
|
|
|