mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-24 08:39:59 -05:00
46 lines
1.7 KiB
Python
Executable File
46 lines
1.7 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# vim: set sw=4 et:
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import logging
|
|
import brozzler
|
|
import brozzler.hq
|
|
import signal
|
|
|
|
arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
|
|
description="brozzler-hq - headquarters of distributed brozzler crawl",
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
arg_parser.add_argument("-d", "--db", dest="db_file", default="./brozzler-hq-0.db",
|
|
help="sqlite3 database filename; if the file does not exist, it will be created")
|
|
arg_parser.add_argument('-u', '--url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f',
|
|
help='URL identifying the amqp server to talk to')
|
|
arg_parser.add_argument("-v", "--verbose", dest="log_level",
|
|
action="store_const", default=logging.INFO, const=logging.DEBUG)
|
|
arg_parser.add_argument("--version", action="version",
|
|
version="brozzler {} - {}".format(brozzler.version, os.path.basename(__file__)))
|
|
args = arg_parser.parse_args(args=sys.argv[1:])
|
|
|
|
logging.basicConfig(stream=sys.stdout, level=args.log_level,
|
|
format="%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s")
|
|
|
|
def sigterm(signum, frame):
|
|
raise brozzler.ShutdownRequested("shutdown requested (caught SIGTERM)")
|
|
def sigint(signum, frame):
|
|
raise brozzler.ShutdownRequested("shutdown requested (caught SIGINT)")
|
|
|
|
signal.signal(signal.SIGTERM, sigterm)
|
|
signal.signal(signal.SIGINT, sigint)
|
|
|
|
logging.info("brozzler-hq starting")
|
|
|
|
db = brozzler.hq.BrozzlerHQDb(db_file=args.db_file)
|
|
hq = brozzler.hq.BrozzlerHQ(amqp_url=args.amqp_url, db=db)
|
|
|
|
try:
|
|
hq.run()
|
|
except brozzler.ShutdownRequested as e:
|
|
logging.info("{}".format(e))
|
|
|