#!/usr/bin/env python # vim: set sw=4 et: import argparse import os import sys import logging import brozzler import threading import time import signal import pprint import traceback import brozzler.worker arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__), description='crawl-url - browse urls, follow links', formatter_class=argparse.ArgumentDefaultsHelpFormatter) arg_parser.add_argument('-u', '--url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f', help='URL identifying the amqp server to talk to') arg_parser.add_argument('-e', '--executable', dest='chrome_exe', default='chromium-browser', help='executable to use to invoke chrome') arg_parser.add_argument('-n', '--max-browsers', dest='max_browsers', default='1', help='max number of chrome instances simultaneously browsing pages') arg_parser.add_argument('-v', '--verbose', dest='log_level', action="store_const", default=logging.INFO, const=logging.DEBUG) arg_parser.add_argument('--version', action='version', version="brozzler {} - {}".format(brozzler.version, os.path.basename(__file__))) args = arg_parser.parse_args(args=sys.argv[1:]) logging.basicConfig(stream=sys.stdout, level=args.log_level, format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s') # the way we're using it, amqp is too verbose at debug level logging.getLogger("amqp").setLevel(logging.INFO) def sigterm(signum, frame): raise brozzler.ShutdownRequested('shutdown requested (caught SIGTERM)') def sigint(signum, frame): raise brozzler.ShutdownRequested('shutdown requested (caught SIGINT)') def dump_state(signum, frame): pp = pprint.PrettyPrinter(indent=4) state_strs = [] for th in threading.enumerate(): state_strs.append(str(th)) stack = traceback.format_stack(sys._current_frames()[th.ident]) state_strs.append("".join(stack)) logging.warn("dumping state (caught signal {})\n{}".format(signum, "\n".join(state_strs))) signal.signal(signal.SIGQUIT, dump_state) signal.signal(signal.SIGTERM, sigterm) signal.signal(signal.SIGINT, sigint) worker = brozzler.worker.BrozzlerWorker(amqp_url=args.amqp_url, max_browsers=int(args.max_browsers), chrome_exe=args.chrome_exe) worker.start() try: while True: time.sleep(0.5) except brozzler.ShutdownRequested as e: worker.shutdown_now() for th in threading.enumerate(): if th != threading.current_thread(): th.join() logging.info("all done, exiting")