#!/usr/bin/env python # vim: set sw=4 et: import logging import argparse import time import umbra import sys import signal import os import umbra import signal import threading import traceback import pprint if __name__ == "__main__": arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__), description='umbra - browser automation tool communicating via AMQP', formatter_class=argparse.ArgumentDefaultsHelpFormatter) arg_parser.add_argument('-e', '--executable', dest='chrome_exe', default='chromium-browser', help='Executable to use to invoke chrome') arg_parser.add_argument('-u', '--url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f', help='URL identifying the amqp server to talk to') arg_parser.add_argument('--exchange', dest='amqp_exchange', default='umbra', help='AMQP exchange name') arg_parser.add_argument('--queue', dest='amqp_queue', default='urls', help='AMQP queue to consume urls from') arg_parser.add_argument('-n', '--max-browsers', dest='max_browsers', default='1', help='Max number of chrome instances simultaneously browsing pages') arg_parser.add_argument('-v', '--verbose', dest='log_level', action="store_const", default=logging.INFO, const=logging.DEBUG) arg_parser.add_argument('--version', action='version', version="umbra {}".format(umbra.version)) args = arg_parser.parse_args(args=sys.argv[1:]) logging.basicConfig(stream=sys.stdout, level=args.log_level, format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s') logging.info("umbra {} starting up".format(umbra.version)) controller = umbra.Umbra(args.amqp_url, args.chrome_exe, max_active_browsers=int(args.max_browsers), exchange_name=args.amqp_exchange, queue_name=args.amqp_queue) def browserdump_str(pp, b): x = [] x.append(pp.pformat(b.__dict__)) if b._chrome_instance: x.append("=> {} chrome instance:".format(b)) x.append(pp.pformat(b._chrome_instance.__dict__)) if b._behavior: x.append("=> {} active behavior:".format(b)) x.append(pp.pformat(b._behavior.__dict__)) return "\n".join(x) def dump_state(signum, frame): pp = pprint.PrettyPrinter(indent=4) state_strs = [] for th in threading.enumerate(): state_strs.append(str(th)) stack = traceback.format_stack(sys._current_frames()[th.ident]) state_strs.append("".join(stack)) state_strs.append("umbra controller:") state_strs.append(pp.pformat(controller.__dict__)) state_strs.append("") for b in controller._browser_pool._in_use: state_strs.append("{} (in use):".format(b)) state_strs.append(browserdump_str(pp, b)) state_strs.append("") for b in controller._browser_pool._available: state_strs.append("{} (not in use):".format(b)) state_strs.append(browserdump_str(pp, b)) state_strs.append("") logging.warn("dumping state (caught signal {})\n{}".format(signum, "\n".join(state_strs))) class ShutdownRequested(Exception): pass def sigterm(signum, frame): raise ShutdownRequested('shutdown requested (caught SIGTERM)') def sigint(signum, frame): raise ShutdownRequested('shutdown requested (caught SIGINT)') signal.signal(signal.SIGQUIT, dump_state) signal.signal(signal.SIGHUP, controller.reconnect) signal.signal(signal.SIGTERM, sigterm) signal.signal(signal.SIGINT, sigint) controller.start() try: while True: time.sleep(0.5) except ShutdownRequested as e: logging.info(e) except BaseException as e: logging.fatal(e, exc_info=sys.exc_info()) finally: try: controller.shutdown() for th in threading.enumerate(): if th != threading.current_thread(): th.join() except BaseException as e: logging.warn("caught exception {}".format(e)) for i in range(6,0,-1): controller.shutdown_now() try: for th in threading.enumerate(): if th != threading.current_thread(): th.join() break # if we get here, we're done, all threads finished except: logging.warn("caught exception {}".format(e)) logging.info("all finished, exiting")