mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-24 08:39:59 -05:00
72 lines
2.5 KiB
Python
Executable File
72 lines
2.5 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# vim: set sw=4 et:
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import logging
|
|
import brozzler
|
|
import threading
|
|
import time
|
|
import signal
|
|
import pprint
|
|
import traceback
|
|
import brozzler.worker
|
|
|
|
arg_parser = argparse.ArgumentParser(prog=os.path.basename(__file__),
|
|
description='crawl-url - browse urls, follow links',
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
arg_parser.add_argument('-u', '--url', dest='amqp_url', default='amqp://guest:guest@localhost:5672/%2f',
|
|
help='URL identifying the amqp server to talk to')
|
|
arg_parser.add_argument('-e', '--executable', dest='chrome_exe', default='chromium-browser',
|
|
help='executable to use to invoke chrome')
|
|
arg_parser.add_argument('-n', '--max-browsers', dest='max_browsers', default='1',
|
|
help='max number of chrome instances simultaneously browsing pages')
|
|
arg_parser.add_argument('-v', '--verbose', dest='log_level',
|
|
action="store_const", default=logging.INFO, const=logging.DEBUG)
|
|
arg_parser.add_argument('--version', action='version',
|
|
version="brozzler {} - {}".format(brozzler.version, os.path.basename(__file__)))
|
|
args = arg_parser.parse_args(args=sys.argv[1:])
|
|
|
|
logging.basicConfig(stream=sys.stdout, level=args.log_level,
|
|
format='%(asctime)s %(process)d %(levelname)s %(threadName)s %(name)s.%(funcName)s(%(filename)s:%(lineno)d) %(message)s')
|
|
# the way we're using it, amqp is too verbose at debug level
|
|
logging.getLogger("amqp").setLevel(logging.INFO)
|
|
|
|
def sigterm(signum, frame):
|
|
raise brozzler.ShutdownRequested('shutdown requested (caught SIGTERM)')
|
|
def sigint(signum, frame):
|
|
raise brozzler.ShutdownRequested('shutdown requested (caught SIGINT)')
|
|
|
|
def dump_state(signum, frame):
|
|
pp = pprint.PrettyPrinter(indent=4)
|
|
state_strs = []
|
|
|
|
for th in threading.enumerate():
|
|
state_strs.append(str(th))
|
|
stack = traceback.format_stack(sys._current_frames()[th.ident])
|
|
state_strs.append("".join(stack))
|
|
|
|
logging.warn("dumping state (caught signal {})\n{}".format(signum, "\n".join(state_strs)))
|
|
|
|
signal.signal(signal.SIGQUIT, dump_state)
|
|
signal.signal(signal.SIGTERM, sigterm)
|
|
signal.signal(signal.SIGINT, sigint)
|
|
|
|
worker = brozzler.worker.BrozzlerWorker(amqp_url=args.amqp_url,
|
|
max_browsers=int(args.max_browsers), chrome_exe=args.chrome_exe)
|
|
|
|
worker.start()
|
|
|
|
try:
|
|
while True:
|
|
time.sleep(0.5)
|
|
except brozzler.ShutdownRequested as e:
|
|
worker.shutdown_now()
|
|
|
|
for th in threading.enumerate():
|
|
if th != threading.current_thread():
|
|
th.join()
|
|
|
|
logging.info("all done, exiting")
|