Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-08 06:22:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

BrozzlerWorkerThread separate from MainThread to avoid SIGTERM/SIGINT raising exception inside of some rethinkdb code or other sensitive code in that BrozzlerWorker.run() calls

Browse source
This commit is contained in:
Noah Levitt 2017-05-01 13:46:19 -07:00
parent 52433ade78
commit 389db01458
2 changed files with 7 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

16
brozzler/cli.py
View file

@ -304,11 +304,6 @@ def brozzler_worker(argv=None):
args = arg_parser.parse_args(args=argv[1:]) args = arg_parser.parse_args(args=argv[1:])
configure_logging(args) configure_logging(args)
def sigterm(signum, frame):
raise brozzler.ShutdownRequested('shutdown requested (caught SIGTERM)')
def sigint(signum, frame):
raise brozzler.ShutdownRequested('shutdown requested (caught SIGINT)')
def dump_state(signum, frame): def dump_state(signum, frame):
signal.signal(signal.SIGQUIT, signal.SIG_IGN) signal.signal(signal.SIGQUIT, signal.SIG_IGN)
try: try:
@ -330,10 +325,6 @@ def brozzler_worker(argv=None):
finally: finally:
signal.signal(signal.SIGQUIT, dump_state) signal.signal(signal.SIGQUIT, dump_state)
signal.signal(signal.SIGQUIT, dump_state)
signal.signal(signal.SIGTERM, sigterm)
signal.signal(signal.SIGINT, sigint)
rr = rethinker(args) rr = rethinker(args)
frontier = brozzler.RethinkDbFrontier(rr) frontier = brozzler.RethinkDbFrontier(rr)
service_registry = doublethink.ServiceRegistry(rr) service_registry = doublethink.ServiceRegistry(rr)
@ -342,8 +333,13 @@ def brozzler_worker(argv=None):
chrome_exe=args.chrome_exe, proxy=args.proxy, chrome_exe=args.chrome_exe, proxy=args.proxy,
warcprox_auto=args.warcprox_auto) warcprox_auto=args.warcprox_auto)
worker.run() signal.signal(signal.SIGQUIT, dump_state)
signal.signal(signal.SIGTERM, lambda s,f: worker.stop())
signal.signal(signal.SIGINT, lambda s,f: worker.stop())
th = threading.Thread(target=worker.run, name='BrozzlerWorkerThread')
th.start()
th.join()
logging.info('brozzler-worker is all done, exiting') logging.info('brozzler-worker is all done, exiting')
def brozzler_ensure_tables(argv=None): def brozzler_ensure_tables(argv=None):

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b11.dev241', version='1.1b11.dev242',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',