Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-09-24 22:48:33 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

make whole process die if main worker thread dies

Browse source
This commit is contained in:
Noah Levitt 2016-03-16 23:35:33 +00:00
parent ca9e62f5cf
commit c2e80ed6ff
2 changed files with 11 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

15
bin/brozzler-worker
View file

@ -52,7 +52,8 @@ def dump_state(signum, frame):
stack = traceback.format_stack(sys._current_frames()[th.ident]) stack = traceback.format_stack(sys._current_frames()[th.ident])
state_strs.append("".join(stack)) state_strs.append("".join(stack))
logging.warn("dumping state (caught signal {})\n{}".format(signum, "\n".join(state_strs))) logging.warn("dumping state (caught signal {})\n{}".format(
signum, "\n".join(state_strs)))
signal.signal(signal.SIGQUIT, dump_state) signal.signal(signal.SIGQUIT, dump_state)
signal.signal(signal.SIGTERM, sigterm) signal.signal(signal.SIGTERM, sigterm)
@ -61,16 +62,20 @@ signal.signal(signal.SIGINT, sigint)
r = rethinkstuff.Rethinker(args.rethinkdb_servers.split(","), args.rethinkdb_db) r = rethinkstuff.Rethinker(args.rethinkdb_servers.split(","), args.rethinkdb_db)
frontier = brozzler.RethinkDbFrontier(r) frontier = brozzler.RethinkDbFrontier(r)
service_registry = rethinkstuff.ServiceRegistry(r) service_registry = rethinkstuff.ServiceRegistry(r)
worker = brozzler.worker.BrozzlerWorker(frontier, service_registry, max_browsers=int(args.max_browsers), chrome_exe=args.chrome_exe) worker = brozzler.worker.BrozzlerWorker(
frontier, service_registry, max_browsers=int(args.max_browsers),
chrome_exe=args.chrome_exe)
worker.start() worker_thread = worker.start()
try: try:
while True: while worker_thread.is_alive():
time.sleep(0.5) time.sleep(0.5)
logging.critical("worker thread has died, shutting down")
except brozzler.ShutdownRequested as e: except brozzler.ShutdownRequested as e:
pass
finally:
worker.shutdown_now() worker.shutdown_now()
for th in threading.enumerate(): for th in threading.enumerate():
if th != threading.current_thread(): if th != threading.current_thread():
th.join() th.join()

1
brozzler/worker.py
View file

@ -221,6 +221,7 @@ class BrozzlerWorker:
def start(self): def start(self):
th = threading.Thread(target=self.run, name="BrozzlerWorker") th = threading.Thread(target=self.run, name="BrozzlerWorker")
th.start() th.start()
return th
def shutdown_now(self): def shutdown_now(self):
self.logger.info("brozzler worker shutting down") self.logger.info("brozzler worker shutting down")