mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-08-02 11:36:15 -04:00
make whole process die if main worker thread dies
This commit is contained in:
parent
ca9e62f5cf
commit
c2e80ed6ff
2 changed files with 11 additions and 5 deletions
|
@ -52,7 +52,8 @@ def dump_state(signum, frame):
|
|||
stack = traceback.format_stack(sys._current_frames()[th.ident])
|
||||
state_strs.append("".join(stack))
|
||||
|
||||
logging.warn("dumping state (caught signal {})\n{}".format(signum, "\n".join(state_strs)))
|
||||
logging.warn("dumping state (caught signal {})\n{}".format(
|
||||
signum, "\n".join(state_strs)))
|
||||
|
||||
signal.signal(signal.SIGQUIT, dump_state)
|
||||
signal.signal(signal.SIGTERM, sigterm)
|
||||
|
@ -61,16 +62,20 @@ signal.signal(signal.SIGINT, sigint)
|
|||
r = rethinkstuff.Rethinker(args.rethinkdb_servers.split(","), args.rethinkdb_db)
|
||||
frontier = brozzler.RethinkDbFrontier(r)
|
||||
service_registry = rethinkstuff.ServiceRegistry(r)
|
||||
worker = brozzler.worker.BrozzlerWorker(frontier, service_registry, max_browsers=int(args.max_browsers), chrome_exe=args.chrome_exe)
|
||||
worker = brozzler.worker.BrozzlerWorker(
|
||||
frontier, service_registry, max_browsers=int(args.max_browsers),
|
||||
chrome_exe=args.chrome_exe)
|
||||
|
||||
worker.start()
|
||||
worker_thread = worker.start()
|
||||
|
||||
try:
|
||||
while True:
|
||||
while worker_thread.is_alive():
|
||||
time.sleep(0.5)
|
||||
logging.critical("worker thread has died, shutting down")
|
||||
except brozzler.ShutdownRequested as e:
|
||||
pass
|
||||
finally:
|
||||
worker.shutdown_now()
|
||||
|
||||
for th in threading.enumerate():
|
||||
if th != threading.current_thread():
|
||||
th.join()
|
||||
|
|
|
@ -221,6 +221,7 @@ class BrozzlerWorker:
|
|||
def start(self):
|
||||
th = threading.Thread(target=self.run, name="BrozzlerWorker")
|
||||
th.start()
|
||||
return th
|
||||
|
||||
def shutdown_now(self):
|
||||
self.logger.info("brozzler worker shutting down")
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue