mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-08-07 22:12:15 -04:00
restore BrozzlerWorker built-in support for managing its own thread
This commit is contained in:
parent
70b67942a5
commit
c2704b18be
2 changed files with 29 additions and 1 deletions
|
@ -109,6 +109,9 @@ class BrozzlerWorker:
|
||||||
self._browsing_threads = set()
|
self._browsing_threads = set()
|
||||||
self._browsing_threads_lock = threading.Lock()
|
self._browsing_threads_lock = threading.Lock()
|
||||||
|
|
||||||
|
self._thread = None
|
||||||
|
self._start_stop_lock = threading.Lock()
|
||||||
|
|
||||||
def _proxy(self, site):
|
def _proxy(self, site):
|
||||||
if site.proxy:
|
if site.proxy:
|
||||||
return site.proxy
|
return site.proxy
|
||||||
|
@ -458,3 +461,28 @@ class BrozzlerWorker:
|
||||||
for th in thredz:
|
for th in thredz:
|
||||||
th.join()
|
th.join()
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
with self._start_stop_lock:
|
||||||
|
if self._thread:
|
||||||
|
self.logger.warn(
|
||||||
|
'ignoring start request because self._thread is '
|
||||||
|
'not None')
|
||||||
|
return
|
||||||
|
self._thread = threading.Thread(
|
||||||
|
target=self.run, name="BrozzlerWorker")
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
def shutdown_now(self):
|
||||||
|
self.stop()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
with self._start_stop_lock:
|
||||||
|
if self._thread and self._thread.is_alive():
|
||||||
|
self.logger.info("brozzler worker shutting down")
|
||||||
|
brozzler.thread_raise(self._thread, brozzler.ShutdownRequested)
|
||||||
|
self._thread.join()
|
||||||
|
self._thread = None
|
||||||
|
|
||||||
|
def is_alive(self):
|
||||||
|
return self._thread and self._thread.is_alive()
|
||||||
|
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -32,7 +32,7 @@ def find_package_data(package):
|
||||||
|
|
||||||
setuptools.setup(
|
setuptools.setup(
|
||||||
name='brozzler',
|
name='brozzler',
|
||||||
version='1.1b9.dev156',
|
version='1.1b9.dev157',
|
||||||
description='Distributed web crawling with browsers',
|
description='Distributed web crawling with browsers',
|
||||||
url='https://github.com/internetarchive/brozzler',
|
url='https://github.com/internetarchive/brozzler',
|
||||||
author='Noah Levitt',
|
author='Noah Levitt',
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue