Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-19 23:35:54 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

set amqp prefetch count to the number of urls we can handle at a time, i.e. max_active_browsers (with prefetch=1 umbra was only browsing one url at a time, after quickly burning through urls already on the queue when started)

Browse Source
This commit is contained in:
Noah Levitt 2014-07-02 10:30:51 -07:00
parent 6306c16698
commit ae838af25d
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
umbra/controller.py
View File

@ -50,6 +50,7 @@ class AmqpBrowserController:
self.amqp_url = amqp_url
self.queue_name = queue_name
self.exchange_name = exchange_name
self.max_active_browsers = max_active_browsers
self._browser_pool = BrowserPool(size=max_active_browsers, chrome_exe=chrome_exe)
@ -89,7 +90,7 @@ class AmqpBrowserController:
def _wait_for_and_browse_urls(self, conn, consumer, timeout):
start = time.time()
browser = None
consumer.qos(prefetch_count=1)
consumer.qos(prefetch_count=self.max_active_browsers)
while not self._consumer_stop.is_set() and time.time() - start < timeout and not self._reconnect_requested:
try: