Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-19 12:24:20 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

bump up heartbeat interval (see comment)

Browse source
This commit is contained in:
Noah Levitt 2018-07-03 18:35:08 -05:00
parent 783fd0ea87
commit 9d18dc6aeb
2 changed files with 6 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
brozzler/worker.py
View file

@ -113,7 +113,11 @@ class YoutubeDLSpy(urllib.request.BaseHandler):
class BrozzlerWorker: class BrozzlerWorker:
logger = logging.getLogger(__module__ + "." + __qualname__) logger = logging.getLogger(__module__ + "." + __qualname__)
HEARTBEAT_INTERVAL = 20.0 # 3⅓ min heartbeat interval => 10 min ttl
# This is kind of a long time, because `frontier.claim_sites()`, which runs
# in the same thread as the heartbeats, can take a while on a busy brozzler
# cluster with slow rethinkdb.
HEARTBEAT_INTERVAL = 200.0
SITE_SESSION_MINUTES = 15 SITE_SESSION_MINUTES = 15
def __init__( def __init__(

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.4.dev294', version='1.4.dev295',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',