Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

make sure youtube-dl progress thing can't derail youtube-dl operation

Browse Source
This commit is contained in:
Noah Levitt 2017-06-26 16:10:40 -07:00
parent caee2787b0
commit 8ef7972ace
2 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
brozzler/worker.py
View File

@ -162,12 +162,17 @@ class BrozzlerWorker:
def ydl_progress(*args, **kwargs): def ydl_progress(*args, **kwargs):
# in case youtube-dl takes a long time, heartbeat site.last_claimed # in case youtube-dl takes a long time, heartbeat site.last_claimed
# to prevent another brozzler-worker from claiming the site # to prevent another brozzler-worker from claiming the site
try:
if site.rr and doublethink.utcnow() - site.last_claimed > datetime.timedelta(minutes=7): if site.rr and doublethink.utcnow() - site.last_claimed > datetime.timedelta(minutes=7):
self.logger.debug( self.logger.debug(
'heartbeating site.last_claimed to prevent another ' 'heartbeating site.last_claimed to prevent another '
'brozzler-worker claiming this site id=%r', site.id) 'brozzler-worker claiming this site id=%r', site.id)
site.last_claimed = doublethink.utcnow() site.last_claimed = doublethink.utcnow()
site.save() site.save()
except:
self.logger.debug(
'problem heartbeating site.last_claimed site id=%r',
site.id, exc_info=True)
ydl_opts = { ydl_opts = {
"outtmpl": "{}/ydl%(autonumber)s.out".format(destdir), "outtmpl": "{}/ydl%(autonumber)s.out".format(destdir),

2
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b12.dev261', version='1.1b12.dev262',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',