Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-05-31 12:34:35 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

make sure youtube-dl progress thing can't derail youtube-dl operation

Browse source
This commit is contained in:
Noah Levitt 2017-06-26 16:10:40 -07:00
parent caee2787b0
commit 8ef7972ace
2 changed files with 11 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

15
brozzler/worker.py
View file

@ -162,12 +162,17 @@ class BrozzlerWorker:
def ydl_progress(*args, **kwargs):
# in case youtube-dl takes a long time, heartbeat site.last_claimed
# to prevent another brozzler-worker from claiming the site
if site.rr and doublethink.utcnow() - site.last_claimed > datetime.timedelta(minutes=7):
try:
if site.rr and doublethink.utcnow() - site.last_claimed > datetime.timedelta(minutes=7):
self.logger.debug(
'heartbeating site.last_claimed to prevent another '
'brozzler-worker claiming this site id=%r', site.id)
site.last_claimed = doublethink.utcnow()
site.save()
except:
self.logger.debug(
'heartbeating site.last_claimed to prevent another '
'brozzler-worker claiming this site id=%r', site.id)
site.last_claimed = doublethink.utcnow()
site.save()
'problem heartbeating site.last_claimed site id=%r',
site.id, exc_info=True)
ydl_opts = {
"outtmpl": "{}/ydl%(autonumber)s.out".format(destdir),

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup(
name='brozzler',
version='1.1b12.dev261',
version='1.1b12.dev262',
description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler',
author='Noah Levitt',