Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-06-19 04:19:30 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

calm logging, don't print stacktrace on 430 from youtube-dl

Browse source
This commit is contained in:
Noah Levitt 2016-06-28 23:19:28 +00:00
parent e9c398caea
commit 7a805a43d1
2 changed files with 10 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

12
brozzler/worker.py
View file

@ -223,9 +223,15 @@ class BrozzlerWorker:
self._try_youtube_dl(ydl, site, page) self._try_youtube_dl(ydl, site, page)
except brozzler.ReachedLimit as e: except brozzler.ReachedLimit as e:
raise raise
except: except Exception as e:
self.logger.error("youtube_dl raised exception on %s", if (hasattr(e, 'exc_info') and len(e.exc_info) >= 2
page, exc_info=True) and e.exc_info[1].code == 430):
self.logger.info(
'youtube-dl got %s %s processing %s',
e.exc_info[1].code, e.exc_info[1].msg, page.url)
else:
self.logger.error(
"youtube_dl raised exception on %s", page, exc_info=True)
if self._needs_browsing(page, ydl_spy): if self._needs_browsing(page, ydl_spy):
self.logger.info('needs browsing: %s', page) self.logger.info('needs browsing: %s', page)

2
setup.py
View file

@ -21,7 +21,7 @@ import setuptools
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1.dev29', version='1.1.dev30',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',