Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Make youtube-dl optional in BrozzlerWorker.brozzle_page

Browse Source 
Enabled by default (of course).
Speed is significantly improved when disabled.
This commit is contained in:
Vangelis Banos 2017-07-31 08:57:47 +00:00
parent a563e9eb0c
commit 9c81a7bbda
1 changed files with 25 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
brozzler/worker.py
View File

@ -316,30 +316,32 @@ class BrozzlerWorker:
return full_jpeg, thumb_jpeg return full_jpeg, thumb_jpeg
def brozzle_page(self, browser, site, page, on_screenshot=None): def brozzle_page(self, browser, site, page, on_screenshot=None,
enable_youtube_dl=True):
self.logger.info("brozzling {}".format(page)) self.logger.info("brozzling {}".format(page))
try: if enable_youtube_dl:
with tempfile.TemporaryDirectory(prefix='brzl-ydl-') as tempdir: try:
ydl = self._youtube_dl(tempdir, site) with tempfile.TemporaryDirectory(prefix='brzl-ydl-') as tempdir:
ydl_spy = ydl.brozzler_spy # remember for later ydl = self._youtube_dl(tempdir, site)
self._try_youtube_dl(ydl, site, page) ydl_spy = ydl.brozzler_spy # remember for later
except brozzler.ReachedLimit as e: self._try_youtube_dl(ydl, site, page)
raise except brozzler.ReachedLimit as e:
except brozzler.ShutdownRequested: raise
raise except brozzler.ShutdownRequested:
except brozzler.ProxyError: raise
raise except brozzler.ProxyError:
except Exception as e: raise
if (hasattr(e, 'exc_info') and len(e.exc_info) >= 2 except Exception as e:
and hasattr(e.exc_info[1], 'code') if (hasattr(e, 'exc_info') and len(e.exc_info) >= 2
and e.exc_info[1].code == 430): and hasattr(e.exc_info[1], 'code')
self.logger.info( and e.exc_info[1].code == 430):
'youtube-dl got %s %s processing %s', self.logger.info(
e.exc_info[1].code, e.exc_info[1].msg, page.url) 'youtube-dl got %s %s processing %s',
else: e.exc_info[1].code, e.exc_info[1].msg, page.url)
self.logger.error( else:
'youtube_dl raised exception on %s', page, self.logger.error(
exc_info=True) 'youtube_dl raised exception on %s', page,
exc_info=True)
if self._needs_browsing(page, ydl_spy): if self._needs_browsing(page, ydl_spy):
self.logger.info('needs browsing: %s', page) self.logger.info('needs browsing: %s', page)