Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-08 06:22:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into qa

Browse source 
* master:
  hopefully fixes lingering ydl concurrency issue
This commit is contained in:
Noah Levitt 2018-10-11 13:43:06 -07:00
commit 16c56fed5a
2 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

6
brozzler/ydl.py
View file

@ -180,6 +180,9 @@ def _build_youtube_dl(worker, destdir, site):
}) })
def process_info(self, info_dict): def process_info(self, info_dict):
# lock this section to prevent race condition between threads that
# want to monkey patch _finish_frag_download() at the same time
with global_ydl_lock:
_orig__finish_frag_download = youtube_dl.downloader.fragment.FragmentFD._finish_frag_download _orig__finish_frag_download = youtube_dl.downloader.fragment.FragmentFD._finish_frag_download
def _finish_frag_download(ffd_self, ctx): def _finish_frag_download(ffd_self, ctx):
@ -187,9 +190,6 @@ def _build_youtube_dl(worker, destdir, site):
if worker._using_warcprox(site): if worker._using_warcprox(site):
self._push_stitched_up_vid_to_warcprox(site, info_dict, ctx) self._push_stitched_up_vid_to_warcprox(site, info_dict, ctx)
# lock this section to prevent race condition between threads that
# want to monkey patch _finish_frag_download() at the same time
with global_ydl_lock:
try: try:
youtube_dl.downloader.fragment.FragmentFD._finish_frag_download = _finish_frag_download youtube_dl.downloader.fragment.FragmentFD._finish_frag_download = _finish_frag_download
return super().process_info(info_dict) return super().process_info(info_dict)

2
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.5.dev307', version='1.5.dev308',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',