Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-09-23 06:04:47 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'predup_type_playlist' into qa

Browse source
This commit is contained in:
Barbara Miller 2025-08-27 13:56:16 -07:00
commit fcf966fca4
1 changed files with 20 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

37
brozzler/worker.py
View file

@ -98,6 +98,8 @@ class BrozzlerWorker:
from brozzler.video_data import VideoDataClient
self._video_data = VideoDataClient()
else:
self._video_data = None
self._warcprox_auto = warcprox_auto
self._proxy = proxy
@ -301,24 +303,25 @@ class BrozzlerWorker:
return False
# predup...
logger.info("checking for recent previous captures of %s", ytdlp_url)
recent = 90 if "youtube.com/watch" in ytdlp_url else 30
try:
recent_capture_exists = self._video_data.recent_video_capture_exists(
site, ytdlp_url, recent
)
if recent_capture_exists:
logger.info(
"recent capture of %s found, skipping ytdlp",
ytdlp_url,
if self._video_data:
logger.info("checking for recent previous captures of %s", ytdlp_url)
recent = 90 if "youtube.com/watch" in ytdlp_url else 30
try:
recent_capture_exists = self._video_data.recent_video_capture_exists(
site, ytdlp_url, recent
)
if recent_capture_exists:
logger.info(
"recent capture of %s found, skipping ytdlp",
ytdlp_url,
)
return False
except Exception as e:
logger.warning(
"exception querying for previous capture for %s: %s",
ytdlp_url,
str(e),
)
return False
except Exception as e:
logger.warning(
"exception querying for previous capture for %s: %s",
ytdlp_url,
str(e),
)
return True