Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-09 15:02:28 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

check captured_youtube_watch_pages not None

Browse source
This commit is contained in:
Barbara Miller 2025-07-28 15:41:07 -07:00
parent 4a73b1e8d8
commit de56dc3d13
1 changed files with 25 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

20
brozzler/ydl.py
View file

@ -635,25 +635,33 @@ def do_youtube_dl(worker, site, page, ytdlp_proxy_endpoints):
site["metadata"]["ait_seed_id"], site["metadata"]["ait_seed_id"],
) )
try: try:
captured_youtube_watch_pages = set() captured_youtube_watch_pages = (
captured_youtube_watch_pages.update(
worker._video_data.get_video_captures(site, source="youtube") worker._video_data.get_video_captures(site, source="youtube")
) )
uncaptured_youtube_watch_pages = [] if captured_youtube_watch_pages:
logger.info(
"found %s previously captured youtube watch pages for account %s, seed_id %s",
len(captured_youtube_watch_pages),
site["metadata"]["ait_account_id"],
site["metadata"]["ait_seed_id"],
)
captured_watch_pages = set()
captured_watch_pages.update(captured_youtube_watch_pages)
uncaptured_watch_pages = []
for e in ie_result.get("entries_no_dl", []): for e in ie_result.get("entries_no_dl", []):
# note: http needed for match # note: http matches, not https
youtube_watch_url = str( youtube_watch_url = str(
urlcanon.aggressive( urlcanon.aggressive(
f"http://www.youtube.com/watch?v={e['id']}" f"http://www.youtube.com/watch?v={e['id']}"
) )
) )
if youtube_watch_url in captured_youtube_watch_pages: if youtube_watch_url in captured_watch_pages:
logger.info( logger.info(
"skipping adding %s to yt-dlp outlinks", "skipping adding %s to yt-dlp outlinks",
youtube_watch_url, youtube_watch_url,
) )
continue continue
uncaptured_youtube_watch_pages.append( uncaptured_watch_pages.append(
f"https://www.youtube.com/watch?v={e['id']}" f"https://www.youtube.com/watch?v={e['id']}"
) )
except Exception as e: except Exception as e: