Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'rotary2024' into qa

Browse Source
This commit is contained in:
Barbara Miller 2024-04-19 14:02:26 -07:00
commit 0485d9c83a
1 changed files with 22 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
brozzler/ydl.py
View File

@ -25,12 +25,13 @@ import tempfile
import urlcanon
import os
import json
import doublethink
import datetime
from cassandra import ReadTimeout
from cassandra.cluster import Cluster
import threading
import traceback
import doublethink
thread_local = threading.local()
@ -230,23 +231,26 @@ def _build_youtube_dl(worker, destdir, site, page):
# transfer, which warcprox currently rejects
extra_headers = dict(site.extra_headers())
extra_headers["content-length"] = size
request, response = worker._warcprox_write_record(
warcprox_address=worker._proxy_for(site),
url=url,
warc_type="resource",
content_type=mimetype,
payload=f,
extra_headers=extra_headers,
)
# consulted by _remember_videos()
ydl.pushed_videos.append(
{
"url": url,
"response_code": response.code,
"content-type": mimetype,
"content-length": size,
}
)
try:
request, response = worker._warcprox_write_record(
warcprox_address=worker._proxy_for(site),
url=url,
warc_type="resource",
content_type=mimetype,
payload=f,
extra_headers=extra_headers,
)
# consulted by _remember_videos()
ydl.pushed_videos.append(
{
"url": url,
"response_code": response.code,
"content-type": mimetype,
"content-length": size,
}
)
except:
traceback.print_exc()
def maybe_heartbeat_site_last_claimed(*args, **kwargs):
# in case yt-dlp takes a long time, heartbeat site.last_claimed