Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'ytdlp_tmpdir' into qa

Browse Source
This commit is contained in:
Barbara Miller 2024-12-06 18:50:01 -08:00
commit db56c8fb00
3 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
brozzler/worker.py
View File

@ -444,7 +444,7 @@ class BrozzlerWorker:
self.logger.trace("%r", chrome_msg) self.logger.trace("%r", chrome_msg)
if chrome_msg.get("params", {}).get("versions"): if chrome_msg.get("params", {}).get("versions"):
url = chrome_msg.get("params", {}).get("versions")[0].get("scriptURL") url = chrome_msg.get("params", {}).get("versions")[0].get("scriptURL")
if url and url.startswith('http') and url not in sw_fetched: if url and url.startswith("http") and url not in sw_fetched:
self.logger.info("fetching service worker script %s", url) self.logger.info("fetching service worker script %s", url)
self._fetch_url(site, url=url) self._fetch_url(site, url=url)
sw_fetched.add(url) sw_fetched.add(url)

2
brozzler/ydl.py
View File

@ -428,7 +428,7 @@ def do_youtube_dl(worker, site, page):
Returns: Returns:
`list` of `str`: outlink urls `list` of `str`: outlink urls
""" """
with tempfile.TemporaryDirectory(prefix="brzl-ydl-") as tempdir: with tempfile.TemporaryDirectory(prefix="brzl-ydl-", dir="/tmp") as tempdir:
ydl = _build_youtube_dl(worker, tempdir, site, page) ydl = _build_youtube_dl(worker, tempdir, site, page)
ie_result = _try_youtube_dl(worker, ydl, site, page) ie_result = _try_youtube_dl(worker, ydl, site, page)
outlinks = set() outlinks = set()

2
setup.py
View File

@ -34,7 +34,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name="brozzler", name="brozzler",
version="1.6.4qa", version="1.6.5qa",
description="Distributed web crawling with browsers", description="Distributed web crawling with browsers",
url="https://github.com/internetarchive/brozzler", url="https://github.com/internetarchive/brozzler",
author="Noah Levitt", author="Noah Levitt",