Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

fixes for set, passed parameter

Browse Source
This commit is contained in:
Barbara Miller 2024-05-30 10:13:05 -07:00
parent 6fc409f554
commit 2e4a19e838
3 changed files with 14 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
brozzler/cli.py
View File

@ -545,10 +545,12 @@ def brozzler_worker(argv=None):
signal.signal(signal.SIGQUIT, dump_state)
def get_skip_av_seeds():
# TODO: develop UI and refactor
SKIP_AV_SEEDS_FILE = "/opt/local/brozzler/skip_av_seeds.txt"
try:
with open(skip_av_seeds_file) as skips:
skip_av_seeds = set(skips.readlines())
# make set from seed IDs in SKIP_AV_SEEDS_FILE
with open(SKIP_AV_SEEDS_FILE) as skips:
skip_av_seeds = {int(l) for l in skips.readlines()}
logging.info("running with skip_av_seeds file %s" % SKIP_AV_SEEDS_FILE)
except Exception as e:
skip_av_seeds = set()
@ -562,7 +564,7 @@ def brozzler_worker(argv=None):
worker = brozzler.worker.BrozzlerWorker(
frontier,
service_registry,
skip_av_seeds,
skip_av_seeds=skip_av_seeds,
max_browsers=int(args.max_browsers),
chrome_exe=args.chrome_exe,
proxy=args.proxy,

2
brozzler/worker.py
View File

@ -262,7 +262,7 @@ class BrozzlerWorker:
except brozzler.PageInterstitialShown:
self.logger.info("page interstitial shown (http auth): %s", page)
if enable_youtube_dl and ydl.should_ytdlp(self, site, page):
if enable_youtube_dl and ydl.should_ytdlp(site, page, self.skip_av_seeds):
try:
ydl_outlinks = ydl.do_youtube_dl(self, site, page)
outlinks.update(ydl_outlinks)

15
brozzler/ydl.py
View File

@ -32,7 +32,7 @@ import threading
thread_local = threading.local()
def should_ytdlp(worker, site, page):
def should_ytdlp(site, page, skip_av_seeds):
# called only after we've passed needs_browsing() check
if page.status_code != 200:
logging.info("skipping ytdlp: non-200 page status")
@ -41,22 +41,23 @@ def should_ytdlp(worker, site, page):
logging.info("skipping ytdlp: site marked skip_ytdlp")
return False
ytdlp_url = page.redirect_url if page.redirect_url else page.url
if "chrome-error:" in ytdlp_url:
return False
ytdlp_seed = (
site["metadata"]["ait_seed_id"]
if "metadata" in site and "ait_seed_id" in site["metadata"]
else None
)
if ytdlp_seed and ytdlp_seed in worker.skip_av_seeds:
# TODO: develop UI and refactor
if ytdlp_seed and ytdlp_seed in skip_av_seeds:
logging.info("skipping ytdlp: site in skip_av_seeds")
site.skip_ytdlp = True
return False
ytdlp_url = page.redirect_url if page.redirect_url else page.url
if "chrome-error:" in ytdlp_url:
return False
return True