Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-25 00:59:52 -05:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

YTDLPStatus enum

Browse Source
This commit is contained in:
Barbara Miller 2024-05-30 14:52:53 -07:00
parent 9e64d2b7e8
commit e562bc0ebe
2 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
brozzler/model.py
View File

@ -34,6 +34,7 @@ import urllib
import uuid import uuid
import yaml import yaml
import zlib import zlib
from enum import Enum
from typing import Optional from typing import Optional
@ -191,6 +192,12 @@ class ElapsedMixIn(object):
return dt return dt
class YTDLPStatus(Enum):
UNKNOWN = 0
SKIP = 1
CAPTURE = 2
class Job(doublethink.Document, ElapsedMixIn): class Job(doublethink.Document, ElapsedMixIn):
logger = logging.getLogger(__module__ + "." + __qualname__) logger = logging.getLogger(__module__ + "." + __qualname__)
table = "jobs" table = "jobs"
@ -236,7 +243,7 @@ class Site(doublethink.Document, ElapsedMixIn):
if not "scope" in self: if not "scope" in self:
self.scope = {} self.scope = {}
if not "skip_ytdlp" in self: if not "skip_ytdlp" in self:
self.skip_ytdlp = False self.skip_ytdlp = YTDLPStatus.UNKNOWN
# backward compatibility # backward compatibility
if "surt" in self.scope: if "surt" in self.scope:

10
brozzler/ydl.py
View File

@ -34,10 +34,11 @@ thread_local = threading.local()
def should_ytdlp(site, page, skip_av_seeds): def should_ytdlp(site, page, skip_av_seeds):
# called only after we've passed needs_browsing() check # called only after we've passed needs_browsing() check
from .model import YTDLPStatus
if page.status_code != 200: if page.status_code != 200:
logging.info("skipping ytdlp: non-200 page status") logging.info("skipping ytdlp: non-200 page status")
return False return False
if site.skip_ytdlp: if site.skip_ytdlp == "SKIP":
logging.info("skipping ytdlp: site marked skip_ytdlp") logging.info("skipping ytdlp: site marked skip_ytdlp")
return False return False
@ -53,10 +54,13 @@ def should_ytdlp(site, page, skip_av_seeds):
) )
# TODO: develop UI and refactor # TODO: develop UI and refactor
if ytdlp_seed and ytdlp_seed in skip_av_seeds: if ytdlp_seed
if site.skip_ytdlp == "UNKNOWN" and ytdlp_seed in skip_av_seeds:
logging.info("skipping ytdlp: site in skip_av_seeds") logging.info("skipping ytdlp: site in skip_av_seeds")
site.skip_ytdlp = True site.skip_ytdlp = YTDLPStatus.SKIP
return False return False
else:
site.skip_ytdlp = YTDLPStatus.CAPTURE
return True return True