Overhaul extractors.cr to use modules

This commit is contained in:
syeopite 2021-08-03 00:22:31 -07:00
parent 3dea670091
commit 142317c2be
No known key found for this signature in database
GPG Key ID: 6FA616E5A5294A82

View File

@ -3,47 +3,34 @@
# Tuple of Parsers/Extractors so we can easily cycle through them. # Tuple of Parsers/Extractors so we can easily cycle through them.
private ITEM_CONTAINER_EXTRACTOR = { private ITEM_CONTAINER_EXTRACTOR = {
YoutubeTabsExtractor.new, Extractors::YouTubeTabs,
SearchResultsExtractor.new, Extractors::SearchResults,
ContinuationExtractor.new, Extractors::Continuation,
} }
private ITEM_PARSERS = { private ITEM_PARSERS = {
VideoParser.new, Parsers::VideoRendererParser,
ChannelParser.new, Parsers::ChannelRendererParser,
GridPlaylistParser.new, Parsers::GridPlaylistRendererParser,
PlaylistParser.new, Parsers::PlaylistRendererParser,
CategoryParser.new, Parsers::CategoryRendererParser,
} }
private struct AuthorFallback record AuthorFallback, name : String? = nil, id : String? = nil
property name, id
def initialize(@name : String? = nil, @id : String? = nil)
end
end
# The following are the parsers for parsing raw item data into neatly packaged structs. # The following are the parsers for parsing raw item data into neatly packaged structs.
# They're accessed through the process() method which validates the given data as applicable # They're accessed through the process() method which validates the given data as applicable
# to their specific struct and then use the internal parse() method to assemble the struct # to their specific struct and then use the internal parse() method to assemble the struct
# specific to their category. # specific to their category.
private abstract struct ItemParser private module Parsers
# Base type for all item parsers. module VideoRendererParser
def process(item : JSON::Any, author_fallback : AuthorFallback) def self.process(item : JSON::Any, author_fallback : AuthorFallback)
end
private def parse(item_contents : JSON::Any, author_fallback : AuthorFallback)
end
end
private struct VideoParser < ItemParser
def process(item, author_fallback)
if item_contents = (item["videoRenderer"]? || item["gridVideoRenderer"]?) if item_contents = (item["videoRenderer"]? || item["gridVideoRenderer"]?)
return self.parse(item_contents, author_fallback) return self.parse(item_contents, author_fallback)
end end
end end
private def parse(item_contents, author_fallback) private def self.parse(item_contents, author_fallback)
video_id = item_contents["videoId"].as_s video_id = item_contents["videoId"].as_s
title = item_contents["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || "" title = item_contents["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || ""
@ -94,14 +81,14 @@ private struct VideoParser < ItemParser
end end
end end
private struct ChannelParser < ItemParser module ChannelRendererParser
def process(item, author_fallback) def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?) if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?)
return self.parse(item_contents, author_fallback) return self.parse(item_contents, author_fallback)
end end
end end
private def parse(item_contents, author_fallback) private def self.parse(item_contents, author_fallback)
author = item_contents["title"]["simpleText"]?.try &.as_s || author_fallback.name || "" author = item_contents["title"]["simpleText"]?.try &.as_s || author_fallback.name || ""
author_id = item_contents["channelId"]?.try &.as_s || author_fallback.id || "" author_id = item_contents["channelId"]?.try &.as_s || author_fallback.id || ""
@ -125,14 +112,14 @@ private struct ChannelParser < ItemParser
end end
end end
private struct GridPlaylistParser < ItemParser module GridPlaylistRendererParser
def process(item, author_fallback) def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["gridPlaylistRenderer"]? if item_contents = item["gridPlaylistRenderer"]?
return self.parse(item_contents, author_fallback) return self.parse(item_contents, author_fallback)
end end
end end
private def parse(item_contents, author_fallback) private def self.parse(item_contents, author_fallback)
title = item_contents["title"]["runs"].as_a[0]?.try &.["text"].as_s || "" title = item_contents["title"]["runs"].as_a[0]?.try &.["text"].as_s || ""
plid = item_contents["playlistId"]?.try &.as_s || "" plid = item_contents["playlistId"]?.try &.as_s || ""
@ -151,14 +138,14 @@ private struct GridPlaylistParser < ItemParser
end end
end end
private struct PlaylistParser < ItemParser module PlaylistRendererParser
def process(item, author_fallback) def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["playlistRenderer"]? if item_contents = item["playlistRenderer"]?
return self.parse(item_contents, author_fallback) return self.parse(item_contents, author_fallback)
end end
end end
def parse(item_contents, author_fallback) private def self.parse(item_contents, author_fallback)
title = item_contents["title"]["simpleText"]?.try &.as_s || "" title = item_contents["title"]["simpleText"]?.try &.as_s || ""
plid = item_contents["playlistId"]?.try &.as_s || "" plid = item_contents["playlistId"]?.try &.as_s || ""
@ -195,14 +182,14 @@ private struct PlaylistParser < ItemParser
end end
end end
private struct CategoryParser < ItemParser module CategoryRendererParser
def process(item, author_fallback) def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["shelfRenderer"]? if item_contents = item["shelfRenderer"]?
return self.parse(item_contents, author_fallback) return self.parse(item_contents, author_fallback)
end end
end end
def parse(item_contents, author_fallback) private def self.parse(item_contents, author_fallback)
# Title extraction is a bit complicated. There are two possible routes for it # Title extraction is a bit complicated. There are two possible routes for it
# as well as times when the title attribute just isn't sent by YT. # as well as times when the title attribute just isn't sent by YT.
title_container = item_contents["title"]? || "" title_container = item_contents["title"]? || ""
@ -256,28 +243,22 @@ private struct CategoryParser < ItemParser
}) })
end end
end end
end
# The following are the extractors for extracting an array of items from # The following are the extractors for extracting an array of items from
# the internal Youtube API's JSON response. The result is then packaged into # the internal Youtube API's JSON response. The result is then packaged into
# a structure we can more easily use via the parsers above. Their internals are # a structure we can more easily use via the parsers above. Their internals are
# identical to the item parsers. # identical to the item parsers.
private abstract struct ItemsContainerExtractor private module Extractors
def process(item : Hash(String, JSON::Any)) module YouTubeTabs
end def self.process(initial_data : Hash(String, JSON::Any))
private def extract(target : JSON::Any)
end
end
private struct YoutubeTabsExtractor < ItemsContainerExtractor
def process(initial_data)
if target = initial_data["twoColumnBrowseResultsRenderer"]? if target = initial_data["twoColumnBrowseResultsRenderer"]?
self.extract(target) self.extract(target)
end end
end end
private def extract(target) private def self.extract(target)
raw_items = [] of JSON::Any raw_items = [] of JSON::Any
selected_tab = extract_selected_tab(target["tabs"]) selected_tab = extract_selected_tab(target["tabs"])
content = selected_tab["content"] content = selected_tab["content"]
@ -304,14 +285,14 @@ private struct YoutubeTabsExtractor < ItemsContainerExtractor
end end
end end
private struct SearchResultsExtractor < ItemsContainerExtractor module SearchResults
def process(initial_data) def self.process(initial_data : Hash(String, JSON::Any))
if target = initial_data["twoColumnSearchResultsRenderer"]? if target = initial_data["twoColumnSearchResultsRenderer"]?
self.extract(target) self.extract(target)
end end
end end
private def extract(target) private def self.extract(target)
raw_items = [] of Array(JSON::Any) raw_items = [] of Array(JSON::Any)
content = target["primaryContents"] content = target["primaryContents"]
renderer = content["sectionListRenderer"]["contents"].as_a.each do |node| renderer = content["sectionListRenderer"]["contents"].as_a.each do |node|
@ -326,8 +307,8 @@ private struct SearchResultsExtractor < ItemsContainerExtractor
end end
end end
private struct ContinuationExtractor < ItemsContainerExtractor module Continuation
def process(initial_data) def self.process(initial_data : Hash(String, JSON::Any))
if target = initial_data["continuationContents"]? if target = initial_data["continuationContents"]?
self.extract(target) self.extract(target)
elsif target = initial_data["appendContinuationItemsAction"]? elsif target = initial_data["appendContinuationItemsAction"]?
@ -335,7 +316,7 @@ private struct ContinuationExtractor < ItemsContainerExtractor
end end
end end
private def extract(target) private def self.extract(target)
raw_items = [] of JSON::Any raw_items = [] of JSON::Any
if content = target["gridContinuation"]? if content = target["gridContinuation"]?
raw_items = content["items"].as_a raw_items = content["items"].as_a
@ -346,6 +327,7 @@ private struct ContinuationExtractor < ItemsContainerExtractor
return raw_items return raw_items
end end
end end
end
# Parses an item from Youtube's JSON response into a more usable structure. # Parses an item from Youtube's JSON response into a more usable structure.
# The end result can either be a SearchVideo, SearchPlaylist or SearchChannel. # The end result can either be a SearchVideo, SearchPlaylist or SearchChannel.