Add objects to represent youtube item containers

This commit is contained in:
syeopite 2021-06-24 17:25:03 -07:00
parent 46fe3c5195
commit 9b6d09c7b6
No known key found for this signature in database
GPG Key ID: 6FA616E5A5294A82
2 changed files with 83 additions and 9 deletions

View File

@ -32,6 +32,10 @@ private class ItemParser
def process(item : JSON::Any, author_fallback : AuthorFallback) def process(item : JSON::Any, author_fallback : AuthorFallback)
end end
def process(item : Containers, author_fallback)
return self.process(item.contents)
end
private def parse(item_contents : JSON::Any, author_fallback : AuthorFallback) private def parse(item_contents : JSON::Any, author_fallback : AuthorFallback)
end end
end end
@ -282,6 +286,8 @@ private class YoutubeTabsExtractor < ItemsContainerExtractor
private def extract(target) private def extract(target)
raw_items = [] of JSON::Any raw_items = [] of JSON::Any
content_filters = [] of Tuple(String, String)
selected_tab = extract_selected_tab(target["tabs"]) selected_tab = extract_selected_tab(target["tabs"])
content = selected_tab["content"] content = selected_tab["content"]
@ -289,6 +295,14 @@ private class YoutubeTabsExtractor < ItemsContainerExtractor
renderer_container = renderer_container["itemSectionRenderer"] renderer_container = renderer_container["itemSectionRenderer"]
renderer_container_contents = renderer_container["contents"].as_a[0] renderer_container_contents = renderer_container["contents"].as_a[0]
submenu = renderer_container["subMenu"]?.try &.["channelSubMenuRenderer"]["contentTypeSubMenuItems"].as_a || nil
if submenu
submenu.each do |option|
content_filters << {option["title"].as_s, option["endpoint"]["browseEndpoint"]["params"].as_s}
end
end
# Category extraction # Category extraction
if items_container = renderer_container_contents["shelfRenderer"]? if items_container = renderer_container_contents["shelfRenderer"]?
raw_items << renderer_container_contents raw_items << renderer_container_contents
@ -306,7 +320,7 @@ private class YoutubeTabsExtractor < ItemsContainerExtractor
end end
end end
return raw_items return YoutubeTab.new({contents: raw_items, content_filters: content_filters})
end end
end end
@ -323,7 +337,7 @@ private class SearchResultsExtractor < ItemsContainerExtractor
renderer = content["sectionListRenderer"]["contents"].as_a[0]["itemSectionRenderer"] renderer = content["sectionListRenderer"]["contents"].as_a[0]["itemSectionRenderer"]
raw_items = renderer["contents"].as_a raw_items = renderer["contents"].as_a
return raw_items return SearchResults.new({contents: raw_items})
end end
end end
@ -344,7 +358,7 @@ private class ContinuationExtractor < ItemsContainerExtractor
raw_items = content.as_a raw_items = content.as_a
end end
return raw_items return ContinuationItems.new({contents: raw_items})
end end
end end
@ -366,6 +380,7 @@ def extract_item(item : JSON::Any, author_fallback : String? = nil,
# TODO radioRenderer, showRenderer, shelfRenderer, horizontalCardListRenderer, searchPyvRenderer # TODO radioRenderer, showRenderer, shelfRenderer, horizontalCardListRenderer, searchPyvRenderer
end end
# Extract items from the youtube initial data response
def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
author_id_fallback : String? = nil) author_id_fallback : String? = nil)
items = [] of SearchItem items = [] of SearchItem
@ -381,7 +396,7 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
ITEM_CONTAINER_EXTRACTOR.each do |extractor| ITEM_CONTAINER_EXTRACTOR.each do |extractor|
results = extractor.process(unpackaged_data) results = extractor.process(unpackaged_data)
if !results.nil? if !results.nil?
results.each do |item| results.contents.each do |item|
parsed_result = extract_item(item, author_fallback, author_id_fallback) parsed_result = extract_item(item, author_fallback, author_id_fallback)
if !parsed_result.nil? if !parsed_result.nil?
@ -394,3 +409,36 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
return items return items
end end
# Extract items from a container object
def extract_items(item_container : Containers, author_fallback : String? = nil,
author_id_fallback : String? = nil)
items = [] of SearchItem
# This is identicial to the parser cyling of extract_item().
item_container.contents.each do |item|
parsed_result = extract_item(item, author_fallback, author_id_fallback)
if !parsed_result.nil?
items << parsed_result
end
end
return items
end
def extract_item_container(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
author_id_fallback : String? = nil)
if unpackaged_data = initial_data["contents"]?.try &.as_h
elsif unpackaged_data = initial_data["response"]?.try &.as_h
elsif unpackaged_data = initial_data["onResponseReceivedActions"]?.try &.as_a.[0].as_h
else
unpackaged_data = initial_data
end
ITEM_CONTAINER_EXTRACTOR.each do |extractor|
results = extractor.process(unpackaged_data)
if !results.nil?
return results
end
end
end

View File

@ -228,6 +228,7 @@ struct SearchChannel
end end
end end
# Object representing a category of items from Youtube. Such as "artists on the rise", different sections on a channel homepage, etc.
class Category class Category
include DB::Serializable include DB::Serializable
@ -255,4 +256,29 @@ class Category
end end
end end
# Item containers.
# Object representing a YoutubeTab
struct YoutubeTab
include DB::Serializable
property contents : Array(JSON::Any)
# Useful in channel video and playlist tabs.
property content_filters : Array(Tuple(String, String)) # Name, browse param
end
struct SearchResults
include DB::Serializable
property contents : Array(JSON::Any)
end
struct ContinuationItems
include DB::Serializable
property contents : Array(JSON::Any)
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category
alias Containers = YoutubeTab | SearchResults | ContinuationItems