From 9b6d09c7b6e9e25a5bec5fc5ef5ba241c5047935 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 24 Jun 2021 17:25:03 -0700 Subject: [PATCH] Add objects to represent youtube item containers --- src/invidious/helpers/extractors.cr | 66 ++++++++++++++++++--- src/invidious/helpers/serialized_yt_data.cr | 26 ++++++++ 2 files changed, 83 insertions(+), 9 deletions(-) diff --git a/src/invidious/helpers/extractors.cr b/src/invidious/helpers/extractors.cr index 9da0bd6b..3d3d88ab 100644 --- a/src/invidious/helpers/extractors.cr +++ b/src/invidious/helpers/extractors.cr @@ -32,6 +32,10 @@ private class ItemParser def process(item : JSON::Any, author_fallback : AuthorFallback) end + def process(item : Containers, author_fallback) + return self.process(item.contents) + end + private def parse(item_contents : JSON::Any, author_fallback : AuthorFallback) end end @@ -251,11 +255,11 @@ private class CategoryParser < ItemParser end Category.new({ - title: title, - contents: contents, - description_html: description_html, - url: url, - badges: badges, + title: title, + contents: contents, + description_html: description_html, + url: url, + badges: badges, }) end end @@ -282,6 +286,8 @@ private class YoutubeTabsExtractor < ItemsContainerExtractor private def extract(target) raw_items = [] of JSON::Any + content_filters = [] of Tuple(String, String) + selected_tab = extract_selected_tab(target["tabs"]) content = selected_tab["content"] @@ -289,6 +295,14 @@ private class YoutubeTabsExtractor < ItemsContainerExtractor renderer_container = renderer_container["itemSectionRenderer"] renderer_container_contents = renderer_container["contents"].as_a[0] + submenu = renderer_container["subMenu"]?.try &.["channelSubMenuRenderer"]["contentTypeSubMenuItems"].as_a || nil + + if submenu + submenu.each do |option| + content_filters << {option["title"].as_s, option["endpoint"]["browseEndpoint"]["params"].as_s} + end + end + # Category extraction if items_container = renderer_container_contents["shelfRenderer"]? raw_items << renderer_container_contents @@ -306,7 +320,7 @@ private class YoutubeTabsExtractor < ItemsContainerExtractor end end - return raw_items + return YoutubeTab.new({contents: raw_items, content_filters: content_filters}) end end @@ -323,7 +337,7 @@ private class SearchResultsExtractor < ItemsContainerExtractor renderer = content["sectionListRenderer"]["contents"].as_a[0]["itemSectionRenderer"] raw_items = renderer["contents"].as_a - return raw_items + return SearchResults.new({contents: raw_items}) end end @@ -344,7 +358,7 @@ private class ContinuationExtractor < ItemsContainerExtractor raw_items = content.as_a end - return raw_items + return ContinuationItems.new({contents: raw_items}) end end @@ -366,6 +380,7 @@ def extract_item(item : JSON::Any, author_fallback : String? = nil, # TODO radioRenderer, showRenderer, shelfRenderer, horizontalCardListRenderer, searchPyvRenderer end +# Extract items from the youtube initial data response def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) items = [] of SearchItem @@ -381,7 +396,7 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri ITEM_CONTAINER_EXTRACTOR.each do |extractor| results = extractor.process(unpackaged_data) if !results.nil? - results.each do |item| + results.contents.each do |item| parsed_result = extract_item(item, author_fallback, author_id_fallback) if !parsed_result.nil? @@ -394,3 +409,36 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri return items end + +# Extract items from a container object +def extract_items(item_container : Containers, author_fallback : String? = nil, + author_id_fallback : String? = nil) + items = [] of SearchItem + # This is identicial to the parser cyling of extract_item(). + item_container.contents.each do |item| + parsed_result = extract_item(item, author_fallback, author_id_fallback) + + if !parsed_result.nil? + items << parsed_result + end + end + + return items +end + +def extract_item_container(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, + author_id_fallback : String? = nil) + if unpackaged_data = initial_data["contents"]?.try &.as_h + elsif unpackaged_data = initial_data["response"]?.try &.as_h + elsif unpackaged_data = initial_data["onResponseReceivedActions"]?.try &.as_a.[0].as_h + else + unpackaged_data = initial_data + end + + ITEM_CONTAINER_EXTRACTOR.each do |extractor| + results = extractor.process(unpackaged_data) + if !results.nil? + return results + end + end +end diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 9bb7e867..7f32c7b9 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -228,6 +228,7 @@ struct SearchChannel end end +# Object representing a category of items from Youtube. Such as "artists on the rise", different sections on a channel homepage, etc. class Category include DB::Serializable @@ -255,4 +256,29 @@ class Category end end +# Item containers. + +# Object representing a YoutubeTab +struct YoutubeTab + include DB::Serializable + + property contents : Array(JSON::Any) + + # Useful in channel video and playlist tabs. + property content_filters : Array(Tuple(String, String)) # Name, browse param +end + +struct SearchResults + include DB::Serializable + + property contents : Array(JSON::Any) +end + +struct ContinuationItems + include DB::Serializable + + property contents : Array(JSON::Any) +end + alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category +alias Containers = YoutubeTab | SearchResults | ContinuationItems