From c1a69e4a4a8b581ec743b7b3f741097d6596cb3b Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 16 Jul 2023 17:23:23 +0200 Subject: [PATCH] Channels: Use innertube to fetch the community tab --- src/invidious/channels/community.cr | 54 +++++++++----------------- src/invidious/yt_backend/extractors.cr | 26 ++++++++----- 2 files changed, 34 insertions(+), 46 deletions(-) diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index aac4bc8a..1a54a946 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -1,49 +1,31 @@ private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000} # TODO: Add "sort_by" -def fetch_channel_community(ucid, continuation, locale, format, thin_mode) - response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en") - if response.status_code != 200 - response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en") - end +def fetch_channel_community(ucid, cursor, locale, format, thin_mode) + if cursor.nil? + # Egljb21tdW5pdHk%3D is the protobuf object to load "community" + initial_data = YoutubeAPI.browse(ucid, params: "Egljb21tdW5pdHk%3D") - if response.status_code != 200 - raise NotFoundException.new("This channel does not exist.") - end - - ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"] - - if !continuation || continuation.empty? - initial_data = extract_initial_data(response.body) - body = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"] - - if !body - raise InfoException.new("Could not extract community tab.") + items = [] of JSON::Any + extract_items(initial_data) do |item| + items << item end else - continuation = produce_channel_community_continuation(ucid, continuation) + continuation = produce_channel_community_continuation(ucid, cursor) + initial_data = YoutubeAPI.browse(continuation: continuation) - headers = HTTP::Headers.new - headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] + container = initial_data.dig?("continuationContents", "itemSectionContinuation", "contents") - session_token = response.body.match(/"XSRF_TOKEN":"(?[^"]+)"/).try &.["session_token"]? || "" - post_req = { - session_token: session_token, - } + raise InfoException.new("Can't extract community data") if container.nil? - body = YoutubeAPI.browse(continuation) - - body = body.dig?("continuationContents", "itemSectionContinuation") || - body.dig?("continuationContents", "backstageCommentsContinuation") - - if !body - raise InfoException.new("Could not extract continuation.") - end + items = container.as_a end - posts = body["contents"].as_a + return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode) +end - if message = posts[0]["messageRenderer"]? +def extract_channel_community(items, *, ucid, locale, format, thin_mode) + if message = items[0]["messageRenderer"]? error_message = (message["text"]["simpleText"]? || message["text"]["runs"]?.try &.[0]?.try &.["text"]?) .try &.as_s || "" @@ -59,7 +41,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) json.field "authorId", ucid json.field "comments" do json.array do - posts.each do |post| + items.each do |post| comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? || post["backstageCommentsContinuation"]? @@ -242,7 +224,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode) end end end - if cont = posts.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") + if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") json.field "continuation", extract_channel_community_cursor(cont.as_s) end end diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index e5029dc5..8cf59d50 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -608,19 +608,25 @@ private module Extractors private def self.unpack_section_list(contents) raw_items = [] of JSON::Any - contents.as_a.each do |renderer_container| - renderer_container_contents = renderer_container["itemSectionRenderer"]["contents"][0] - - # Category extraction - if items_container = renderer_container_contents["shelfRenderer"]? - raw_items << renderer_container_contents - next - elsif items_container = renderer_container_contents["gridRenderer"]? + contents.as_a.each do |item| + if item_section_content = item.dig?("itemSectionRenderer", "contents") + raw_items += self.unpack_item_section(item_section_content) else - items_container = renderer_container_contents + raw_items << item end + end - items_container["items"]?.try &.as_a.each do |item| + return raw_items + end + + private def self.unpack_item_section(contents) + raw_items = [] of JSON::Any + + contents.as_a.each do |item| + # Category extraction + if container = item.dig?("gridRenderer", "items") || item.dig?("items") + raw_items += container.as_a + else raw_items << item end end