Channels: Use innertube to fetch the community tab (#3988)

src/invidious/channels/community.cr

@@ -1,49 +1,31 @@
 private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000}
 
 # TODO: Add "sort_by"
-def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
+def fetch_channel_community(ucid, cursor, locale, format, thin_mode)
-  response = YT_POOL.client &.get("/channel/#{ucid}/community?gl=US&hl=en")
+  if cursor.nil?
-  if response.status_code != 200
+    # Egljb21tdW5pdHk%3D is the protobuf object to load "community"
-    response = YT_POOL.client &.get("/user/#{ucid}/community?gl=US&hl=en")
+    initial_data = YoutubeAPI.browse(ucid, params: "Egljb21tdW5pdHk%3D")
-  end
 
-  if response.status_code != 200
+    items = [] of JSON::Any
-    raise NotFoundException.new("This channel does not exist.")
+    extract_items(initial_data) do |item|
-  end
+      items << item
-
-  ucid = response.body.match(/https:\/\/www.youtube.com\/channel\/(?<ucid>UC[a-zA-Z0-9_-]{22})/).not_nil!["ucid"]
-
-  if !continuation || continuation.empty?
-    initial_data = extract_initial_data(response.body)
-    body = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]
-
-    if !body
-      raise InfoException.new("Could not extract community tab.")
     end
   else
-    continuation = produce_channel_community_continuation(ucid, continuation)
+    continuation = produce_channel_community_continuation(ucid, cursor)
+    initial_data = YoutubeAPI.browse(continuation: continuation)
 
-    headers = HTTP::Headers.new
+    container = initial_data.dig?("continuationContents", "itemSectionContinuation", "contents")
-    headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"]
 
-    session_token = response.body.match(/"XSRF_TOKEN":"(?<session_token>[^"]+)"/).try &.["session_token"]? || ""
+    raise InfoException.new("Can't extract community data") if container.nil?
-    post_req = {
-      session_token: session_token,
-    }
 
-    body = YoutubeAPI.browse(continuation)
+    items = container.as_a
-
-    body = body.dig?("continuationContents", "itemSectionContinuation") ||
-           body.dig?("continuationContents", "backstageCommentsContinuation")
-
-    if !body
-      raise InfoException.new("Could not extract continuation.")
-    end
   end
 
-  posts = body["contents"].as_a
+  return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode)
+end
 
-  if message = posts[0]["messageRenderer"]?
+def extract_channel_community(items, *, ucid, locale, format, thin_mode)
+  if message = items[0]["messageRenderer"]?
     error_message = (message["text"]["simpleText"]? ||
                      message["text"]["runs"]?.try &.[0]?.try &.["text"]?)
       .try &.as_s || ""
@@ -59,7 +41,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
       json.field "authorId", ucid
       json.field "comments" do
         json.array do
-          posts.each do |post|
+          items.each do |post|
             comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? ||
                        post["backstageCommentsContinuation"]?
 
@@ -242,7 +224,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
           end
         end
       end
-      if cont = posts.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token")
+      if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token")
         json.field "continuation", extract_channel_community_cursor(cont.as_s)
       end
     end

src/invidious/yt_backend/extractors.cr

@@ -608,19 +608,25 @@ private module Extractors
     private def self.unpack_section_list(contents)
       raw_items = [] of JSON::Any
 
-      contents.as_a.each do |renderer_container|
+      contents.as_a.each do |item|
-        renderer_container_contents = renderer_container["itemSectionRenderer"]["contents"][0]
+        if item_section_content = item.dig?("itemSectionRenderer", "contents")
-
+          raw_items += self.unpack_item_section(item_section_content)
-        # Category extraction
-        if items_container = renderer_container_contents["shelfRenderer"]?
-          raw_items << renderer_container_contents
-          next
-        elsif items_container = renderer_container_contents["gridRenderer"]?
         else
-          items_container = renderer_container_contents
+          raw_items << item
         end
+      end
 
-        items_container["items"]?.try &.as_a.each do |item|
+      return raw_items
+    end
+
+    private def self.unpack_item_section(contents)
+      raw_items = [] of JSON::Any
+
+      contents.as_a.each do |item|
+        # Category extraction
+        if container = item.dig?("gridRenderer", "items") || item.dig?("items")
+          raw_items += container.as_a
+        else
           raw_items << item
         end
       end