Merge 58d294fe3e into 53e8a5d62d

* Update to latest version WEB_CREATOR

* fix comment about using web embed as a fallback

.github/CODEOWNERS

@@ -6,7 +6,7 @@ docker/ @unixfox
 kubernetes/ @unixfox
 
 README.md @thefrenchghosty
-config/config.example.yml @thefrenchghosty @SamantazFox @unixfox
+config/config.example.yml @SamantazFox @unixfox
 
 scripts/ @syeopite
 shards.lock @syeopite

src/invidious/hashtag.cr

@@ -1,42 +1,89 @@
 module Invidious::Hashtag
   extend self
 
-  def fetch(hashtag : String, page : Int, region : String? = nil) : Array(SearchItem)
+  struct HashtagPage
+    include DB::Serializable
+
+    property videos : Array(SearchItem) | Array(Video)
+    property header : SearchHashtag?
+    property has_next_continuation : Bool
+
+    def to_json(locale : String?, json : JSON::Builder)
+      json.object do
+        json.field "type", "hashtagPage"
+        if self.header != nil
+          json.field "header" do
+            self.header.try &.as(SearchHashtag).to_json(locale, json)
+          end
+        end
+        json.field "results" do
+          json.array do
+            self.videos.each do |item|
+              item.to_json(locale, json)
+            end
+          end
+        end
+        json.field "hasNextPage", self.has_next_continuation
+      end
+    end
+  end
+
+  def fetch(hashtag : String, page : Int, region : String? = nil) : HashtagPage
     cursor = (page - 1) * 60
-    ctoken = generate_continuation(hashtag, cursor)
-
+    header = nil
     client_config = YoutubeAPI::ClientConfig.new(region: region)
-    response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config)
+    item = generate_continuation(hashtag, cursor)
+    # item is a ctoken
+    if cursor > 0
+      response = YoutubeAPI.browse(continuation: item, client_config: client_config)
+    else
+      # item browses the first page (including metadata)
+      response = YoutubeAPI.browse("FEhashtag", params: item, client_config: client_config)
+      if item_contents = response.dig?("header")
+        header = parse_item(item_contents).try &.as(SearchHashtag)
+      end
+    end
 
-    items, _ = extract_items(response)
-    return items
+    items, next_continuation = extract_items(response)
+    return HashtagPage.new({
+      videos:                items,
+      header:                header,
+      has_next_continuation: next_continuation != nil,
+    })
   end
 
   def generate_continuation(hashtag : String, cursor : Int)
     object = {
-      "80226972:embedded" => {
-        "2:string" => "FEhashtag",
-        "3:base64" => {
-          "1:varint"  => 60_i64, # result count
-          "15:base64" => {
-            "1:varint" => cursor.to_i64,
-            "2:varint" => 0_i64,
-          },
-          "93:2:embedded" => {
-            "1:string" => hashtag,
-            "2:varint" => 0_i64,
-            "3:varint" => 1_i64,
-          },
-        },
-        "35:string" => "browse-feedFEhashtag",
+      "93:2:embedded" => {
+        "1:string" => hashtag,
+        "2:varint" => 0_i64,
+        "3:varint" => 1_i64,
       },
     }
+    if cursor > 0
+      object = {
+        "80226972:embedded" => {
+          "2:string" => "FEhashtag",
+          "3:base64" => {
+            "1:varint"  => 60_i64, # result count
+            "15:base64" => {
+              "1:varint" => cursor.to_i64,
+              "2:varint" => 0_i64,
+            },
+            "93:2:embedded" => {
+              "1:string" => hashtag,
+              "2:varint" => 0_i64,
+              "3:varint" => 1_i64,
+            },
+          },
+          "35:string" => "browse-feedFEhashtag",
+        },
+      }
+    end
 
-    continuation = object.try { |i| Protodec::Any.cast_json(i) }
+    return object.try { |i| Protodec::Any.cast_json(i) }
       .try { |i| Protodec::Any.from_json(i) }
       .try { |i| Base64.urlsafe_encode(i) }
       .try { |i| URI.encode_www_form(i) }
-
-    return continuation
   end
 end

src/invidious/routes/api/v1/search.cr

@@ -69,21 +69,13 @@ module Invidious::Routes::API::V1::Search
     env.response.content_type = "application/json"
 
     begin
-      results = Invidious::Hashtag.fetch(hashtag, page, region)
+      hashtag_page = Invidious::Hashtag.fetch(hashtag, page, region)
     rescue ex
       return error_json(400, ex)
     end
 
     JSON.build do |json|
-      json.object do
-        json.field "results" do
-          json.array do
-            results.each do |item|
-              item.to_json(locale, json)
-            end
-          end
-        end
-      end
+      hashtag_page.to_json(locale, json)
     end
   end
 end

src/invidious/routes/search.cr

@@ -101,7 +101,8 @@ module Invidious::Routes::Search
     end
 
     begin
-      items = Invidious::Hashtag.fetch(hashtag, page)
+      hashtag_page = Invidious::Hashtag.fetch(hashtag, page)
+      items = hashtag_page.videos
     rescue ex
       return error_template(500, ex)
     end
@@ -111,7 +112,7 @@ module Invidious::Routes::Search
     page_nav_html = Frontend::Pagination.nav_numeric(locale,
       base_url: "/hashtag/#{hashtag_encoded}",
       current_page: page,
-      show_next: (items.size >= 60)
+      show_next: hashtag_page.has_next_continuation
     )
 
     templated "hashtag"

src/invidious/videos/parser.cr

@@ -53,6 +53,10 @@ end
 def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
+  # Use the WEB_CREATOR when po_token is configured because it fully only works on this client
+  if CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebCreator
+  end
 
   # Fetch data from the player endpoint
   player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
@@ -102,6 +106,13 @@ def extract_video_info(video_id : String)
 
   new_player_response = nil
 
+  # Second try in case WEB_CREATOR doesn't work with po_token.
+  # Only trigger if reason found and po_token configured.
+  if reason && CONFIG.po_token
+    client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer
+    new_player_response = try_fetch_streaming_data(video_id, client_config)
+  end
+
   # Don't use Android client if po_token is passed because po_token doesn't
   # work for Android client.
   if reason.nil? && CONFIG.po_token.nil?
@@ -114,10 +125,9 @@ def extract_video_info(video_id : String)
   end
 
   # Last hope
-  # Only trigger if reason found and po_token or didn't work wth Android client.
-  # TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
-  # if the IP address is not blocked.
-  if CONFIG.po_token && reason || CONFIG.po_token.nil? && new_player_response.nil?
+  # Only trigger if reason found or didn't work wth Android client.
+  # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token.
+  if reason && CONFIG.po_token.nil?
     client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   end
@@ -185,10 +195,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   end
 
   video_details = player_response.dig?("videoDetails")
-  microformat = player_response.dig?("microformat", "playerMicroformatRenderer")
+  if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
+    microformat = {} of String => JSON::Any
+  end
 
   raise BrokenTubeException.new("videoDetails") if !video_details
-  raise BrokenTubeException.new("microformat") if !microformat
 
   # Basic video infos
 
@@ -225,7 +236,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     .try &.as_a.map &.as_s || [] of String
 
   allow_ratings = video_details["allowRatings"]?.try &.as_bool
-  family_friendly = microformat["isFamilySafe"].try &.as_bool
+  family_friendly = microformat["isFamilySafe"]?.try &.as_bool
   is_listed = video_details["isCrawlable"]?.try &.as_bool
   is_upcoming = video_details["isUpcoming"]?.try &.as_bool
 

src/invidious/yt_backend/extractors.cr

@@ -217,15 +217,18 @@ private module Parsers
   #
   # A `hashtagTileRenderer` is a kind of search result.
   # It can be found when searching for any hashtag (e.g "#hi" or "#shorts")
+  #
+  # A `hashtagHeaderRenderer` is displayed on the first page of the hashtag page.
   module HashtagRendererParser
     def self.process(item : JSON::Any, author_fallback : AuthorFallback)
-      if item_contents = item["hashtagTileRenderer"]?
+      if item_contents = (item["hashtagTileRenderer"]? || item["hashtagHeaderRenderer"]? || item["pageHeaderRenderer"]?)
         return self.parse(item_contents)
       end
     end
 
     private def self.parse(item_contents)
-      title = extract_text(item_contents["hashtag"]).not_nil! # E.g "#hi"
+      title = item_contents.dig?("pageTitle").try &.as_s
+      title ||= extract_text(item_contents["hashtag"]).not_nil! # E.g "#hi"
 
       # E.g "/hashtag/hi"
       url = item_contents.dig?("onTapCommand", "commandMetadata", "webCommandMetadata", "url").try &.as_s
@@ -236,8 +239,10 @@ private module Parsers
 
       # Fallback for video/channel counts
       if channel_count_txt.nil? || video_count_txt.nil?
+        info_text = (item_contents.dig?("content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows", 0, "metadataParts", 0, "text", "content").try &.as_s ||
+                     extract_text(item_contents.dig?("hashtagInfoText"))).try &.split(" • ")
+
         # E.g: "203K videos • 81K channels"
-        info_text = extract_text(item_contents["hashtagInfoText"]?).try &.split(" • ")
 
         if info_text && info_text.size == 2
           video_count_txt ||= info_text[0]

src/invidious/yt_backend/youtube_api.cr

@@ -29,6 +29,7 @@ module YoutubeAPI
     WebEmbeddedPlayer
     WebMobile
     WebScreenEmbed
+    WebCreator
 
     Android
     AndroidEmbeddedPlayer
@@ -80,6 +81,14 @@ module YoutubeAPI
       os_version: WINDOWS_VERSION,
       platform:   "DESKTOP",
     },
+    ClientType::WebCreator => {
+      name:       "WEB_CREATOR",
+      name_proto: "62",
+      version:    "1.20240918.03.00",
+      os_name:    "Windows",
+      os_version: WINDOWS_VERSION,
+      platform:   "DESKTOP",
+    },
 
     # Android
 
@@ -291,8 +300,9 @@ module YoutubeAPI
     end
 
     if client_config.screen == "EMBED"
+      # embedUrl https://www.google.com allow loading almost all video that are configured not embeddable
       client_context["thirdParty"] = {
-        "embedUrl" => "https://www.youtube.com/embed/#{video_id}",
+        "embedUrl" => "https://www.google.com/",
       } of String => String | Int64
     end