Merge pull request #1985 from SamantazFox/improve-youtube-api-helper

Improve youtube api calls

src/invidious/channels.cr

@@ -229,22 +229,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
   page = 1
 
   LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
-  response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
+  initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
-
+  videos = extract_videos(initial_data, author, ucid)
-  videos = [] of SearchVideo
-  begin
-    initial_data = JSON.parse(response_body)
-    raise InfoException.new("Could not extract channel JSON") if !initial_data
-
-    LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel videos page initial_data")
-    videos = extract_videos(initial_data.as_h, author, ucid)
-  rescue ex
-    if response_body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
-       response_body.includes?("https://www.google.com/sorry/index")
-      raise InfoException.new("Could not extract channel info. Instance is likely blocked.")
-    end
-    raise ex
-  end
 
   LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
   rss.xpath_nodes("//feed/entry").each do |entry|
@@ -304,10 +290,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
     ids = [] of String
 
     loop do
-      response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
+      initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
-      initial_data = JSON.parse(response_body)
+      videos = extract_videos(initial_data, author, ucid)
-      raise InfoException.new("Could not extract channel JSON") if !initial_data
-      videos = extract_videos(initial_data.as_h, author, ucid)
 
       count = videos.size
       videos = videos.map { |video| ChannelVideo.new({
@@ -358,8 +342,7 @@ end
 def fetch_channel_playlists(ucid, author, continuation, sort_by)
   if continuation
     response_json = request_youtube_api_browse(continuation)
-    result = JSON.parse(response_json)
+    continuationItems = response_json["onResponseReceivedActions"]?
-    continuationItems = result["onResponseReceivedActions"]?
       .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
 
     return [] of SearchItem, nil if !continuationItems
@@ -964,21 +947,16 @@ def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
   videos = [] of SearchVideo
 
   2.times do |i|
-    response_json = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
+    initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
-    initial_data = JSON.parse(response_json)
+    videos.concat extract_videos(initial_data, author, ucid)
-    break if !initial_data
-    videos.concat extract_videos(initial_data.as_h, author, ucid)
   end
 
   return videos.size, videos
 end
 
 def get_latest_videos(ucid)
-  response_json = get_channel_videos_response(ucid)
+  initial_data = get_channel_videos_response(ucid)
-  initial_data = JSON.parse(response_json)
-  return [] of SearchVideo if !initial_data
   author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
-  items = extract_videos(initial_data.as_h, author, ucid)
 
-  return items
+  return extract_videos(initial_data, author, ucid)
 end

src/invidious/helpers/youtube_api.cr

@@ -4,28 +4,116 @@
 
 # Hard-coded constants required by the API
 HARDCODED_API_KEY     = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
-HARDCODED_CLIENT_VERS = "2.20210318.08.00"
+HARDCODED_CLIENT_VERS = "2.20210330.08.00"
 
-def request_youtube_api_browse(continuation)
+####################################################################
+# make_youtube_api_context(region)
+#
+# Return, as a Hash, the "context" data required to request the
+# youtube API endpoints.
+#
+def make_youtube_api_context(region : String | Nil) : Hash
+  return {
+    "client" => {
+      "hl"            => "en",
+      "gl"            => region || "US", # Can't be empty!
+      "clientName"    => "WEB",
+      "clientVersion" => HARDCODED_CLIENT_VERS,
+    },
+  }
+end
+
+####################################################################
+# request_youtube_api_browse(continuation)
+# request_youtube_api_browse(browse_id, params)
+#
+# Requests the youtubei/v1/browse endpoint with the required headers
+# and POST data in order to get a JSON reply in english US that can
+# be easily parsed.
+#
+# The requested data can either be:
+#
+#  - A continuation token (ctoken). Depending on this token's
+#    contents, the returned data can be comments, playlist videos,
+#    search results, channel community tab, ...
+#
+#  - A playlist ID (parameters MUST be an empty string)
+#
+def request_youtube_api_browse(continuation : String)
   # JSON Request data, required by the API
   data = {
-    "context": {
+    "context"      => make_youtube_api_context("US"),
-      "client": {
+    "continuation" => continuation,
-        "hl":            "en",
-        "gl":            "US",
-        "clientName":    "WEB",
-        "clientVersion": HARDCODED_CLIENT_VERS,
-      },
-    },
-    "continuation": continuation,
   }
 
-  # Send the POST request and return result
+  return _youtube_api_post_json("/youtubei/v1/browse", data)
+end
+
+def request_youtube_api_browse(browse_id : String, params : String)
+  # JSON Request data, required by the API
+  data = {
+    "browseId" => browse_id,
+    "context"  => make_youtube_api_context("US"),
+  }
+
+  # Append the additionnal parameters if those were provided
+  # (this is required for channel info, playlist and community, e.g)
+  if params != ""
+    data["params"] = params
+  end
+
+  return _youtube_api_post_json("/youtubei/v1/browse", data)
+end
+
+####################################################################
+# request_youtube_api_search(search_query, params, region)
+#
+# Requests the youtubei/v1/search endpoint with the required headers
+# and POST data in order to get a JSON reply. As the search results
+# vary depending on the region, a region code can be specified in
+# order to get non-US results.
+#
+# The requested data is a search string, with some additional
+# paramters, formatted as a base64 string.
+#
+def request_youtube_api_search(search_query : String, params : String, region = nil)
+  # JSON Request data, required by the API
+  data = {
+    "query"   => URI.encode_www_form(search_query),
+    "context" => make_youtube_api_context(region),
+    "params"  => params,
+  }
+
+  return _youtube_api_post_json("/youtubei/v1/search", data)
+end
+
+####################################################################
+# _youtube_api_post_json(endpoint, data)
+#
+# Internal function that does the actual request to youtube servers
+# and handles errors.
+#
+# The requested data is an endpoint (URL without the domain part)
+# and the data as a Hash object.
+#
+def _youtube_api_post_json(endpoint, data)
+  # Send the POST request and parse result
   response = YT_POOL.client &.post(
-    "/youtubei/v1/browse?key=#{HARDCODED_API_KEY}",
+    "#{endpoint}?key=#{HARDCODED_API_KEY}",
-    headers: HTTP::Headers{"content-type" => "application/json"},
+    headers: HTTP::Headers{"content-type" => "application/json; charset=UTF-8"},
     body: data.to_json
   )
 
-  return response.body
+  initial_data = JSON.parse(response.body).as_h
+
+  # Error handling
+  if initial_data.has_key?("error")
+    code = initial_data["error"]["code"]
+    message = initial_data["error"]["message"].to_s.sub(/(\\n)+\^$/, "")
+
+    raise InfoException.new("Could not extract JSON. Youtube API returned \
+      error #{code} with message:<br>\"#{message}\"")
+  end
+
+  return initial_data
 end

src/invidious/playlists.cr

@@ -361,16 +361,7 @@ def fetch_playlist(plid, locale)
     plid = "UU#{plid.lchop("UC")}"
   end
 
-  response = YT_POOL.client &.get("/playlist?list=#{plid}&hl=en")
+  initial_data = request_youtube_api_browse("VL" + plid, params: "")
-  if response.status_code != 200
-    if response.headers["location"]?.try &.includes? "/sorry/index"
-      raise InfoException.new("Could not extract playlist info. Instance is likely blocked.")
-    else
-      raise InfoException.new("Not a playlist.")
-    end
-  end
-
-  initial_data = extract_initial_data(response.body)
 
   playlist_sidebar_renderer = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?
   raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer
@@ -451,17 +442,12 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
       offset = (offset / 100).to_i64 * 100_i64
 
       ctoken = produce_playlist_continuation(playlist.id, offset)
-      initial_data = JSON.parse(request_youtube_api_browse(ctoken)).as_h
+      initial_data = request_youtube_api_browse(ctoken)
     else
-      response = YT_POOL.client &.get("/playlist?list=#{playlist.id}&gl=US&hl=en")
+      initial_data = request_youtube_api_browse("VL" + playlist.id, params: "")
-      initial_data = extract_initial_data(response.body)
     end
 
-    if initial_data
+    return extract_playlist_videos(initial_data)
-      return extract_playlist_videos(initial_data)
-    else
-      return [] of PlaylistVideo
-    end
   end
 end
 

src/invidious/search.cr

@@ -246,8 +246,7 @@ def channel_search(query, page, channel)
   continuation = produce_channel_search_continuation(ucid, query, page)
   response_json = request_youtube_api_browse(continuation)
 
-  result = JSON.parse(response_json)
+  continuationItems = response_json["onResponseReceivedActions"]?
-  continuationItems = result["onResponseReceivedActions"]?
     .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
 
   return 0, [] of SearchItem if !continuationItems
@@ -264,14 +263,9 @@ end
 def search(query, search_params = produce_search_params(content_type: "all"), region = nil)
   return 0, [] of SearchItem if query.empty?
 
-  body = YT_POOL.client(region, &.get("/results?search_query=#{URI.encode_www_form(query)}&sp=#{search_params}&hl=en").body)
+  initial_data = request_youtube_api_search(query, search_params, region)
-  return 0, [] of SearchItem if body.empty?
-
-  initial_data = extract_initial_data(body)
   items = extract_items(initial_data)
 
-  # initial_data["estimatedResults"]?.try &.as_s.to_i64
-
   return items.size, items
 end