From 43bd331e48ad1a19cd3c7a6d5beb72e3127c5edc Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Thu, 1 Apr 2021 02:36:43 +0000
Subject: [PATCH] Multiple youtube_api.cr helper fixes
Add documentation
Bump web client version string
Add charset=UTF-8 to the 'content-type' header
Parse JSON and return it as a Hash
Handle API error messages
---
src/invidious/channels.cr | 40 +++++++---------------------
src/invidious/helpers/youtube_api.cr | 29 +++++++++++++++++---
src/invidious/playlists.cr | 2 +-
src/invidious/search.cr | 3 +--
4 files changed, 36 insertions(+), 38 deletions(-)
diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr
index 3109b508..bbef3d4f 100644
--- a/src/invidious/channels.cr
+++ b/src/invidious/channels.cr
@@ -229,22 +229,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
page = 1
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
- response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
-
- videos = [] of SearchVideo
- begin
- initial_data = JSON.parse(response_body)
- raise InfoException.new("Could not extract channel JSON") if !initial_data
-
- LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel videos page initial_data")
- videos = extract_videos(initial_data.as_h, author, ucid)
- rescue ex
- if response_body.includes?("To continue with your YouTube experience, please fill out the form below.") ||
- response_body.includes?("https://www.google.com/sorry/index")
- raise InfoException.new("Could not extract channel info. Instance is likely blocked.")
- end
- raise ex
- end
+ initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
+ videos = extract_videos(initial_data, author, ucid)
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
rss.xpath_nodes("//feed/entry").each do |entry|
@@ -304,10 +290,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
ids = [] of String
loop do
- response_body = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
- initial_data = JSON.parse(response_body)
- raise InfoException.new("Could not extract channel JSON") if !initial_data
- videos = extract_videos(initial_data.as_h, author, ucid)
+ initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
+ videos = extract_videos(initial_data, author, ucid)
count = videos.size
videos = videos.map { |video| ChannelVideo.new({
@@ -358,8 +342,7 @@ end
def fetch_channel_playlists(ucid, author, continuation, sort_by)
if continuation
response_json = request_youtube_api_browse(continuation)
- result = JSON.parse(response_json)
- continuationItems = result["onResponseReceivedActions"]?
+ continuationItems = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return [] of SearchItem, nil if !continuationItems
@@ -964,21 +947,16 @@ def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
videos = [] of SearchVideo
2.times do |i|
- response_json = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
- initial_data = JSON.parse(response_json)
- break if !initial_data
- videos.concat extract_videos(initial_data.as_h, author, ucid)
+ initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
+ videos.concat extract_videos(initial_data, author, ucid)
end
return videos.size, videos
end
def get_latest_videos(ucid)
- response_json = get_channel_videos_response(ucid)
- initial_data = JSON.parse(response_json)
- return [] of SearchVideo if !initial_data
+ initial_data = get_channel_videos_response(ucid)
author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
- items = extract_videos(initial_data.as_h, author, ucid)
- return items
+ return extract_videos(initial_data, author, ucid)
end
diff --git a/src/invidious/helpers/youtube_api.cr b/src/invidious/helpers/youtube_api.cr
index 30413532..84e0c38f 100644
--- a/src/invidious/helpers/youtube_api.cr
+++ b/src/invidious/helpers/youtube_api.cr
@@ -4,8 +4,18 @@
# Hard-coded constants required by the API
HARDCODED_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
-HARDCODED_CLIENT_VERS = "2.20210318.08.00"
+HARDCODED_CLIENT_VERS = "2.20210330.08.00"
+####################################################################
+# request_youtube_api_browse(continuation)
+#
+# Requests the youtubei/vi/browse endpoint with the required headers
+# to get JSON in en-US (english US).
+#
+# The requested data is a continuation token (ctoken). Depending on
+# this token's contents, the returned data can be comments, playlist
+# videos, search results, channel community tab, ...
+#
def request_youtube_api_browse(continuation)
# JSON Request data, required by the API
data = {
@@ -20,12 +30,23 @@ def request_youtube_api_browse(continuation)
"continuation": continuation,
}
- # Send the POST request and return result
+ # Send the POST request and parse result
response = YT_POOL.client &.post(
"/youtubei/v1/browse?key=#{HARDCODED_API_KEY}",
- headers: HTTP::Headers{"content-type" => "application/json"},
+ headers: HTTP::Headers{"content-type" => "application/json; charset=UTF-8"},
body: data.to_json
)
- return response.body
+ initial_data = JSON.parse(response.body).as_h
+
+ # Error handling
+ if initial_data.has_key?("error")
+ code = initial_data["error"]["code"]
+ message = initial_data["error"]["message"].to_s.sub(/(\\n)+\^$/, "")
+
+ raise InfoException.new("Could not extract JSON. Youtube API returned \
+ error #{code} with message:<br>\"#{message}\"")
+ end
+
+ return initial_data
end
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr
index 073a9986..150f1c15 100644
--- a/src/invidious/playlists.cr
+++ b/src/invidious/playlists.cr
@@ -451,7 +451,7 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
offset = (offset / 100).to_i64 * 100_i64
ctoken = produce_playlist_continuation(playlist.id, offset)
- initial_data = JSON.parse(request_youtube_api_browse(ctoken)).as_h
+ initial_data = request_youtube_api_browse(ctoken)
else
response = YT_POOL.client &.get("/playlist?list=#{playlist.id}&gl=US&hl=en")
initial_data = extract_initial_data(response.body)
diff --git a/src/invidious/search.cr b/src/invidious/search.cr
index 4b216613..7c9c389e 100644
--- a/src/invidious/search.cr
+++ b/src/invidious/search.cr
@@ -246,8 +246,7 @@ def channel_search(query, page, channel)
continuation = produce_channel_search_continuation(ucid, query, page)
response_json = request_youtube_api_browse(continuation)
- result = JSON.parse(response_json)
- continuationItems = result["onResponseReceivedActions"]?
+ continuationItems = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return 0, [] of SearchItem if !continuationItems